Mercurial
diff infinite_canvas/web_surface_native.cc @ 280:49e9e591c9bb
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Tue, 18 Aug 2026 19:14:53 -0700 |
| parents | 8d560f50ed4c |
| children |
line wrap: on
line diff
--- a/infinite_canvas/web_surface_native.cc Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface_native.cc Tue Aug 18 19:14:53 2026 -0700 @@ -6,16 +6,20 @@ #include "include/cef_browser.h" #include "include/cef_client.h" #include "include/cef_display_handler.h" +#include "include/cef_permission_handler.h" #include "include/cef_render_handler.h" +#include "include/cef_request_context.h" #include "include/capi/cef_app_capi.h" #include <algorithm> +#include <cctype> #include <cmath> #include <cstdio> #include <cstdlib> #include <cstring> #include <filesystem> #include <new> +#include <string> #if defined(OS_LINUX) #include <dlfcn.h> @@ -29,6 +33,8 @@ constexpr int32 kMinTextureHeight = 120; constexpr int32 kMaxTextureWidth = 1280; constexpr int32 kMaxTextureHeight = 800; +constexpr uint32 kDictationViewId = 0xffffffffu; +constexpr const char *kDictationUrl = "http://127.0.0.1:8090/?canvas=1"; int g_argc = 0; char **g_argv = nullptr; char *g_cef_argv[128] = {}; @@ -41,6 +47,26 @@ void FindResourcePaths(); +void PercentDecode(const char *source, char *target, size_t target_size) { + size_t output = 0; + for (size_t index = 0; + source[index] && output + 1 < target_size; + ++index) { + if (source[index] == '%' && + std::isxdigit(static_cast<unsigned char>(source[index + 1])) && + std::isxdigit(static_cast<unsigned char>(source[index + 2]))) { + char hex[3] = {source[index + 1], source[index + 2], '\0'}; + target[output++] = static_cast<char>(std::strtol(hex, nullptr, 16)); + index += 2; + } else if (source[index] == '+') { + target[output++] = ' '; + } else { + target[output++] = source[index]; + } + } + target[output] = '\0'; +} + void ApplyColorScheme(CefRefPtr<CefBrowser> browser, boolean dark_mode) { if (!browser) return; CefRefPtr<CefDictionaryValue> params = CefDictionaryValue::Create(); @@ -62,6 +88,12 @@ frame->ExecuteJavaScript(script, frame->GetURL(), 0); } +boolean IsLocalDictationOrigin(const CefString &requesting_origin) { + std::string origin = requesting_origin.ToString(); + return origin.rfind("http://127.0.0.1:8090", 0) == 0 || + origin.rfind("http://localhost:8090", 0) == 0; +} + const Canvas_Entity *FindSurfaceEntity( const Canvas_Scene *scene, uint32 entity_id) { @@ -167,12 +199,16 @@ class CanvasCefClient : public CefClient, public CefRenderHandler, public CefDisplayHandler, + public CefPermissionHandler, public CefLifeSpanHandler { public: explicit CanvasCefClient(Canvas_Web_View *view) : view_(view) {} CefRefPtr<CefRenderHandler> GetRenderHandler() override { return this; } CefRefPtr<CefDisplayHandler> GetDisplayHandler() override { return this; } + CefRefPtr<CefPermissionHandler> GetPermissionHandler() override { + return this; + } CefRefPtr<CefLifeSpanHandler> GetLifeSpanHandler() override { return this; } void OnAddressChange( @@ -189,6 +225,83 @@ view_->can_go_forward = browser->CanGoForward() ? TRUE : FALSE; } + void OnTitleChange( + CefRefPtr<CefBrowser> browser, + const CefString &title) override { + (void)browser; + std::string value = title.ToString(); + constexpr const char *prefix = "Zenbu Dictation|"; + if (value.rfind(prefix, 0) != 0) return; + const char *kind = std::strchr( + value.c_str() + std::strlen(prefix), + '|'); + if (!kind) return; + kind++; + const char *encoded = std::strchr(kind, '|'); + if (!encoded) return; + std::string event_kind(kind, static_cast<size_t>(encoded - kind)); + if (event_kind == "status") { + view_->dictation_event = CANVAS_DICTATION_EVENT_STATUS; + } else if (event_kind == "partial") { + view_->dictation_event = CANVAS_DICTATION_EVENT_PARTIAL; + } else if (event_kind == "final") { + view_->dictation_event = CANVAS_DICTATION_EVENT_FINAL; + } else { + return; + } + PercentDecode( + encoded + 1, + view_->dictation_text, + sizeof(view_->dictation_text)); + view_->dictation_sequence++; + } + + bool OnRequestMediaAccessPermission( + CefRefPtr<CefBrowser> browser, + CefRefPtr<CefFrame> frame, + const CefString &requesting_origin, + uint32_t requested_permissions, + CefRefPtr<CefMediaAccessCallback> callback) override { + (void)browser; + (void)frame; + boolean local_dictation = + view_->is_dictation_transport && + IsLocalDictationOrigin(requesting_origin); + uint32_t audio = CEF_MEDIA_PERMISSION_DEVICE_AUDIO_CAPTURE; + boolean requests_audio = (requested_permissions & audio) != 0; + boolean requests_other_media = (requested_permissions & ~audio) != 0; + if (local_dictation && requests_audio && !requests_other_media) { + callback->Continue(audio); + } else { + callback->Cancel(); + } + return true; + } + + bool OnShowPermissionPrompt( + CefRefPtr<CefBrowser> browser, + uint64_t prompt_id, + const CefString &requesting_origin, + uint32_t requested_permissions, + CefRefPtr<CefPermissionPromptCallback> callback) override { + (void)browser; + (void)prompt_id; + if (!view_->is_dictation_transport || + !IsLocalDictationOrigin(requesting_origin)) { + return false; + } + uint32_t microphone = CEF_PERMISSION_TYPE_MIC_STREAM; + boolean requests_microphone = + (requested_permissions & microphone) != 0; + boolean requests_other_permissions = + (requested_permissions & ~microphone) != 0; + callback->Continue( + requests_microphone && !requests_other_permissions + ? CEF_PERMISSION_RESULT_ACCEPT + : CEF_PERMISSION_RESULT_DENY); + return true; + } + void GetViewRect(CefRefPtr<CefBrowser> browser, CefRect &rect) override { (void)browser; rect = CefRect(0, 0, view_->texture_width, view_->texture_height); @@ -243,9 +356,17 @@ struct NativeState { CefRefPtr<CanvasCefClient> clients[CANVAS_MAX_WEB_VIEWS]; + CefRefPtr<CefRequestContext> dictation_context; boolean cef_initialized; }; +void EnsureDictationRequestContext(NativeState *state) { + if (state->dictation_context) return; + CefRequestContextSettings settings; + state->dictation_context = + CefRequestContext::CreateContext(settings, nullptr); +} + CefMainArgs MainArgs(int argc, char **argv) { #if defined(OS_WIN) (void)argc; @@ -396,13 +517,18 @@ : CefColorSetARGB(255, 255, 255, 255); const char *initial_url = view->entity_type == CANVAS_ENTITY_IMAGE ? "about:blank" : url; + CefRefPtr<CefRequestContext> request_context; + if (view->is_dictation_transport) { + EnsureDictationRequestContext(state); + request_context = state->dictation_context; + } if (!CefBrowserHost::CreateBrowser( window_info, state->clients[index], initial_url, browser_settings, nullptr, - nullptr)) { + request_context)) { state->clients[index] = nullptr; return FALSE; } @@ -410,6 +536,47 @@ return TRUE; } +Canvas_Web_View *EnsureDictationView(Canvas_Web_Surface *surface) { + Canvas_Web_View *view = FindView(surface, kDictationViewId); + if (view) { + view->active = TRUE; + return view; + } + view = FindInactiveView(surface); + if (!view) return nullptr; + boolean browser_created = view->browser_created; + view->active = TRUE; + view->entity_id = kDictationViewId; + view->entity_type = CANVAS_ENTITY_WEB_CONTENT; + view->is_dictation_transport = TRUE; + std::snprintf( + view->source_url, + sizeof(view->source_url), + "%s", + kDictationUrl); + std::snprintf(view->url, sizeof(view->url), "%s", kDictationUrl); + Rectangle bounds = {0.0f, 0.0f, 320.0f, 240.0f}; + if (!browser_created && + !CreateBrowser(surface, view, kDictationUrl, bounds)) { + view->active = FALSE; + view->entity_id = 0; + view->is_dictation_transport = FALSE; + view->source_url[0] = '\0'; + view->url[0] = '\0'; + return nullptr; + } + if (browser_created) { + EnsureViewResolution(surface, view, bounds); + NativeState *state = static_cast<NativeState *>(surface->p_native); + CefRefPtr<CanvasCefClient> client = + state->clients[ViewIndex(surface, view)]; + if (client && client->browser()) { + client->browser()->GetMainFrame()->LoadURL(kDictationUrl); + } + } + return view; +} + void PercentEncode( const char *source, char *target, @@ -528,8 +695,9 @@ const Canvas_Scene *scene) { for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { Canvas_Web_View *view = &surface->views[index]; - view->active = view->entity_id != 0 && - SceneHasVisibleEntity(scene, camera, view->entity_id); + view->active = view->is_dictation_transport || + (view->entity_id != 0 && + SceneHasVisibleEntity(scene, camera, view->entity_id)); } for (size_t index = 0; index < Dowa_Array_Length(scene->p_entities); ++index) { @@ -562,7 +730,7 @@ Canvas_Web_View *view = &surface->views[index]; CefRefPtr<CefBrowserHost> host = client->browser()->GetHost(); host->WasHidden(!view->active); - if (view->active) { + if (view->active && !view->is_dictation_transport) { const Canvas_Entity *entity = FindSurfaceEntity(scene, view->entity_id); if (entity) { host->SetWindowlessFrameRate( @@ -805,12 +973,22 @@ const Canvas_Scene *p_scene) { if (!p_surface->initialized) return; CefDoMessageLoopWork(); + EnsureDictationView(p_surface); ReconcileViews(p_surface, p_camera, p_scene); + if (p_surface->pending_dictation_command != 0) { + Canvas_Web_View *view = EnsureDictationView(p_surface); + if (view) { + view->dictation_command = p_surface->pending_dictation_command; + p_surface->pending_dictation_command = 0; + } + } for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { Canvas_Web_View *view = &p_surface->views[index]; if (!view->browser_created) continue; - if (!view->texture_ready && IsWindowReady()) { + if (!view->is_dictation_transport && + !view->texture_ready && + IsWindowReady()) { Image image = { .data = view->p_pixels, .width = view->texture_width, @@ -824,7 +1002,9 @@ SetTextureFilter(view->texture, TEXTURE_FILTER_BILINEAR); } } - if (view->texture_ready && view->pixels_dirty) { + if (!view->is_dictation_transport && + view->texture_ready && + view->pixels_dirty) { UpdateTexture(view->texture, view->p_pixels); view->pixels_dirty = FALSE; } @@ -840,6 +1020,29 @@ } view->can_go_back = client->browser()->CanGoBack() ? TRUE : FALSE; view->can_go_forward = client->browser()->CanGoForward() ? TRUE : FALSE; + if (view->dictation_command != 0) { + CefRefPtr<CefFrame> frame = client->browser()->GetMainFrame(); + const char *function_name = + view->dictation_command == 2 + ? "ZenbuDictationCommit" + : (view->dictation_command > 0 + ? "ZenbuDictationStart" + : "ZenbuDictationStop"); + char script[512]; + std::snprintf( + script, + sizeof(script), + "(()=>{let attempts=0;const invoke=()=>{" + "if(window.%s){window.%s();return;}" + "if(attempts++<40)setTimeout(invoke,250);};invoke();})()", + function_name, + function_name); + frame->ExecuteJavaScript( + script, + frame->GetURL(), + 0); + view->dictation_command = 0; + } } } } @@ -955,6 +1158,42 @@ } } +extern "C" void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active) { + int32 command = active ? 1 : -1; + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (view) { + view->dictation_command = command; + } else { + p_surface->pending_dictation_command = command; + } +} + +extern "C" void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface) { + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (view) { + view->dictation_command = 2; + } else { + p_surface->pending_dictation_command = 2; + } +} + +extern "C" Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity) { + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (!view || + view->dictation_sequence == view->consumed_dictation_sequence) { + return CANVAS_DICTATION_EVENT_NONE; + } + snprintf(p_text, text_capacity, "%s", view->dictation_text); + view->consumed_dictation_sequence = view->dictation_sequence; + return view->dictation_event; +} + extern "C" boolean Canvas_Web_Surface_Is_Focused( const Canvas_Web_Surface *p_surface) { return p_surface->focused_entity_id != 0;