diff infinite_canvas/web_surface_native.cc @ 280:49e9e591c9bb

Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
author MrJuneJune <me@mrjunejune.com>
date Tue, 18 Aug 2026 19:14:53 -0700
parents 8d560f50ed4c
children
line wrap: on
line diff
--- a/infinite_canvas/web_surface_native.cc	Mon Aug 17 22:22:36 2026 -0700
+++ b/infinite_canvas/web_surface_native.cc	Tue Aug 18 19:14:53 2026 -0700
@@ -6,16 +6,20 @@
 #include "include/cef_browser.h"
 #include "include/cef_client.h"
 #include "include/cef_display_handler.h"
+#include "include/cef_permission_handler.h"
 #include "include/cef_render_handler.h"
+#include "include/cef_request_context.h"
 #include "include/capi/cef_app_capi.h"
 
 #include <algorithm>
+#include <cctype>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
 #include <new>
+#include <string>
 
 #if defined(OS_LINUX)
 #include <dlfcn.h>
@@ -29,6 +33,8 @@
 constexpr int32 kMinTextureHeight = 120;
 constexpr int32 kMaxTextureWidth = 1280;
 constexpr int32 kMaxTextureHeight = 800;
+constexpr uint32 kDictationViewId = 0xffffffffu;
+constexpr const char *kDictationUrl = "http://127.0.0.1:8090/?canvas=1";
 int g_argc = 0;
 char **g_argv = nullptr;
 char *g_cef_argv[128] = {};
@@ -41,6 +47,26 @@
 
 void FindResourcePaths();
 
+void PercentDecode(const char *source, char *target, size_t target_size) {
+  size_t output = 0;
+  for (size_t index = 0;
+      source[index] && output + 1 < target_size;
+      ++index) {
+    if (source[index] == '%' &&
+        std::isxdigit(static_cast<unsigned char>(source[index + 1])) &&
+        std::isxdigit(static_cast<unsigned char>(source[index + 2]))) {
+      char hex[3] = {source[index + 1], source[index + 2], '\0'};
+      target[output++] = static_cast<char>(std::strtol(hex, nullptr, 16));
+      index += 2;
+    } else if (source[index] == '+') {
+      target[output++] = ' ';
+    } else {
+      target[output++] = source[index];
+    }
+  }
+  target[output] = '\0';
+}
+
 void ApplyColorScheme(CefRefPtr<CefBrowser> browser, boolean dark_mode) {
   if (!browser) return;
   CefRefPtr<CefDictionaryValue> params = CefDictionaryValue::Create();
@@ -62,6 +88,12 @@
   frame->ExecuteJavaScript(script, frame->GetURL(), 0);
 }
 
+boolean IsLocalDictationOrigin(const CefString &requesting_origin) {
+  std::string origin = requesting_origin.ToString();
+  return origin.rfind("http://127.0.0.1:8090", 0) == 0 ||
+      origin.rfind("http://localhost:8090", 0) == 0;
+}
+
 const Canvas_Entity *FindSurfaceEntity(
     const Canvas_Scene *scene,
     uint32 entity_id) {
@@ -167,12 +199,16 @@
 class CanvasCefClient : public CefClient,
                         public CefRenderHandler,
                         public CefDisplayHandler,
+                        public CefPermissionHandler,
                         public CefLifeSpanHandler {
  public:
   explicit CanvasCefClient(Canvas_Web_View *view) : view_(view) {}
 
   CefRefPtr<CefRenderHandler> GetRenderHandler() override { return this; }
   CefRefPtr<CefDisplayHandler> GetDisplayHandler() override { return this; }
+  CefRefPtr<CefPermissionHandler> GetPermissionHandler() override {
+    return this;
+  }
   CefRefPtr<CefLifeSpanHandler> GetLifeSpanHandler() override { return this; }
 
   void OnAddressChange(
@@ -189,6 +225,83 @@
     view_->can_go_forward = browser->CanGoForward() ? TRUE : FALSE;
   }
 
+  void OnTitleChange(
+      CefRefPtr<CefBrowser> browser,
+      const CefString &title) override {
+    (void)browser;
+    std::string value = title.ToString();
+    constexpr const char *prefix = "Zenbu Dictation|";
+    if (value.rfind(prefix, 0) != 0) return;
+    const char *kind = std::strchr(
+        value.c_str() + std::strlen(prefix),
+        '|');
+    if (!kind) return;
+    kind++;
+    const char *encoded = std::strchr(kind, '|');
+    if (!encoded) return;
+    std::string event_kind(kind, static_cast<size_t>(encoded - kind));
+    if (event_kind == "status") {
+      view_->dictation_event = CANVAS_DICTATION_EVENT_STATUS;
+    } else if (event_kind == "partial") {
+      view_->dictation_event = CANVAS_DICTATION_EVENT_PARTIAL;
+    } else if (event_kind == "final") {
+      view_->dictation_event = CANVAS_DICTATION_EVENT_FINAL;
+    } else {
+      return;
+    }
+    PercentDecode(
+        encoded + 1,
+        view_->dictation_text,
+        sizeof(view_->dictation_text));
+    view_->dictation_sequence++;
+  }
+
+  bool OnRequestMediaAccessPermission(
+      CefRefPtr<CefBrowser> browser,
+      CefRefPtr<CefFrame> frame,
+      const CefString &requesting_origin,
+      uint32_t requested_permissions,
+      CefRefPtr<CefMediaAccessCallback> callback) override {
+    (void)browser;
+    (void)frame;
+    boolean local_dictation =
+        view_->is_dictation_transport &&
+        IsLocalDictationOrigin(requesting_origin);
+    uint32_t audio = CEF_MEDIA_PERMISSION_DEVICE_AUDIO_CAPTURE;
+    boolean requests_audio = (requested_permissions & audio) != 0;
+    boolean requests_other_media = (requested_permissions & ~audio) != 0;
+    if (local_dictation && requests_audio && !requests_other_media) {
+      callback->Continue(audio);
+    } else {
+      callback->Cancel();
+    }
+    return true;
+  }
+
+  bool OnShowPermissionPrompt(
+      CefRefPtr<CefBrowser> browser,
+      uint64_t prompt_id,
+      const CefString &requesting_origin,
+      uint32_t requested_permissions,
+      CefRefPtr<CefPermissionPromptCallback> callback) override {
+    (void)browser;
+    (void)prompt_id;
+    if (!view_->is_dictation_transport ||
+        !IsLocalDictationOrigin(requesting_origin)) {
+      return false;
+    }
+    uint32_t microphone = CEF_PERMISSION_TYPE_MIC_STREAM;
+    boolean requests_microphone =
+        (requested_permissions & microphone) != 0;
+    boolean requests_other_permissions =
+        (requested_permissions & ~microphone) != 0;
+    callback->Continue(
+        requests_microphone && !requests_other_permissions
+            ? CEF_PERMISSION_RESULT_ACCEPT
+            : CEF_PERMISSION_RESULT_DENY);
+    return true;
+  }
+
   void GetViewRect(CefRefPtr<CefBrowser> browser, CefRect &rect) override {
     (void)browser;
     rect = CefRect(0, 0, view_->texture_width, view_->texture_height);
@@ -243,9 +356,17 @@
 
 struct NativeState {
   CefRefPtr<CanvasCefClient> clients[CANVAS_MAX_WEB_VIEWS];
+  CefRefPtr<CefRequestContext> dictation_context;
   boolean cef_initialized;
 };
 
+void EnsureDictationRequestContext(NativeState *state) {
+  if (state->dictation_context) return;
+  CefRequestContextSettings settings;
+  state->dictation_context =
+      CefRequestContext::CreateContext(settings, nullptr);
+}
+
 CefMainArgs MainArgs(int argc, char **argv) {
 #if defined(OS_WIN)
   (void)argc;
@@ -396,13 +517,18 @@
       : CefColorSetARGB(255, 255, 255, 255);
   const char *initial_url =
       view->entity_type == CANVAS_ENTITY_IMAGE ? "about:blank" : url;
+  CefRefPtr<CefRequestContext> request_context;
+  if (view->is_dictation_transport) {
+    EnsureDictationRequestContext(state);
+    request_context = state->dictation_context;
+  }
   if (!CefBrowserHost::CreateBrowser(
           window_info,
           state->clients[index],
           initial_url,
           browser_settings,
           nullptr,
-          nullptr)) {
+          request_context)) {
     state->clients[index] = nullptr;
     return FALSE;
   }
@@ -410,6 +536,47 @@
   return TRUE;
 }
 
+Canvas_Web_View *EnsureDictationView(Canvas_Web_Surface *surface) {
+  Canvas_Web_View *view = FindView(surface, kDictationViewId);
+  if (view) {
+    view->active = TRUE;
+    return view;
+  }
+  view = FindInactiveView(surface);
+  if (!view) return nullptr;
+  boolean browser_created = view->browser_created;
+  view->active = TRUE;
+  view->entity_id = kDictationViewId;
+  view->entity_type = CANVAS_ENTITY_WEB_CONTENT;
+  view->is_dictation_transport = TRUE;
+  std::snprintf(
+      view->source_url,
+      sizeof(view->source_url),
+      "%s",
+      kDictationUrl);
+  std::snprintf(view->url, sizeof(view->url), "%s", kDictationUrl);
+  Rectangle bounds = {0.0f, 0.0f, 320.0f, 240.0f};
+  if (!browser_created &&
+      !CreateBrowser(surface, view, kDictationUrl, bounds)) {
+    view->active = FALSE;
+    view->entity_id = 0;
+    view->is_dictation_transport = FALSE;
+    view->source_url[0] = '\0';
+    view->url[0] = '\0';
+    return nullptr;
+  }
+  if (browser_created) {
+    EnsureViewResolution(surface, view, bounds);
+    NativeState *state = static_cast<NativeState *>(surface->p_native);
+    CefRefPtr<CanvasCefClient> client =
+        state->clients[ViewIndex(surface, view)];
+    if (client && client->browser()) {
+      client->browser()->GetMainFrame()->LoadURL(kDictationUrl);
+    }
+  }
+  return view;
+}
+
 void PercentEncode(
     const char *source,
     char *target,
@@ -528,8 +695,9 @@
     const Canvas_Scene *scene) {
   for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) {
     Canvas_Web_View *view = &surface->views[index];
-    view->active = view->entity_id != 0 &&
-        SceneHasVisibleEntity(scene, camera, view->entity_id);
+    view->active = view->is_dictation_transport ||
+        (view->entity_id != 0 &&
+         SceneHasVisibleEntity(scene, camera, view->entity_id));
   }
 
   for (size_t index = 0; index < Dowa_Array_Length(scene->p_entities); ++index) {
@@ -562,7 +730,7 @@
       Canvas_Web_View *view = &surface->views[index];
       CefRefPtr<CefBrowserHost> host = client->browser()->GetHost();
       host->WasHidden(!view->active);
-      if (view->active) {
+      if (view->active && !view->is_dictation_transport) {
         const Canvas_Entity *entity = FindSurfaceEntity(scene, view->entity_id);
         if (entity) {
           host->SetWindowlessFrameRate(
@@ -805,12 +973,22 @@
     const Canvas_Scene *p_scene) {
   if (!p_surface->initialized) return;
   CefDoMessageLoopWork();
+  EnsureDictationView(p_surface);
   ReconcileViews(p_surface, p_camera, p_scene);
+  if (p_surface->pending_dictation_command != 0) {
+    Canvas_Web_View *view = EnsureDictationView(p_surface);
+    if (view) {
+      view->dictation_command = p_surface->pending_dictation_command;
+      p_surface->pending_dictation_command = 0;
+    }
+  }
 
   for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) {
     Canvas_Web_View *view = &p_surface->views[index];
     if (!view->browser_created) continue;
-    if (!view->texture_ready && IsWindowReady()) {
+    if (!view->is_dictation_transport &&
+        !view->texture_ready &&
+        IsWindowReady()) {
       Image image = {
           .data = view->p_pixels,
           .width = view->texture_width,
@@ -824,7 +1002,9 @@
         SetTextureFilter(view->texture, TEXTURE_FILTER_BILINEAR);
       }
     }
-    if (view->texture_ready && view->pixels_dirty) {
+    if (!view->is_dictation_transport &&
+        view->texture_ready &&
+        view->pixels_dirty) {
       UpdateTexture(view->texture, view->p_pixels);
       view->pixels_dirty = FALSE;
     }
@@ -840,6 +1020,29 @@
       }
       view->can_go_back = client->browser()->CanGoBack() ? TRUE : FALSE;
       view->can_go_forward = client->browser()->CanGoForward() ? TRUE : FALSE;
+      if (view->dictation_command != 0) {
+        CefRefPtr<CefFrame> frame = client->browser()->GetMainFrame();
+        const char *function_name =
+            view->dictation_command == 2
+                ? "ZenbuDictationCommit"
+                : (view->dictation_command > 0
+                    ? "ZenbuDictationStart"
+                    : "ZenbuDictationStop");
+        char script[512];
+        std::snprintf(
+            script,
+            sizeof(script),
+            "(()=>{let attempts=0;const invoke=()=>{"
+            "if(window.%s){window.%s();return;}"
+            "if(attempts++<40)setTimeout(invoke,250);};invoke();})()",
+            function_name,
+            function_name);
+        frame->ExecuteJavaScript(
+            script,
+            frame->GetURL(),
+            0);
+        view->dictation_command = 0;
+      }
     }
   }
 }
@@ -955,6 +1158,42 @@
   }
 }
 
+extern "C" void Canvas_Web_Surface_Set_Dictation_Active(
+    Canvas_Web_Surface *p_surface,
+    boolean active) {
+  int32 command = active ? 1 : -1;
+  Canvas_Web_View *view = FindView(p_surface, kDictationViewId);
+  if (view) {
+    view->dictation_command = command;
+  } else {
+    p_surface->pending_dictation_command = command;
+  }
+}
+
+extern "C" void Canvas_Web_Surface_Commit_Dictation(
+    Canvas_Web_Surface *p_surface) {
+  Canvas_Web_View *view = FindView(p_surface, kDictationViewId);
+  if (view) {
+    view->dictation_command = 2;
+  } else {
+    p_surface->pending_dictation_command = 2;
+  }
+}
+
+extern "C" Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation(
+    Canvas_Web_Surface *p_surface,
+    char *p_text,
+    size_t text_capacity) {
+  Canvas_Web_View *view = FindView(p_surface, kDictationViewId);
+  if (!view ||
+      view->dictation_sequence == view->consumed_dictation_sequence) {
+    return CANVAS_DICTATION_EVENT_NONE;
+  }
+  snprintf(p_text, text_capacity, "%s", view->dictation_text);
+  view->consumed_dictation_sequence = view->dictation_sequence;
+  return view->dictation_event;
+}
+
 extern "C" boolean Canvas_Web_Surface_Is_Focused(
     const Canvas_Web_Surface *p_surface) {
   return p_surface->focused_entity_id != 0;