Mercurial

comparison infinite_canvas/web_surface_native.cc @ 280:49e9e591c9bb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
author MrJuneJune <me@mrjunejune.com>
date Tue, 18 Aug 2026 19:14:53 -0700
parents 8d560f50ed4c
children
comparison
equal deleted inserted replaced
279:b3b547563ec7 280:49e9e591c9bb
4 4
5 #include "include/cef_app.h" 5 #include "include/cef_app.h"
6 #include "include/cef_browser.h" 6 #include "include/cef_browser.h"
7 #include "include/cef_client.h" 7 #include "include/cef_client.h"
8 #include "include/cef_display_handler.h" 8 #include "include/cef_display_handler.h"
9 #include "include/cef_permission_handler.h"
9 #include "include/cef_render_handler.h" 10 #include "include/cef_render_handler.h"
11 #include "include/cef_request_context.h"
10 #include "include/capi/cef_app_capi.h" 12 #include "include/capi/cef_app_capi.h"
11 13
12 #include <algorithm> 14 #include <algorithm>
15 #include <cctype>
13 #include <cmath> 16 #include <cmath>
14 #include <cstdio> 17 #include <cstdio>
15 #include <cstdlib> 18 #include <cstdlib>
16 #include <cstring> 19 #include <cstring>
17 #include <filesystem> 20 #include <filesystem>
18 #include <new> 21 #include <new>
22 #include <string>
19 23
20 #if defined(OS_LINUX) 24 #if defined(OS_LINUX)
21 #include <dlfcn.h> 25 #include <dlfcn.h>
22 #include <limits.h> 26 #include <limits.h>
23 #endif 27 #endif
27 constexpr int32 kResolutionStep = 64; 31 constexpr int32 kResolutionStep = 64;
28 constexpr int32 kMinTextureWidth = 160; 32 constexpr int32 kMinTextureWidth = 160;
29 constexpr int32 kMinTextureHeight = 120; 33 constexpr int32 kMinTextureHeight = 120;
30 constexpr int32 kMaxTextureWidth = 1280; 34 constexpr int32 kMaxTextureWidth = 1280;
31 constexpr int32 kMaxTextureHeight = 800; 35 constexpr int32 kMaxTextureHeight = 800;
36 constexpr uint32 kDictationViewId = 0xffffffffu;
37 constexpr const char *kDictationUrl = "http://127.0.0.1:8090/?canvas=1";
32 int g_argc = 0; 38 int g_argc = 0;
33 char **g_argv = nullptr; 39 char **g_argv = nullptr;
34 char *g_cef_argv[128] = {}; 40 char *g_cef_argv[128] = {};
35 #if defined(OS_LINUX) 41 #if defined(OS_LINUX)
36 char g_resources_path[PATH_MAX + 32] = {}; 42 char g_resources_path[PATH_MAX + 32] = {};
38 char g_resources_switch[PATH_MAX + 64] = {}; 44 char g_resources_switch[PATH_MAX + 64] = {};
39 char g_locales_switch[PATH_MAX + 64] = {}; 45 char g_locales_switch[PATH_MAX + 64] = {};
40 #endif 46 #endif
41 47
42 void FindResourcePaths(); 48 void FindResourcePaths();
49
50 void PercentDecode(const char *source, char *target, size_t target_size) {
51 size_t output = 0;
52 for (size_t index = 0;
53 source[index] && output + 1 < target_size;
54 ++index) {
55 if (source[index] == '%' &&
56 std::isxdigit(static_cast<unsigned char>(source[index + 1])) &&
57 std::isxdigit(static_cast<unsigned char>(source[index + 2]))) {
58 char hex[3] = {source[index + 1], source[index + 2], '\0'};
59 target[output++] = static_cast<char>(std::strtol(hex, nullptr, 16));
60 index += 2;
61 } else if (source[index] == '+') {
62 target[output++] = ' ';
63 } else {
64 target[output++] = source[index];
65 }
66 }
67 target[output] = '\0';
68 }
43 69
44 void ApplyColorScheme(CefRefPtr<CefBrowser> browser, boolean dark_mode) { 70 void ApplyColorScheme(CefRefPtr<CefBrowser> browser, boolean dark_mode) {
45 if (!browser) return; 71 if (!browser) return;
46 CefRefPtr<CefDictionaryValue> params = CefDictionaryValue::Create(); 72 CefRefPtr<CefDictionaryValue> params = CefDictionaryValue::Create();
47 CefRefPtr<CefListValue> features = CefListValue::Create(); 73 CefRefPtr<CefListValue> features = CefListValue::Create();
58 CefRefPtr<CefFrame> frame = browser->GetMainFrame(); 84 CefRefPtr<CefFrame> frame = browser->GetMainFrame();
59 const char *script = dark_mode 85 const char *script = dark_mode
60 ? "document.documentElement.style.colorScheme='dark'" 86 ? "document.documentElement.style.colorScheme='dark'"
61 : "document.documentElement.style.colorScheme='light'"; 87 : "document.documentElement.style.colorScheme='light'";
62 frame->ExecuteJavaScript(script, frame->GetURL(), 0); 88 frame->ExecuteJavaScript(script, frame->GetURL(), 0);
89 }
90
91 boolean IsLocalDictationOrigin(const CefString &requesting_origin) {
92 std::string origin = requesting_origin.ToString();
93 return origin.rfind("http://127.0.0.1:8090", 0) == 0 ||
94 origin.rfind("http://localhost:8090", 0) == 0;
63 } 95 }
64 96
65 const Canvas_Entity *FindSurfaceEntity( 97 const Canvas_Entity *FindSurfaceEntity(
66 const Canvas_Scene *scene, 98 const Canvas_Scene *scene,
67 uint32 entity_id) { 99 uint32 entity_id) {
165 }; 197 };
166 198
167 class CanvasCefClient : public CefClient, 199 class CanvasCefClient : public CefClient,
168 public CefRenderHandler, 200 public CefRenderHandler,
169 public CefDisplayHandler, 201 public CefDisplayHandler,
202 public CefPermissionHandler,
170 public CefLifeSpanHandler { 203 public CefLifeSpanHandler {
171 public: 204 public:
172 explicit CanvasCefClient(Canvas_Web_View *view) : view_(view) {} 205 explicit CanvasCefClient(Canvas_Web_View *view) : view_(view) {}
173 206
174 CefRefPtr<CefRenderHandler> GetRenderHandler() override { return this; } 207 CefRefPtr<CefRenderHandler> GetRenderHandler() override { return this; }
175 CefRefPtr<CefDisplayHandler> GetDisplayHandler() override { return this; } 208 CefRefPtr<CefDisplayHandler> GetDisplayHandler() override { return this; }
209 CefRefPtr<CefPermissionHandler> GetPermissionHandler() override {
210 return this;
211 }
176 CefRefPtr<CefLifeSpanHandler> GetLifeSpanHandler() override { return this; } 212 CefRefPtr<CefLifeSpanHandler> GetLifeSpanHandler() override { return this; }
177 213
178 void OnAddressChange( 214 void OnAddressChange(
179 CefRefPtr<CefBrowser> browser, 215 CefRefPtr<CefBrowser> browser,
180 CefRefPtr<CefFrame> frame, 216 CefRefPtr<CefFrame> frame,
185 sizeof(view_->url), 221 sizeof(view_->url),
186 "%s", 222 "%s",
187 url.ToString().c_str()); 223 url.ToString().c_str());
188 view_->can_go_back = browser->CanGoBack() ? TRUE : FALSE; 224 view_->can_go_back = browser->CanGoBack() ? TRUE : FALSE;
189 view_->can_go_forward = browser->CanGoForward() ? TRUE : FALSE; 225 view_->can_go_forward = browser->CanGoForward() ? TRUE : FALSE;
226 }
227
228 void OnTitleChange(
229 CefRefPtr<CefBrowser> browser,
230 const CefString &title) override {
231 (void)browser;
232 std::string value = title.ToString();
233 constexpr const char *prefix = "Zenbu Dictation|";
234 if (value.rfind(prefix, 0) != 0) return;
235 const char *kind = std::strchr(
236 value.c_str() + std::strlen(prefix),
237 '|');
238 if (!kind) return;
239 kind++;
240 const char *encoded = std::strchr(kind, '|');
241 if (!encoded) return;
242 std::string event_kind(kind, static_cast<size_t>(encoded - kind));
243 if (event_kind == "status") {
244 view_->dictation_event = CANVAS_DICTATION_EVENT_STATUS;
245 } else if (event_kind == "partial") {
246 view_->dictation_event = CANVAS_DICTATION_EVENT_PARTIAL;
247 } else if (event_kind == "final") {
248 view_->dictation_event = CANVAS_DICTATION_EVENT_FINAL;
249 } else {
250 return;
251 }
252 PercentDecode(
253 encoded + 1,
254 view_->dictation_text,
255 sizeof(view_->dictation_text));
256 view_->dictation_sequence++;
257 }
258
259 bool OnRequestMediaAccessPermission(
260 CefRefPtr<CefBrowser> browser,
261 CefRefPtr<CefFrame> frame,
262 const CefString &requesting_origin,
263 uint32_t requested_permissions,
264 CefRefPtr<CefMediaAccessCallback> callback) override {
265 (void)browser;
266 (void)frame;
267 boolean local_dictation =
268 view_->is_dictation_transport &&
269 IsLocalDictationOrigin(requesting_origin);
270 uint32_t audio = CEF_MEDIA_PERMISSION_DEVICE_AUDIO_CAPTURE;
271 boolean requests_audio = (requested_permissions & audio) != 0;
272 boolean requests_other_media = (requested_permissions & ~audio) != 0;
273 if (local_dictation && requests_audio && !requests_other_media) {
274 callback->Continue(audio);
275 } else {
276 callback->Cancel();
277 }
278 return true;
279 }
280
281 bool OnShowPermissionPrompt(
282 CefRefPtr<CefBrowser> browser,
283 uint64_t prompt_id,
284 const CefString &requesting_origin,
285 uint32_t requested_permissions,
286 CefRefPtr<CefPermissionPromptCallback> callback) override {
287 (void)browser;
288 (void)prompt_id;
289 if (!view_->is_dictation_transport ||
290 !IsLocalDictationOrigin(requesting_origin)) {
291 return false;
292 }
293 uint32_t microphone = CEF_PERMISSION_TYPE_MIC_STREAM;
294 boolean requests_microphone =
295 (requested_permissions & microphone) != 0;
296 boolean requests_other_permissions =
297 (requested_permissions & ~microphone) != 0;
298 callback->Continue(
299 requests_microphone && !requests_other_permissions
300 ? CEF_PERMISSION_RESULT_ACCEPT
301 : CEF_PERMISSION_RESULT_DENY);
302 return true;
190 } 303 }
191 304
192 void GetViewRect(CefRefPtr<CefBrowser> browser, CefRect &rect) override { 305 void GetViewRect(CefRefPtr<CefBrowser> browser, CefRect &rect) override {
193 (void)browser; 306 (void)browser;
194 rect = CefRect(0, 0, view_->texture_width, view_->texture_height); 307 rect = CefRect(0, 0, view_->texture_width, view_->texture_height);
241 DISALLOW_COPY_AND_ASSIGN(CanvasCefClient); 354 DISALLOW_COPY_AND_ASSIGN(CanvasCefClient);
242 }; 355 };
243 356
244 struct NativeState { 357 struct NativeState {
245 CefRefPtr<CanvasCefClient> clients[CANVAS_MAX_WEB_VIEWS]; 358 CefRefPtr<CanvasCefClient> clients[CANVAS_MAX_WEB_VIEWS];
359 CefRefPtr<CefRequestContext> dictation_context;
246 boolean cef_initialized; 360 boolean cef_initialized;
247 }; 361 };
362
363 void EnsureDictationRequestContext(NativeState *state) {
364 if (state->dictation_context) return;
365 CefRequestContextSettings settings;
366 state->dictation_context =
367 CefRequestContext::CreateContext(settings, nullptr);
368 }
248 369
249 CefMainArgs MainArgs(int argc, char **argv) { 370 CefMainArgs MainArgs(int argc, char **argv) {
250 #if defined(OS_WIN) 371 #if defined(OS_WIN)
251 (void)argc; 372 (void)argc;
252 (void)argv; 373 (void)argv;
394 browser_settings.background_color = surface->dark_mode 515 browser_settings.background_color = surface->dark_mode
395 ? CefColorSetARGB(255, 15, 15, 17) 516 ? CefColorSetARGB(255, 15, 15, 17)
396 : CefColorSetARGB(255, 255, 255, 255); 517 : CefColorSetARGB(255, 255, 255, 255);
397 const char *initial_url = 518 const char *initial_url =
398 view->entity_type == CANVAS_ENTITY_IMAGE ? "about:blank" : url; 519 view->entity_type == CANVAS_ENTITY_IMAGE ? "about:blank" : url;
520 CefRefPtr<CefRequestContext> request_context;
521 if (view->is_dictation_transport) {
522 EnsureDictationRequestContext(state);
523 request_context = state->dictation_context;
524 }
399 if (!CefBrowserHost::CreateBrowser( 525 if (!CefBrowserHost::CreateBrowser(
400 window_info, 526 window_info,
401 state->clients[index], 527 state->clients[index],
402 initial_url, 528 initial_url,
403 browser_settings, 529 browser_settings,
404 nullptr, 530 nullptr,
405 nullptr)) { 531 request_context)) {
406 state->clients[index] = nullptr; 532 state->clients[index] = nullptr;
407 return FALSE; 533 return FALSE;
408 } 534 }
409 view->browser_created = TRUE; 535 view->browser_created = TRUE;
410 return TRUE; 536 return TRUE;
537 }
538
539 Canvas_Web_View *EnsureDictationView(Canvas_Web_Surface *surface) {
540 Canvas_Web_View *view = FindView(surface, kDictationViewId);
541 if (view) {
542 view->active = TRUE;
543 return view;
544 }
545 view = FindInactiveView(surface);
546 if (!view) return nullptr;
547 boolean browser_created = view->browser_created;
548 view->active = TRUE;
549 view->entity_id = kDictationViewId;
550 view->entity_type = CANVAS_ENTITY_WEB_CONTENT;
551 view->is_dictation_transport = TRUE;
552 std::snprintf(
553 view->source_url,
554 sizeof(view->source_url),
555 "%s",
556 kDictationUrl);
557 std::snprintf(view->url, sizeof(view->url), "%s", kDictationUrl);
558 Rectangle bounds = {0.0f, 0.0f, 320.0f, 240.0f};
559 if (!browser_created &&
560 !CreateBrowser(surface, view, kDictationUrl, bounds)) {
561 view->active = FALSE;
562 view->entity_id = 0;
563 view->is_dictation_transport = FALSE;
564 view->source_url[0] = '\0';
565 view->url[0] = '\0';
566 return nullptr;
567 }
568 if (browser_created) {
569 EnsureViewResolution(surface, view, bounds);
570 NativeState *state = static_cast<NativeState *>(surface->p_native);
571 CefRefPtr<CanvasCefClient> client =
572 state->clients[ViewIndex(surface, view)];
573 if (client && client->browser()) {
574 client->browser()->GetMainFrame()->LoadURL(kDictationUrl);
575 }
576 }
577 return view;
411 } 578 }
412 579
413 void PercentEncode( 580 void PercentEncode(
414 const char *source, 581 const char *source,
415 char *target, 582 char *target,
526 Canvas_Web_Surface *surface, 693 Canvas_Web_Surface *surface,
527 const Canvas_Camera *camera, 694 const Canvas_Camera *camera,
528 const Canvas_Scene *scene) { 695 const Canvas_Scene *scene) {
529 for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { 696 for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) {
530 Canvas_Web_View *view = &surface->views[index]; 697 Canvas_Web_View *view = &surface->views[index];
531 view->active = view->entity_id != 0 && 698 view->active = view->is_dictation_transport ||
532 SceneHasVisibleEntity(scene, camera, view->entity_id); 699 (view->entity_id != 0 &&
700 SceneHasVisibleEntity(scene, camera, view->entity_id));
533 } 701 }
534 702
535 for (size_t index = 0; index < Dowa_Array_Length(scene->p_entities); ++index) { 703 for (size_t index = 0; index < Dowa_Array_Length(scene->p_entities); ++index) {
536 const Canvas_Entity *entity = &scene->p_entities[index]; 704 const Canvas_Entity *entity = &scene->p_entities[index];
537 if ((entity->type != CANVAS_ENTITY_WEB_CONTENT && 705 if ((entity->type != CANVAS_ENTITY_WEB_CONTENT &&
560 CefRefPtr<CanvasCefClient> client = state->clients[index]; 728 CefRefPtr<CanvasCefClient> client = state->clients[index];
561 if (client && client->browser()) { 729 if (client && client->browser()) {
562 Canvas_Web_View *view = &surface->views[index]; 730 Canvas_Web_View *view = &surface->views[index];
563 CefRefPtr<CefBrowserHost> host = client->browser()->GetHost(); 731 CefRefPtr<CefBrowserHost> host = client->browser()->GetHost();
564 host->WasHidden(!view->active); 732 host->WasHidden(!view->active);
565 if (view->active) { 733 if (view->active && !view->is_dictation_transport) {
566 const Canvas_Entity *entity = FindSurfaceEntity(scene, view->entity_id); 734 const Canvas_Entity *entity = FindSurfaceEntity(scene, view->entity_id);
567 if (entity) { 735 if (entity) {
568 host->SetWindowlessFrameRate( 736 host->SetWindowlessFrameRate(
569 FrameRateForView( 737 FrameRateForView(
570 surface, 738 surface,
803 Canvas_Web_Surface *p_surface, 971 Canvas_Web_Surface *p_surface,
804 const Canvas_Camera *p_camera, 972 const Canvas_Camera *p_camera,
805 const Canvas_Scene *p_scene) { 973 const Canvas_Scene *p_scene) {
806 if (!p_surface->initialized) return; 974 if (!p_surface->initialized) return;
807 CefDoMessageLoopWork(); 975 CefDoMessageLoopWork();
976 EnsureDictationView(p_surface);
808 ReconcileViews(p_surface, p_camera, p_scene); 977 ReconcileViews(p_surface, p_camera, p_scene);
978 if (p_surface->pending_dictation_command != 0) {
979 Canvas_Web_View *view = EnsureDictationView(p_surface);
980 if (view) {
981 view->dictation_command = p_surface->pending_dictation_command;
982 p_surface->pending_dictation_command = 0;
983 }
984 }
809 985
810 for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { 986 for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) {
811 Canvas_Web_View *view = &p_surface->views[index]; 987 Canvas_Web_View *view = &p_surface->views[index];
812 if (!view->browser_created) continue; 988 if (!view->browser_created) continue;
813 if (!view->texture_ready && IsWindowReady()) { 989 if (!view->is_dictation_transport &&
990 !view->texture_ready &&
991 IsWindowReady()) {
814 Image image = { 992 Image image = {
815 .data = view->p_pixels, 993 .data = view->p_pixels,
816 .width = view->texture_width, 994 .width = view->texture_width,
817 .height = view->texture_height, 995 .height = view->texture_height,
818 .mipmaps = 1, 996 .mipmaps = 1,
822 view->texture_ready = view->texture.id != 0 ? TRUE : FALSE; 1000 view->texture_ready = view->texture.id != 0 ? TRUE : FALSE;
823 if (view->texture_ready) { 1001 if (view->texture_ready) {
824 SetTextureFilter(view->texture, TEXTURE_FILTER_BILINEAR); 1002 SetTextureFilter(view->texture, TEXTURE_FILTER_BILINEAR);
825 } 1003 }
826 } 1004 }
827 if (view->texture_ready && view->pixels_dirty) { 1005 if (!view->is_dictation_transport &&
1006 view->texture_ready &&
1007 view->pixels_dirty) {
828 UpdateTexture(view->texture, view->p_pixels); 1008 UpdateTexture(view->texture, view->p_pixels);
829 view->pixels_dirty = FALSE; 1009 view->pixels_dirty = FALSE;
830 } 1010 }
831 NativeState *state = static_cast<NativeState *>(p_surface->p_native); 1011 NativeState *state = static_cast<NativeState *>(p_surface->p_native);
832 CefRefPtr<CanvasCefClient> client = state->clients[index]; 1012 CefRefPtr<CanvasCefClient> client = state->clients[index];
838 view->color_scheme_applied = TRUE; 1018 view->color_scheme_applied = TRUE;
839 view->applied_dark_mode = p_surface->dark_mode; 1019 view->applied_dark_mode = p_surface->dark_mode;
840 } 1020 }
841 view->can_go_back = client->browser()->CanGoBack() ? TRUE : FALSE; 1021 view->can_go_back = client->browser()->CanGoBack() ? TRUE : FALSE;
842 view->can_go_forward = client->browser()->CanGoForward() ? TRUE : FALSE; 1022 view->can_go_forward = client->browser()->CanGoForward() ? TRUE : FALSE;
1023 if (view->dictation_command != 0) {
1024 CefRefPtr<CefFrame> frame = client->browser()->GetMainFrame();
1025 const char *function_name =
1026 view->dictation_command == 2
1027 ? "ZenbuDictationCommit"
1028 : (view->dictation_command > 0
1029 ? "ZenbuDictationStart"
1030 : "ZenbuDictationStop");
1031 char script[512];
1032 std::snprintf(
1033 script,
1034 sizeof(script),
1035 "(()=>{let attempts=0;const invoke=()=>{"
1036 "if(window.%s){window.%s();return;}"
1037 "if(attempts++<40)setTimeout(invoke,250);};invoke();})()",
1038 function_name,
1039 function_name);
1040 frame->ExecuteJavaScript(
1041 script,
1042 frame->GetURL(),
1043 0);
1044 view->dictation_command = 0;
1045 }
843 } 1046 }
844 } 1047 }
845 } 1048 }
846 1049
847 extern "C" void Canvas_Web_Surface_Draw( 1050 extern "C" void Canvas_Web_Surface_Draw(
953 if (client && client->browser() && client->browser()->CanGoForward()) { 1156 if (client && client->browser() && client->browser()->CanGoForward()) {
954 client->browser()->GoForward(); 1157 client->browser()->GoForward();
955 } 1158 }
956 } 1159 }
957 1160
1161 extern "C" void Canvas_Web_Surface_Set_Dictation_Active(
1162 Canvas_Web_Surface *p_surface,
1163 boolean active) {
1164 int32 command = active ? 1 : -1;
1165 Canvas_Web_View *view = FindView(p_surface, kDictationViewId);
1166 if (view) {
1167 view->dictation_command = command;
1168 } else {
1169 p_surface->pending_dictation_command = command;
1170 }
1171 }
1172
1173 extern "C" void Canvas_Web_Surface_Commit_Dictation(
1174 Canvas_Web_Surface *p_surface) {
1175 Canvas_Web_View *view = FindView(p_surface, kDictationViewId);
1176 if (view) {
1177 view->dictation_command = 2;
1178 } else {
1179 p_surface->pending_dictation_command = 2;
1180 }
1181 }
1182
1183 extern "C" Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation(
1184 Canvas_Web_Surface *p_surface,
1185 char *p_text,
1186 size_t text_capacity) {
1187 Canvas_Web_View *view = FindView(p_surface, kDictationViewId);
1188 if (!view ||
1189 view->dictation_sequence == view->consumed_dictation_sequence) {
1190 return CANVAS_DICTATION_EVENT_NONE;
1191 }
1192 snprintf(p_text, text_capacity, "%s", view->dictation_text);
1193 view->consumed_dictation_sequence = view->dictation_sequence;
1194 return view->dictation_event;
1195 }
1196
958 extern "C" boolean Canvas_Web_Surface_Is_Focused( 1197 extern "C" boolean Canvas_Web_Surface_Is_Focused(
959 const Canvas_Web_Surface *p_surface) { 1198 const Canvas_Web_Surface *p_surface) {
960 return p_surface->focused_entity_id != 0; 1199 return p_surface->focused_entity_id != 0;
961 } 1200 }
962 1201