Mercurial
diff infinite_canvas/main.c @ 280:49e9e591c9bb
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Tue, 18 Aug 2026 19:14:53 -0700 |
| parents | 8d560f50ed4c |
| children | c57149ad216e |
line wrap: on
line diff
--- a/infinite_canvas/main.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/main.c Tue Aug 18 19:14:53 2026 -0700 @@ -1,3 +1,4 @@ +#include "infinite_canvas/agent_service.h" #include "infinite_canvas/canvas.h" #include "infinite_canvas/web_surface.h" @@ -9,6 +10,7 @@ #include <emscripten/emscripten.h> #endif +#include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -18,11 +20,25 @@ Canvas_Camera camera; Canvas_Scene scene; Canvas_Web_Surface web_surface; + Canvas_Agent_Service agent_service; Font font; Canvas_Theme theme; boolean owns_font; const char *p_screenshot_path; uint32 frame_count; + uint32 screenshot_frame; + uint32 pending_conversation_id; + uint32 dictation_entity_id; + boolean agent_initialized; + boolean dictation_overlay_visible; + boolean dictation_listening; + boolean dictation_requested_active; + boolean dictation_send_pending; + char dictation_status[128]; + char dictation_partial[CANVAS_ENTITY_TEXT_CAPACITY]; + char dictation_transcript[CANVAS_ENTITY_TEXT_CAPACITY]; + char pending_prompt[CANVAS_ENTITY_TEXT_CAPACITY]; + char agent_context[CANVAS_CONTEXT_MAX_LENGTH]; #if defined(INFINITE_CANVAS_DEV_UI) Canvas_Dev_UI dev_ui; #endif @@ -30,6 +46,381 @@ static Canvas_App *g_app = NULL; +static const char *Canvas_App_JSON_String( + const char *p_json, + const char *p_key, + char *p_output, + size_t output_capacity) +{ + char needle[128]; + snprintf(needle, sizeof(needle), "\"%s\"", p_key); + const char *p_cursor = strstr(p_json, needle); + if (!p_cursor) return NULL; + p_cursor = strchr(p_cursor + strlen(needle), ':'); + if (!p_cursor) return NULL; + while (*++p_cursor == ' ') {} + if (*p_cursor != '"') return NULL; + p_cursor++; + size_t output = 0; + while (*p_cursor && *p_cursor != '"' && output + 1 < output_capacity) { + if (*p_cursor == '\\' && p_cursor[1]) { + p_cursor++; + if (*p_cursor == 'n') p_output[output++] = '\n'; + else if (*p_cursor == 't') p_output[output++] = '\t'; + else if (*p_cursor != 'r') p_output[output++] = *p_cursor; + } else { + p_output[output++] = *p_cursor; + } + p_cursor++; + } + p_output[output] = '\0'; + return p_output; +} + +static uint32 Canvas_App_JSON_Uint(const char *p_json, const char *p_key) +{ + char needle[128]; + snprintf(needle, sizeof(needle), "\"%s\"", p_key); + const char *p_cursor = strstr(p_json, needle); + if (!p_cursor) return 0; + p_cursor = strchr(p_cursor + strlen(needle), ':'); + if (!p_cursor) return 0; + return (uint32)strtoul(p_cursor + 1, NULL, 10); +} + +static Canvas_Entity *Canvas_App_Find_Entity(uint32 entity_id) +{ + for (size_t index = 0; + index < Dowa_Array_Length(g_app->scene.p_entities); + index++) { + if (g_app->scene.p_entities[index].id == entity_id) { + return &g_app->scene.p_entities[index]; + } + } + return NULL; +} + +static void Canvas_App_Set_Conversation_Text( + Canvas_Entity *p_entity, + const char *p_prompt, + const char *p_speaker, + const char *p_response) +{ + size_t length = 0; + int32 written = snprintf( + p_entity->text, + sizeof(p_entity->text), + "You\n"); + if (written > 0) length = (size_t)written; + written = snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "%.*s\n\n%s\n", + 900, + p_prompt, + p_speaker); + if (written > 0) length += (size_t)written; + if (length >= sizeof(p_entity->text)) { + length = sizeof(p_entity->text) - 1; + } + snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "%.*s", + 900, + p_response); + Canvas_Conversation_Scroll_To_End(p_entity, g_app->font); +} + +static boolean Canvas_App_Submit_Prompt(const char *p_prompt) +{ + if (!p_prompt || !p_prompt[0]) return FALSE; + Canvas_Scene_Build_Visible_Context( + &g_app->scene, + &g_app->camera, + g_app->agent_context, + sizeof(g_app->agent_context)); + if (!Canvas_Agent_Service_Submit( + &g_app->agent_service, + p_prompt, + g_app->agent_context)) { + return FALSE; + } + snprintf( + g_app->pending_prompt, + sizeof(g_app->pending_prompt), + "%s", + p_prompt); + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + g_app->pending_conversation_id = Canvas_Scene_Add_Conversation( + &g_app->scene, + "Routing thought...", + p_prompt, + "Copilot is deciding whether to create or extend a session.", + position); + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (p_pending) p_pending->agent_working = TRUE; + return TRUE; +} + +static void Canvas_App_Apply_Agent_Response(const char *p_json) +{ + char action[32] = {0}; + char title[128] = {0}; + char response[CANVAS_ENTITY_TEXT_CAPACITY] = {0}; + Canvas_App_JSON_String(p_json, "action", action, sizeof(action)); + Canvas_App_JSON_String(p_json, "title", title, sizeof(title)); + if (!Canvas_App_JSON_String( + p_json, + "response", + response, + sizeof(response))) { + snprintf(response, sizeof(response), "%s", p_json); + } + uint32 conversation_id = + Canvas_App_JSON_Uint(p_json, "conversation_id"); + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (strcmp(action, "append") == 0 && + conversation_id != 0 && + Canvas_Scene_Append_Conversation( + &g_app->scene, + conversation_id, + g_app->pending_prompt, + response)) { + Canvas_Entity *p_conversation = + Canvas_App_Find_Entity(conversation_id); + Canvas_Conversation_Scroll_To_End( + p_conversation, + g_app->font); + if (p_pending) p_pending->removing = TRUE; + } else if (p_pending) { + snprintf( + p_pending->label, + sizeof(p_pending->label), + "%s", + title[0] ? title : "Copilot session"); + Canvas_App_Set_Conversation_Text( + p_pending, + g_app->pending_prompt, + "Copilot", + response); + p_pending->agent_working = FALSE; + } else { + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + uint32 created_id = Canvas_Scene_Add_Conversation( + &g_app->scene, + title[0] ? title : "Copilot session", + g_app->pending_prompt, + response, + position); + Canvas_Conversation_Scroll_To_End( + Canvas_App_Find_Entity(created_id), + g_app->font); + } + g_app->pending_conversation_id = 0; + g_app->pending_prompt[0] = '\0'; +} + +static Canvas_Entity *Canvas_App_Ensure_Dictation_Entity(void) +{ + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (p_entity) return p_entity; + + Vector2 center = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f, + (float)g_app->camera.viewport_height * 0.5f, + }); + Vector2 size = {520.0f, 240.0f}; + if (!Canvas_Scene_Add( + &g_app->scene, + CANVAS_ENTITY_TEXT_AREA, + (Vector2){ + center.x - size.x * 0.5f, + center.y - size.y * 0.5f, + })) { + return NULL; + } + size_t index = Dowa_Array_Length(g_app->scene.p_entities) - 1; + p_entity = &g_app->scene.p_entities[index]; + p_entity->size = size; + snprintf(p_entity->label, sizeof(p_entity->label), "Dictation"); + g_app->dictation_entity_id = p_entity->id; + g_app->scene.selected_index = (int32)index; + return p_entity; +} + +static void Canvas_App_Sync_Dictation_Entity(void) +{ + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (!p_entity) return; + + char text[CANVAS_ENTITY_TEXT_CAPACITY]; + snprintf(text, sizeof(text), "%s", g_app->dictation_transcript); + size_t length = strlen(text); + if (length > 0 && g_app->dictation_partial[0] && + length + 1 < sizeof(text)) { + text[length++] = ' '; + text[length] = '\0'; + } + snprintf( + text + length, + sizeof(text) - length, + "%s", + g_app->dictation_partial); + Canvas_Text_Area_Set_Content( + p_entity, + g_app->font, + text, + g_app->dictation_partial[0] ? TRUE : FALSE); +} + +static void Canvas_App_Set_Dictation_Active(boolean active) +{ + g_app->dictation_overlay_visible = TRUE; + g_app->dictation_requested_active = active; + g_app->dictation_listening = FALSE; + g_app->dictation_send_pending = FALSE; + if (active) { + Canvas_App_Ensure_Dictation_Entity(); + Canvas_App_Sync_Dictation_Entity(); + } + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "%s", + active ? "Starting microphone..." : "Stopping microphone..."); + Canvas_Web_Surface_Set_Dictation_Active( + &g_app->web_surface, + active); +} + +static boolean Canvas_App_Submit_Dictation(void) +{ + if (!g_app->dictation_transcript[0]) return FALSE; + if (!Canvas_App_Submit_Prompt(g_app->dictation_transcript)) { + g_app->dictation_send_pending = FALSE; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Copilot is busy; press Enter to retry"); + return FALSE; + } + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (p_entity) p_entity->text_muted = FALSE; + g_app->dictation_transcript[0] = '\0'; + g_app->dictation_partial[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + g_app->dictation_send_pending = FALSE; + g_app->dictation_overlay_visible = + g_app->dictation_listening; + return TRUE; +} + +static void Canvas_App_Append_Dictation(const char *p_text) +{ + if (!p_text || !p_text[0]) return; + size_t length = strlen(g_app->dictation_transcript); + if (length > 0 && length + 1 < sizeof(g_app->dictation_transcript)) { + g_app->dictation_transcript[length++] = ' '; + g_app->dictation_transcript[length] = '\0'; + } + size_t available = sizeof(g_app->dictation_transcript) - length - 1; + size_t text_length = strlen(p_text); + size_t copy_length = text_length < available ? text_length : available; + memcpy(g_app->dictation_transcript + length, p_text, copy_length); + g_app->dictation_transcript[length + copy_length] = '\0'; + Canvas_App_Sync_Dictation_Entity(); +} + +static void Canvas_App_Update_Dictation_Overlay_Input(void) +{ + if (!g_app->dictation_overlay_visible) return; + if (g_app->dictation_listening && IsKeyPressed(KEY_BACKSPACE)) { + g_app->dictation_send_pending = FALSE; + g_app->dictation_partial[0] = '\0'; + g_app->dictation_transcript[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Listening"); + return; + } + if (IsKeyPressed(KEY_ESCAPE)) { + if (g_app->dictation_listening) { + Canvas_Web_Surface_Set_Dictation_Active( + &g_app->web_surface, + FALSE); + g_app->dictation_requested_active = FALSE; + g_app->dictation_listening = FALSE; + } + g_app->dictation_send_pending = FALSE; + g_app->dictation_partial[0] = '\0'; + g_app->dictation_transcript[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + g_app->dictation_overlay_visible = FALSE; + return; + } + if (!IsKeyPressed(KEY_ENTER)) return; + + if (g_app->dictation_partial[0]) { + g_app->dictation_send_pending = TRUE; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Finishing dictation..."); + Canvas_Web_Surface_Commit_Dictation( + &g_app->web_surface); + } else if (g_app->dictation_transcript[0]) { + Canvas_App_Submit_Dictation(); + } +} + +static boolean Canvas_App_Dictation_Status_Is_Error(const char *p_status) +{ + return strstr(p_status, "failed") != NULL || + strstr(p_status, "Unable") != NULL || + strstr(p_status, "closed") != NULL || + strstr(p_status, "invalid") != NULL || + strstr(p_status, "error") != NULL; +} + +static void Canvas_App_Draw_Listening_Indicator(void) +{ + if (!g_app->dictation_listening) return; + + float pulse = 0.72f + 0.28f * sinf((float)GetTime() * 5.0f); + Vector2 dot = {28.0f, (float)GetScreenHeight() - 28.0f}; + DrawCircleV( + dot, + 5.0f + pulse * 2.0f, + Fade(g_app->theme.danger, 0.82f)); + DrawTextEx( + g_app->font, + "Listening", + (Vector2){44.0f, dot.y - 8.0f}, + 14.0f, + 0.0f, + g_app->theme.text_muted); +} + static void Canvas_App_Draw_Frame(void) { int32 width = GetScreenWidth(); @@ -43,6 +434,15 @@ Canvas_Web_Surface_Set_Dark_Mode( &g_app->web_surface, g_app->theme.resolved_mode == CANVAS_THEME_DARK ? TRUE : FALSE); + boolean dictation_hotkey = + !Canvas_Scene_Is_Text_Editing(&g_app->scene) && + !Canvas_Web_Chrome_Is_Editing(&g_app->scene) && + IsKeyPressed(KEY_M); + if (dictation_hotkey) { + Canvas_App_Set_Dictation_Active( + g_app->dictation_requested_active ? FALSE : TRUE); + } + Canvas_App_Update_Dictation_Overlay_Input(); Canvas_Web_Chrome_Action chrome_action = CANVAS_WEB_CHROME_NONE; if (!block_pointer_input) { const char *p_current_url = NULL; @@ -76,7 +476,9 @@ } } boolean web_input = FALSE; - if (!block_pointer_input && chrome_action == CANVAS_WEB_CHROME_NONE) { + if (!block_pointer_input && + !dictation_hotkey && + chrome_action == CANVAS_WEB_CHROME_NONE) { web_input = Canvas_Web_Surface_Update_Input( &g_app->web_surface, &g_app->camera, @@ -99,6 +501,7 @@ Canvas_Scene_Toggle_Selected_Pin(&g_app->scene, &g_app->camera); } if (!keyboard_focus && + !g_app->dictation_listening && (IsKeyPressed(KEY_DELETE) || IsKeyPressed(KEY_BACKSPACE))) { Canvas_Scene_Fade_Out_Selected(&g_app->scene); } @@ -109,6 +512,136 @@ Canvas_Scene_Sync_Pinned(&g_app->scene, &g_app->camera); Canvas_Scene_Update_Notification_Stack(&g_app->scene, &g_app->camera); + char dictation_text[CANVAS_ENTITY_TEXT_CAPACITY]; + Canvas_Dictation_Event dictation_event = + Canvas_Web_Surface_Consume_Dictation( + &g_app->web_surface, + dictation_text, + sizeof(dictation_text)); + if (dictation_event == CANVAS_DICTATION_EVENT_STATUS) { + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "%.127s", + dictation_text); + if (strcmp(dictation_text, "Idle") == 0) { + g_app->dictation_listening = FALSE; + if (g_app->dictation_send_pending) { + if (g_app->dictation_partial[0]) { + Canvas_App_Append_Dictation( + g_app->dictation_partial); + g_app->dictation_partial[0] = '\0'; + } + Canvas_App_Submit_Dictation(); + } else if (!g_app->dictation_transcript[0] && + !g_app->dictation_partial[0]) { + g_app->dictation_overlay_visible = FALSE; + } + } else if (strcmp(dictation_text, "Listening") == 0 || + strcmp(dictation_text, "Speech detected") == 0) { + g_app->dictation_listening = + g_app->dictation_requested_active; + if (strcmp(dictation_text, "Listening") == 0 && + g_app->dictation_send_pending) { + if (g_app->dictation_partial[0]) { + Canvas_App_Append_Dictation( + g_app->dictation_partial); + g_app->dictation_partial[0] = '\0'; + } + Canvas_App_Submit_Dictation(); + } + } else if (Canvas_App_Dictation_Status_Is_Error(dictation_text)) { + g_app->dictation_listening = FALSE; + g_app->dictation_overlay_visible = TRUE; + } + } else if (dictation_event == CANVAS_DICTATION_EVENT_PARTIAL) { + g_app->dictation_overlay_visible = TRUE; + snprintf( + g_app->dictation_partial, + sizeof(g_app->dictation_partial), + "%s", + dictation_text); + Canvas_App_Sync_Dictation_Entity(); + } else if (dictation_event == CANVAS_DICTATION_EVENT_FINAL) { + g_app->dictation_overlay_visible = TRUE; + Canvas_App_Append_Dictation(dictation_text); + g_app->dictation_partial[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + if (g_app->dictation_send_pending) { + Canvas_App_Submit_Dictation(); + } else { + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Ready to send"); + } + } + if (!g_app->dictation_overlay_visible && + g_app->scene.selected_index >= 0 && + g_app->scene.selected_index < + (int32)Dowa_Array_Length(g_app->scene.p_entities) && + (IsKeyDown(KEY_LEFT_CONTROL) || + IsKeyDown(KEY_RIGHT_CONTROL) || + IsKeyDown(KEY_LEFT_SUPER) || + IsKeyDown(KEY_RIGHT_SUPER)) && + IsKeyPressed(KEY_ENTER)) { + Canvas_Entity *p_selected = + &g_app->scene.p_entities[g_app->scene.selected_index]; + if (p_selected->type == CANVAS_ENTITY_TEXT_AREA && + p_selected->text[0]) { + size_t length = strlen(p_selected->text); + if (length > 0 && p_selected->text[length - 1] == '\n') { + p_selected->text[length - 1] = '\0'; + } + if (Canvas_App_Submit_Prompt(p_selected->text)) { + p_selected->text[0] = '\0'; + p_selected->text_cursor = 0; + p_selected->text_selection_anchor = 0; + } + } + } + + char agent_response[CANVAS_AGENT_RESPONSE_CAPACITY]; + char agent_error[512]; + Canvas_Agent_Status agent_status = g_app->agent_initialized ? + Canvas_Agent_Service_Poll( + &g_app->agent_service, + agent_response, + sizeof(agent_response), + agent_error, + sizeof(agent_error)) : + CANVAS_AGENT_IDLE; + if (agent_status == CANVAS_AGENT_READY) { + Canvas_App_Apply_Agent_Response(agent_response); + } else if (agent_status == CANVAS_AGENT_ERROR) { + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (p_pending) { + snprintf(p_pending->label, sizeof(p_pending->label), "Copilot unavailable"); + Canvas_App_Set_Conversation_Text( + p_pending, + g_app->pending_prompt, + "System", + agent_error); + p_pending->agent_working = FALSE; + } else if (g_app->pending_prompt[0]) { + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + Canvas_Scene_Add_Conversation( + &g_app->scene, + "Copilot unavailable", + g_app->pending_prompt, + agent_error, + position); + } + g_app->pending_conversation_id = 0; + g_app->pending_prompt[0] = '\0'; + } + Canvas_Web_Surface_Sync(&g_app->web_surface, &g_app->camera, &g_app->scene); BeginDrawing(); @@ -162,6 +695,7 @@ g_app->font, &g_app->theme); #endif + Canvas_App_Draw_Listening_Indicator(); EndDrawing(); g_app->frame_count++; #if !defined(PLATFORM_WEB) @@ -171,7 +705,7 @@ } #endif if (g_app->p_screenshot_path && - g_app->frame_count == 180) { + g_app->frame_count == g_app->screenshot_frame) { TakeScreenshot(g_app->p_screenshot_path); } } @@ -196,7 +730,21 @@ } memset(g_app, 0, sizeof(*g_app)); g_app->p_arena = p_arena; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Idle"); g_app->p_screenshot_path = getenv("INFINITE_CANVAS_SCREENSHOT"); + g_app->screenshot_frame = 180; + const char *p_screenshot_frame = + getenv("INFINITE_CANVAS_SCREENSHOT_FRAME"); + if (p_screenshot_frame) { + uint32 screenshot_frame = + (uint32)strtoul(p_screenshot_frame, NULL, 10); + if (screenshot_frame > 0) { + g_app->screenshot_frame = screenshot_frame; + } + } Canvas_Theme_Resolve( &g_app->theme, Canvas_Theme_Mode_From_String(getenv("INFINITE_CANVAS_THEME"))); @@ -219,6 +767,11 @@ g_app->theme.resolved_mode == CANVAS_THEME_DARK ? TRUE : FALSE)) { TraceLog(LOG_WARNING, "Web surface initialization failed"); } + g_app->agent_initialized = + Canvas_Agent_Service_Init(&g_app->agent_service, p_arena); + if (!g_app->agent_initialized) { + TraceLog(LOG_WARNING, "Agent service initialization failed"); + } g_app->font = LoadFontEx("infinite_canvas/assets/Inter-Variable.ttf", 48, NULL, 0); g_app->owns_font = g_app->font.texture.id != 0; @@ -231,6 +784,13 @@ } else { Canvas_Scene_Add_Demo(&g_app->scene); } + if (getenv("INFINITE_CANVAS_DICTATION_AUTOSTART")) { + Canvas_App_Set_Dictation_Active(TRUE); + } + const char *p_initial_prompt = getenv("INFINITE_CANVAS_AGENT_PROMPT"); + if (p_initial_prompt && p_initial_prompt[0]) { + Canvas_App_Submit_Prompt(p_initial_prompt); + } const char *p_image_source = getenv("INFINITE_CANVAS_IMAGE_SOURCE"); if (p_image_source) { for (size_t index = 0; index < Dowa_Array_Length(g_app->scene.p_entities); index++) { @@ -258,6 +818,9 @@ emscripten_set_main_loop(Canvas_App_Draw_Frame, 0, 1); #else while (!WindowShouldClose()) Canvas_App_Draw_Frame(); + if (g_app->agent_initialized) { + Canvas_Agent_Service_Shutdown(&g_app->agent_service); + } Canvas_Web_Surface_Shutdown(&g_app->web_surface); if (g_app->owns_font) UnloadFont(g_app->font); Dowa_Arena_Free(p_arena);