Mercurial
changeset 280:49e9e591c9bb
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
line wrap: on
line diff
--- a/dictation/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -70,6 +70,7 @@ requirement("nvidia-cublas-cu12"), requirement("nvidia-cudnn-cu12"), ], + visibility = ["//infinite_canvas:__pkg__"], ) for command in [ "server",
--- a/dictation/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -36,11 +36,19 @@ ```bash DICTATION_COMPUTE_TYPE=int8_float16 \ DICTATION_MAX_SESSIONS=1 \ -DICTATION_PARTIAL_INTERVAL_MS=1200 \ -DICTATION_SILENCE_MS=700 \ +DICTATION_PARTIAL_INTERVAL_MS=500 \ +DICTATION_SILENCE_MS=400 \ bazel run //dictation:server ``` The first version intentionally binds to loopback and does not configure STUN or TURN. Remote WebRTC access requires HTTPS, authentication, origin policy, and usually a TURN service. + +Infinite Canvas hosts this page as a hidden CEF/iframe transport. Pressing `M` +shows partial and final transcript text in a centered Raylib caption; pressing +`Enter` sends the accumulated text to Qwen session orchestration: + +```bash +bazel run //infinite_canvas:agent_dev +```
--- a/dictation/audio.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/audio.py Tue Aug 18 19:14:53 2026 -0700 @@ -24,8 +24,8 @@ *, sample_rate: int = 16000, speech_threshold: float = 0.012, - silence_ms: int = 700, - partial_interval_ms: int = 1200, + silence_ms: int = 400, + partial_interval_ms: int = 500, max_utterance_seconds: int = 30, pre_roll_ms: int = 200, ) -> None:
--- a/dictation/config.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/config.py Tue Aug 18 19:14:53 2026 -0700 @@ -84,11 +84,11 @@ max_sessions=_integer("DICTATION_MAX_SESSIONS", 1, 1, 8), partial_interval_ms=_integer( "DICTATION_PARTIAL_INTERVAL_MS", - 1200, 500, + 250, 10000, ), - silence_ms=_integer("DICTATION_SILENCE_MS", 700, 200, 5000), + silence_ms=_integer("DICTATION_SILENCE_MS", 400, 200, 5000), max_utterance_seconds=_integer( "DICTATION_MAX_UTTERANCE_SECONDS", 30,
--- a/dictation/config_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/config_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -14,6 +14,8 @@ self.assertEqual(config.port, 8090) self.assertEqual(config.max_sessions, 1) self.assertEqual(config.compute_type, "int8_float16") + self.assertEqual(config.partial_interval_ms, 500) + self.assertEqual(config.silence_ms, 400) self.assertTrue(str(config.model_dir).endswith("faster-whisper-small")) def test_rejects_invalid_session_limit(self):
--- a/dictation/server_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/server_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -37,7 +37,21 @@ self.assertEqual(health.status_code, 200) self.assertEqual(health.json()["status"], "ok") self.assertIn("Zenbu Dictation", client.get("/").text) - self.assertIn("RTCPeerConnection", client.get("/dictation.js").text) + javascript = client.get("/dictation.js").text + self.assertIn("RTCPeerConnection", javascript) + self.assertIn("zenbu.dictation.event", javascript) + self.assertIn('publishDictation("partial"', javascript) + self.assertIn('publishDictation("final"', javascript) + self.assertIn('setStatus("Starting microphone..."', javascript) + self.assertIn("ZenbuDictationToggle", javascript) + self.assertIn("ZenbuDictationStart", javascript) + self.assertIn("ZenbuDictationStop", javascript) + self.assertIn("ZenbuDictationCommit", javascript) + self.assertIn("zenbu.dictation.set-active", javascript) + self.assertIn("zenbu.dictation.commit", javascript) + self.assertIn("prepareCanvasMicrophone", javascript) + self.assertIn("track.enabled = false", javascript) + self.assertIn("if (!stream)", javascript) def test_rejects_non_offer_sdp(self): with TestClient(create_app(test_config(), FakeTranscriber())) as client:
--- a/dictation/session.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/session.py Tue Aug 18 19:14:53 2026 -0700 @@ -54,8 +54,11 @@ @channel.on("message") def on_message(message: Any) -> None: - if message == "stop": - self.stopping = True + if message in {"commit", "pause", "stop"}: + if message == "stop": + self.stopping = True + if self.partial_task and not self.partial_task.done(): + self.partial_task.cancel() event = self.segmenter.flush() if event and event.samples is not None: self._start_final(event.samples) @@ -155,6 +158,26 @@ ) ) + async def _cancel_pending_ice_transactions(self) -> None: + # aioice 0.10.2 can leave STUN retry timers armed after its datagram + # transport closes. Cancel their futures first so Transaction.run() + # clears each timer before RTCPeerConnection.close() drops sockets. + ice_transports = getattr( + self.peer, + "_RTCPeerConnection__iceTransports", + (), + ) + for ice_transport in ice_transports: + connection = getattr(ice_transport, "_connection", None) + for protocol in getattr(connection, "_protocols", ()): + for transaction in tuple( + getattr(protocol, "transactions", {}).values() + ): + future = getattr(transaction, "_Transaction__future", None) + if future is not None and not future.done(): + future.cancel() + await asyncio.sleep(0) + async def close(self) -> None: if self.closed: return @@ -174,4 +197,5 @@ await asyncio.gather(*tasks, return_exceptions=True) if self.channel and self.channel.readyState != "closed": self.channel.close() + await self._cancel_pending_ice_transactions() await self.peer.close()
--- a/dictation/transcriber.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/transcriber.py Tue Aug 18 19:14:53 2026 -0700 @@ -49,10 +49,10 @@ loop = asyncio.get_running_loop() self._model = await loop.run_in_executor( self._executor, - self._load, + self._load_and_warmup, ) - def _load(self): + def _load_and_warmup(self): from faster_whisper import WhisperModel if not (self._model_dir / "model.bin").is_file(): @@ -60,12 +60,25 @@ f"Model not found at {self._model_dir}. " "Run: bazel run //dictation:download_model" ) - return WhisperModel( + model = WhisperModel( str(self._model_dir), device="cuda", compute_type=self._compute_type, local_files_only=True, ) + # Loading weights does not initialize all CUDA kernels. Execute and + # consume one short silent inference now so the user's first utterance + # does not pay the one-time GPU setup cost. + segments, _ = model.transcribe( + np.zeros(8000, dtype=np.float32), + beam_size=1, + best_of=1, + condition_on_previous_text=False, + vad_filter=False, + without_timestamps=True, + ) + list(segments) + return model async def transcribe( self, @@ -89,10 +102,11 @@ ) -> Transcript: segments, info = self._model.transcribe( samples, - beam_size=5 if final else 1, - best_of=5 if final else 1, + beam_size=1, + best_of=1, condition_on_previous_text=False, vad_filter=False, + without_timestamps=True, ) text = "".join(segment.text for segment in segments).strip() return Transcript(
--- a/dictation/web/dictation.js Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/web/dictation.js Tue Aug 18 19:14:53 2026 -0700 @@ -9,10 +9,26 @@ let stream = null; let sessionId = null; let stopResolver = null; +let captureActive = false; +let connectionPromise = null; +const canvasTransport = + new URLSearchParams(window.location.search).get("canvas") === "1"; + +function publishDictation(kind, text) { + const payload = { + type: "zenbu.dictation.event", + kind, + text, + }; + window.parent?.postMessage(payload, "*"); + document.title = + `Zenbu Dictation|${Date.now()}|${kind}|${encodeURIComponent(text)}`; +} function setStatus(message, state = "") { status.textContent = message; status.dataset.state = state; + publishDictation("status", message); } function waitForIceGathering(connection) { @@ -27,6 +43,24 @@ }); } +function waitForTranscriptChannel(transcriptChannel) { + if (transcriptChannel.readyState === "open") return Promise.resolve(); + return new Promise((resolve, reject) => { + const timeout = window.setTimeout( + () => reject(new Error("Transcript channel readiness timed out")), + 10000, + ); + transcriptChannel.addEventListener("open", () => { + window.clearTimeout(timeout); + resolve(); + }, { once: true }); + transcriptChannel.addEventListener("close", () => { + window.clearTimeout(timeout); + reject(new Error("Transcript channel closed during startup")); + }, { once: true }); + }); +} + function handleTranscript(message) { let event; try { @@ -37,20 +71,22 @@ } switch (event.type) { case "ready": - setStatus("Listening", "ready"); + setStatus(captureActive ? "Listening" : "Idle", "ready"); break; case "speech.started": setStatus("Speech detected", "working"); break; case "transcript.partial": partial.textContent = event.text; + publishDictation("partial", event.text); break; case "transcript.final": finalText.value += `${finalText.value ? " " : ""}${event.text}`; + publishDictation("final", event.text); partial.textContent = "Waiting for speech..."; break; case "speech.ended": - setStatus("Listening", "ready"); + setStatus(captureActive ? "Listening" : "Idle", "ready"); stopResolver?.(); stopResolver = null; break; @@ -60,11 +96,9 @@ } } -async function start() { - if (peer) return; - startButton.disabled = true; - setStatus("Requesting microphone access...", "working"); - try { +async function createConnection() { + if (peer && !["closed", "failed"].includes(peer.connectionState)) return; + if (!stream) { stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, @@ -73,6 +107,9 @@ }, video: false, }); + } + for (const track of stream.getTracks()) track.enabled = false; + try { peer = new RTCPeerConnection(); channel = peer.createDataChannel("transcripts", { ordered: true }); channel.addEventListener("message", handleTranscript); @@ -82,7 +119,7 @@ peer.addEventListener("connectionstatechange", () => { if (peer?.connectionState === "failed") { setStatus("WebRTC connection failed", "error"); - void stop(); + void stop(false, true); } }); for (const track of stream.getTracks()) peer.addTrack(track, stream); @@ -105,23 +142,50 @@ sdp: answer.sdp, type: answer.type, }); + await waitForTranscriptChannel(channel); stopButton.disabled = false; - setStatus("Connecting...", "working"); } catch (error) { - setStatus(error.message || "Unable to start dictation", "error"); - await stop(); + await stop(false, true); + throw error; } } -async function stop(waitForFinal = true) { +async function ensureConnection() { + if (peer && ["connected", "completed"].includes(peer.connectionState)) return; + if (!connectionPromise) { + connectionPromise = createConnection().finally(() => { + connectionPromise = null; + }); + } + await connectionPromise; +} + +async function start() { + captureActive = true; + startButton.disabled = true; + setStatus("Starting microphone...", "working"); + try { + await ensureConnection(); + for (const track of stream.getTracks()) track.enabled = true; + setStatus( + channel?.readyState === "open" ? "Listening" : "Connecting...", + channel?.readyState === "open" ? "ready" : "working", + ); + } catch (error) { + captureActive = false; + setStatus(error.message || "Unable to start dictation", "error"); + } +} + +async function stop(waitForFinal = true, releaseStream = !canvasTransport) { const closingSession = sessionId; - sessionId = null; const tracks = stream?.getTracks() || []; + captureActive = false; if (channel?.readyState === "open" && waitForFinal) { const ended = new Promise(resolve => { stopResolver = resolve; }); - channel.send("stop"); + channel.send(releaseStream ? "stop" : "pause"); for (const track of tracks) track.enabled = false; await Promise.race([ ended, @@ -129,8 +193,18 @@ ]); stopResolver = null; } - for (const track of tracks) track.stop(); - stream = null; + for (const track of tracks) { + track.enabled = false; + if (releaseStream) track.stop(); + } + if (!releaseStream) { + startButton.disabled = false; + stopButton.disabled = true; + setStatus("Idle"); + return; + } + sessionId = null; + if (releaseStream) stream = null; channel?.close(); channel = null; peer?.close(); @@ -148,4 +222,30 @@ startButton.addEventListener("click", () => void start()); stopButton.addEventListener("click", () => void stop()); -window.addEventListener("pagehide", () => void stop(false)); +window.ZenbuDictationStart = () => start(); +window.ZenbuDictationStop = () => stop(); +window.ZenbuDictationCommit = () => { + if (channel?.readyState === "open") channel.send("commit"); +}; +window.ZenbuDictationToggle = () => captureActive ? stop() : start(); +window.addEventListener("message", event => { + if (event.data?.type === "zenbu.dictation.set-active") { + void (event.data.active ? start() : stop()); + } else if (event.data?.type === "zenbu.dictation.commit") { + window.ZenbuDictationCommit(); + } +}); +window.addEventListener("pagehide", () => void stop(false, true)); + +async function prepareCanvasMicrophone() { + if (!canvasTransport || peer) return; + try { + await ensureConnection(); + for (const track of stream.getTracks()) track.enabled = false; + setStatus("Idle"); + } catch (error) { + setStatus(error.message || "Unable to prepare microphone", "error"); + } +} + +void prepareCanvasMicrophone();
--- a/dictation/webrtc_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/webrtc_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -60,6 +60,16 @@ class WebRtcTest(unittest.IsolatedAsyncioTestCase): async def test_audio_track_returns_transcript_events(self): + loop_errors = [] + loop = asyncio.get_running_loop() + previous_exception_handler = loop.get_exception_handler() + loop.set_exception_handler( + lambda _loop, context: loop_errors.append(context) + ) + self.addCleanup( + loop.set_exception_handler, + previous_exception_handler, + ) service = DictationService(test_config(), FakeTranscriber()) await service.start() client = RTCPeerConnection() @@ -131,11 +141,17 @@ self.assertIn("speech.started", event_types) self.assertIn("transcript.partial", event_types) self.assertIn("transcript.final", event_types) + channel.send("commit") + await asyncio.sleep(0.2) + self.assertNotEqual(client.connectionState, "closed") + self.assertIn(answer["sessionId"], service.sessions) finally: await client.close() await service.close() + await asyncio.sleep(0.6) Path(audio_file.name).unlink(missing_ok=True) self.assertEqual(service.sessions, {}) + self.assertEqual(loop_errors, []) if __name__ == "__main__":
--- a/infinite_canvas/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -2,6 +2,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//gui_ze:gui_ze.bzl", "move_files_into_dir") load("//third_party/raylib:raylib.bzl", "raylib_binary") @@ -75,6 +76,28 @@ }), ) +cc_library( + name = "agent_service", + srcs = select({ + "//config:linux": ["agent_service_copilot.c"], + "//config:macos": ["agent_service_stub.c"], + "//config:windows": ["agent_service_stub.c"], + "//conditions:default": ["agent_service_stub.c"], + }), + hdrs = ["agent_service.h"], + deps = [ + ":canvas", + "//dowa:dowa", + ] + select({ + "//config:linux": ["//mrjunejune:inference_bridge"], + "//conditions:default": [], + }), + linkopts = select({ + "//config:linux": ["-lpthread"], + "//conditions:default": [], + }), +) + filegroup( name = "inter_font", srcs = ["assets/Inter-Variable.ttf"], @@ -124,6 +147,7 @@ "//conditions:default": [], }), deps = [ + ":agent_service", ":canvas", ":web_surface", "//dowa:dowa", @@ -153,6 +177,7 @@ }), defines = ["INFINITE_CANVAS_DEV_UI"], deps = [ + ":agent_service", ":canvas", ":dev_ui", ":theme", @@ -174,6 +199,31 @@ data = [":dev_bin"], ) +sh_binary( + name = "orchestration_dev", + srcs = ["run_orchestration_dev.sh"], + data = [ + ":dev", + "//dictation:server", + "//mrjunejune:canvas_orchestration_launcher", + "//mrjunejune/inference:copilot_sidecar_zip", + "//mrjunejune/inference:litellm_config.yaml", + "//mrjunejune/inference:litellm_proxy_zip", + "@copilot_cli_linux_x86_64//:copilot", + "@python_3_11//:files", + "@python_3_11//:python3", + ], + target_compatible_with = select({ + "//config:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +alias( + name = "agent_dev", + actual = ":orchestration_dev", +) + alias( name = "app", actual = select({ @@ -208,6 +258,7 @@ ], defines = ["INFINITE_CANVAS_DEV_UI"], deps = [ + ":agent_service", ":canvas", ":dev_ui", ":web_surface", @@ -245,6 +296,41 @@ deps = [":canvas"], ) +cc_test( + name = "agent_service_test", + srcs = ["agent_service_test.c"], + args = [ + "$(rootpath //mrjunejune/test:inference_bridge_fake_sidecar)", + ], + data = [ + "//mrjunejune/test:inference_bridge_fake_sidecar", + ], + deps = [ + ":agent_service", + "//dowa:dowa", + ], + linkopts = ["-lpthread"], + target_compatible_with = select({ + "//config:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +sh_test( + name = "dictation_policy_test", + srcs = ["dictation_policy_test.sh"], + args = [ + "$(rootpath :web_surface_native.cc)", + "$(rootpath :main.c)", + "$(rootpath :dev_ui.c)", + ], + data = [ + "dev_ui.c", + "main.c", + "web_surface_native.cc", + ], +) + sh_binary( name = "serve", srcs = ["serve.sh"],
--- a/infinite_canvas/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -41,10 +41,14 @@ - Left-mouse drag on an entity: select and move it - `P`: pin or unpin the selected entity in screen space so it remains visible while the camera pans and zooms; pinned entities show a Lucide pin glyph +- `M`: show centered live dictation text and toggle microphone capture +- `Enter`: send visible dictation text to the spatial agent +- `Ctrl/Cmd+Enter`: submit the selected text area's thought to Copilot - `Delete`/`Backspace`: fade out and remove the selected entity - Development toolbar: spawn/clear entities, reset the camera, or restore the deterministic demo scene; spawned entities fade in and Clear fades all - entities out before removal + entities out before removal. Use the header arrow to collapse the toolbar + into a small restore button. - `View camera context`: open a live, scrollable snapshot of entities intersecting the camera viewport and their current semantic values - `Lucide icon gallery`: create or revisit a dedicated canvas region containing @@ -87,6 +91,66 @@ ISC-licensed vector data is generated from Lucide 1.31.0 and the visible grid rows are drawn directly with Raylib primitives. +## Spatial agent sessions + +Authenticate the Copilot CLI once: + +```bash +bazel run //mrjunejune:run_inference_stack -- --authenticate +``` + +Then start dictation, the Copilot LiteLLM gateway, and the canvas with one +target: + +```bash +bazel run //infinite_canvas:orchestration_dev +``` + +`//infinite_canvas:agent_dev` remains an alias. Closing the canvas stops the +supervised local services. + +Press `M` to start or stop the local WebRTC microphone session. The browser +transport is prewarmed and remains hidden: it acquires the native microphone +permission, device, signaling, ICE connection, and transcript data channel +ahead of the first hotkey while keeping the audio track disabled. `M` therefore +only unmutes an already-ready channel, so speech from the beginning of the +utterance is retained. Partial and final speech appear directly as minimal centered text +inside a retained `Dictation` text-area entity spawned near the camera center. +The entity word-wraps, scrolls to the latest line, and can be dragged by its +header to any position on the board. The same scratchpad is reused for every +thought and clears after submission; it does not create duplicate dictation +windows. A small pulsing `Listening` indicator +remains in the bottom-left while capture is active. In-progress speech uses +muted text; `Backspace` clears the entire current thought without stopping +capture. Pausing with `M` and resuming appends to the same thought rather than +replacing it. Press `Enter` to commit and send the current utterance while the +microphone stays live for the next thought; press `M` or `Escape` to stop it. +For typed input, select a text area and press `Ctrl/Cmd+Enter`. +Native CEF grants audio only to the fixed loopback dictation transport; it does +not show an inaccessible browser permission prompt or grant microphone access +to normal canvas web entities. + +The Copilot SDK receives the visible camera context, including existing +conversation entity IDs, through the shared `canvas_orchestrator` prompt +profile. It returns a routing decision to append to an existing session or +create a new one. Requests run off the render thread through the shared +`Inference_Bridge`. Conversation cards retain +their own title, transcript, working state, scroll position, selection, and +pinning state, forming the first rich-context container for future typed image +and component resources. Runtime configuration and Copilot token paths come +from the ignored `mrjunejune/.config`; no separate canvas environment file is +required. The Dictation scratchpad is excluded from camera context because its +text is submitted separately, and only visible conversation cards are eligible +append targets. Conversation cards follow the latest appended message and can +be collapsed from their header. The archive icon moves a session to the +off-canvas parking lot, excluding it from context; `Parking lot (N)` in +Developer Controls jumps there, and the restore icon returns the session to +its previous board position. Collapse and expansion use the same eased retained +animation model as accordion entities rather than snapping between heights. +Canvas startup also pre-creates the persistent orchestration SDK session and +three worker sessions, moving SDK session creation out of the first submitted +thought. + Tables support row selection, and notifications behave as bottom-right toast cards: multiple toasts overlap compactly, fan into a spaced stack on hover, and slide/fade away from either Undo or close. Position, size, stack expansion,
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service.h Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,39 @@ +#ifndef INFINITE_CANVAS_AGENT_SERVICE_H +#define INFINITE_CANVAS_AGENT_SERVICE_H + +#include "dowa/dowa.h" +#include "infinite_canvas/canvas.h" + +#define CANVAS_AGENT_RESPONSE_CAPACITY 16384 + +typedef enum { + CANVAS_AGENT_IDLE = 0, + CANVAS_AGENT_WORKING, + CANVAS_AGENT_READY, + CANVAS_AGENT_ERROR, +} Canvas_Agent_Status; + +typedef struct { + Dowa_Arena *p_arena; + void *p_native; + Canvas_Agent_Status status; + char response[CANVAS_AGENT_RESPONSE_CAPACITY]; + char error[512]; +} Canvas_Agent_Service; + +boolean Canvas_Agent_Service_Init( + Canvas_Agent_Service *p_service, + Dowa_Arena *p_arena); +boolean Canvas_Agent_Service_Submit( + Canvas_Agent_Service *p_service, + const char *p_prompt, + const char *p_context); +Canvas_Agent_Status Canvas_Agent_Service_Poll( + Canvas_Agent_Service *p_service, + char *p_response, + size_t response_capacity, + char *p_error, + size_t error_capacity); +void Canvas_Agent_Service_Shutdown(Canvas_Agent_Service *p_service); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service_copilot.c Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,276 @@ +#include "infinite_canvas/agent_service.h" + +#include "mrjunejune/inference_bridge.h" + +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#define CANVAS_AGENT_WARM_SESSION_COUNT 4 + +typedef struct { + pthread_mutex_t mutex; + Canvas_Agent_Service *p_service; + Inference_Bridge *p_bridge; + uint32 request_sequence; + uint32 warmed_sessions; + char request_id[64]; + char turn_content[CANVAS_AGENT_RESPONSE_CAPACITY]; + char turn_error[512]; + char turn_prompt[ + CANVAS_CONTEXT_MAX_LENGTH + CANVAS_ENTITY_TEXT_CAPACITY + 512]; +} Canvas_Agent_Copilot; + +static boolean Canvas_Agent_Event_Matches( + const Canvas_Agent_Copilot *p_copilot, + const Inference_Event *p_event) +{ + return p_event->request_id && + strcmp(p_copilot->request_id, p_event->request_id) == 0; +} + +static void Canvas_Agent_Handle_Event( + const Inference_Event *p_event, + void *p_user_data) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_user_data; + if (!p_copilot || !p_event) return; + + pthread_mutex_lock(&p_copilot->mutex); + if (strcmp(p_event->type, "bridge.closed") == 0) { + if (p_copilot->p_service->status == CANVAS_AGENT_WORKING) { + snprintf( + p_copilot->p_service->error, + sizeof(p_copilot->p_service->error), + "%s", + p_event->error_message && p_event->error_message[0] + ? p_event->error_message + : "Copilot orchestration sidecar closed"); + p_copilot->p_service->status = CANVAS_AGENT_ERROR; + } + pthread_mutex_unlock(&p_copilot->mutex); + return; + } + if (strcmp(p_event->type, "session.warmed") == 0 && + p_event->request_id && + strncmp(p_event->request_id, "canvas-warm-", 12) == 0) { + p_copilot->warmed_sessions++; + pthread_mutex_unlock(&p_copilot->mutex); + return; + } + if (!Canvas_Agent_Event_Matches(p_copilot, p_event)) { + pthread_mutex_unlock(&p_copilot->mutex); + return; + } + + if (strcmp(p_event->type, "assistant.completed") == 0) { + snprintf( + p_copilot->turn_content, + sizeof(p_copilot->turn_content), + "%s", + p_event->content ? p_event->content : ""); + } else if (strcmp(p_event->type, "turn.error") == 0) { + snprintf( + p_copilot->turn_error, + sizeof(p_copilot->turn_error), + "%s", + p_event->error_message && p_event->error_message[0] + ? p_event->error_message + : "Copilot orchestration failed"); + } else if (strcmp(p_event->type, "turn.done") == 0) { + if (p_event->failed || p_copilot->turn_error[0]) { + snprintf( + p_copilot->p_service->error, + sizeof(p_copilot->p_service->error), + "%s", + p_copilot->turn_error[0] + ? p_copilot->turn_error + : "Copilot orchestration failed"); + p_copilot->p_service->status = CANVAS_AGENT_ERROR; + } else if (p_copilot->turn_content[0]) { + snprintf( + p_copilot->p_service->response, + sizeof(p_copilot->p_service->response), + "%s", + p_copilot->turn_content); + p_copilot->p_service->status = CANVAS_AGENT_READY; + } else { + snprintf( + p_copilot->p_service->error, + sizeof(p_copilot->p_service->error), + "Copilot returned an empty response"); + p_copilot->p_service->status = CANVAS_AGENT_ERROR; + } + } + pthread_mutex_unlock(&p_copilot->mutex); +} + +boolean Canvas_Agent_Service_Init( + Canvas_Agent_Service *p_service, + Dowa_Arena *p_arena) +{ + memset(p_service, 0, sizeof(*p_service)); + p_service->p_arena = p_arena; + const char *p_sidecar = + getenv("INFINITE_CANVAS_COPILOT_SIDECAR_PATH"); + const char *p_cli = + getenv("INFINITE_CANVAS_COPILOT_CLI_PATH"); + if (!p_sidecar || !p_sidecar[0] || !p_cli || !p_cli[0]) { + return FALSE; + } + + Canvas_Agent_Copilot *p_copilot = (Canvas_Agent_Copilot *) + Dowa_Arena_Allocate(p_arena, sizeof(*p_copilot)); + if (!p_copilot) return FALSE; + memset(p_copilot, 0, sizeof(*p_copilot)); + if (pthread_mutex_init(&p_copilot->mutex, NULL) != 0) { + return FALSE; + } + p_copilot->p_service = p_service; + p_copilot->p_bridge = Inference_Bridge_Create( + p_sidecar, + p_cli, + Canvas_Agent_Handle_Event, + p_copilot); + if (!p_copilot->p_bridge || + !Inference_Bridge_Start(p_copilot->p_bridge)) { + if (p_copilot->p_bridge) { + Inference_Bridge_Destroy(p_copilot->p_bridge); + } + pthread_mutex_destroy(&p_copilot->mutex); + return FALSE; + } + const char *warm_sessions[CANVAS_AGENT_WARM_SESSION_COUNT] = { + "infinite-canvas-orchestrator", + "infinite-canvas-worker-1", + "infinite-canvas-worker-2", + "infinite-canvas-worker-3", + }; + for (uint32 index = 0; + index < CANVAS_AGENT_WARM_SESSION_COUNT; + index++) { + char request_id[32]; + snprintf( + request_id, + sizeof(request_id), + "canvas-warm-%u", + index); + if (!Inference_Bridge_Warm_Conversation( + p_copilot->p_bridge, + request_id, + warm_sessions[index], + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR, + 1, + 1, + index == 0 ? TRUE : FALSE)) { + Inference_Bridge_Destroy(p_copilot->p_bridge); + pthread_mutex_destroy(&p_copilot->mutex); + return FALSE; + } + } + for (int32 attempt = 0; attempt < 2000; attempt++) { + pthread_mutex_lock(&p_copilot->mutex); + boolean ready = + p_copilot->warmed_sessions == CANVAS_AGENT_WARM_SESSION_COUNT; + pthread_mutex_unlock(&p_copilot->mutex); + if (ready) break; + usleep(10000); + } + p_service->p_native = p_copilot; + p_service->status = CANVAS_AGENT_IDLE; + return TRUE; +} + +boolean Canvas_Agent_Service_Submit( + Canvas_Agent_Service *p_service, + const char *p_prompt, + const char *p_context) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_service->p_native; + if (!p_copilot || !p_prompt || !p_prompt[0]) return FALSE; + + pthread_mutex_lock(&p_copilot->mutex); + if (p_service->status == CANVAS_AGENT_WORKING) { + pthread_mutex_unlock(&p_copilot->mutex); + return FALSE; + } + p_copilot->request_sequence++; + snprintf( + p_copilot->request_id, + sizeof(p_copilot->request_id), + "canvas-%u", + p_copilot->request_sequence); + snprintf( + p_copilot->turn_prompt, + sizeof(p_copilot->turn_prompt), + "Visible canvas context:\n%s\n\nNew user thought:\n%s", + p_context ? p_context : "", + p_prompt); + p_copilot->turn_content[0] = '\0'; + p_copilot->turn_error[0] = '\0'; + p_service->response[0] = '\0'; + p_service->error[0] = '\0'; + p_service->status = CANVAS_AGENT_WORKING; + char request_id[sizeof(p_copilot->request_id)]; + snprintf(request_id, sizeof(request_id), "%s", p_copilot->request_id); + pthread_mutex_unlock(&p_copilot->mutex); + + if (!Inference_Bridge_Start_Turn( + p_copilot->p_bridge, + request_id, + "infinite-canvas-orchestrator", + p_copilot->turn_prompt, + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR, + 1, + 1, + NULL, + 0)) { + pthread_mutex_lock(&p_copilot->mutex); + snprintf( + p_service->error, + sizeof(p_service->error), + "Unable to start the Copilot orchestration turn"); + p_service->status = CANVAS_AGENT_ERROR; + pthread_mutex_unlock(&p_copilot->mutex); + return FALSE; + } + return TRUE; +} + +Canvas_Agent_Status Canvas_Agent_Service_Poll( + Canvas_Agent_Service *p_service, + char *p_response, + size_t response_capacity, + char *p_error, + size_t error_capacity) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_service->p_native; + if (!p_copilot) return CANVAS_AGENT_ERROR; + pthread_mutex_lock(&p_copilot->mutex); + Canvas_Agent_Status status = p_service->status; + if (status == CANVAS_AGENT_READY) { + snprintf(p_response, response_capacity, "%s", p_service->response); + p_service->status = CANVAS_AGENT_IDLE; + } else if (status == CANVAS_AGENT_ERROR) { + snprintf(p_error, error_capacity, "%s", p_service->error); + p_service->status = CANVAS_AGENT_IDLE; + } + pthread_mutex_unlock(&p_copilot->mutex); + return status; +} + +void Canvas_Agent_Service_Shutdown(Canvas_Agent_Service *p_service) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_service->p_native; + if (!p_copilot) return; + Inference_Bridge_Destroy(p_copilot->p_bridge); + p_copilot->p_bridge = NULL; + pthread_mutex_destroy(&p_copilot->mutex); + p_service->p_native = NULL; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service_stub.c Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,51 @@ +#include "infinite_canvas/agent_service.h" + +#include <stdio.h> +#include <string.h> + +boolean Canvas_Agent_Service_Init( + Canvas_Agent_Service *p_service, + Dowa_Arena *p_arena) +{ + memset(p_service, 0, sizeof(*p_service)); + p_service->p_arena = p_arena; + p_service->status = CANVAS_AGENT_IDLE; + return TRUE; +} + +boolean Canvas_Agent_Service_Submit( + Canvas_Agent_Service *p_service, + const char *p_prompt, + const char *p_context) +{ + (void)p_prompt; + (void)p_context; + snprintf( + p_service->error, + sizeof(p_service->error), + "Copilot SDK orchestration is not available on this platform yet"); + p_service->status = CANVAS_AGENT_ERROR; + return TRUE; +} + +Canvas_Agent_Status Canvas_Agent_Service_Poll( + Canvas_Agent_Service *p_service, + char *p_response, + size_t response_capacity, + char *p_error, + size_t error_capacity) +{ + (void)p_response; + (void)response_capacity; + Canvas_Agent_Status status = p_service->status; + if (status == CANVAS_AGENT_ERROR) { + snprintf(p_error, error_capacity, "%s", p_service->error); + p_service->status = CANVAS_AGENT_IDLE; + } + return status; +} + +void Canvas_Agent_Service_Shutdown(Canvas_Agent_Service *p_service) +{ + (void)p_service; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service_test.c Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,49 @@ +#include "infinite_canvas/agent_service.h" + +#include <assert.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +int main(int argc, char **p_argv) +{ + assert(argc == 2); + assert(setenv( + "INFINITE_CANVAS_COPILOT_SIDECAR_PATH", + p_argv[1], + 1) == 0); + assert(setenv( + "INFINITE_CANVAS_COPILOT_CLI_PATH", + p_argv[1], + 1) == 0); + + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + Canvas_Agent_Service service; + assert(Canvas_Agent_Service_Init(&service, p_arena)); + assert(Canvas_Agent_Service_Submit( + &service, + "Create a rendering session", + "entity id=42 type=Agent conversation")); + + Canvas_Agent_Status status = CANVAS_AGENT_WORKING; + char response[CANVAS_AGENT_RESPONSE_CAPACITY] = {0}; + char error[512] = {0}; + for (int32 attempt = 0; + attempt < 200 && status == CANVAS_AGENT_WORKING; + attempt++) { + usleep(10000); + status = Canvas_Agent_Service_Poll( + &service, + response, + sizeof(response), + error, + sizeof(error)); + } + assert(status == CANVAS_AGENT_READY); + assert(strstr(response, "hello traveler")); + + Canvas_Agent_Service_Shutdown(&service); + Dowa_Arena_Free(p_arena); + return 0; +}
--- a/infinite_canvas/canvas.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/canvas.c Tue Aug 18 19:14:53 2026 -0700 @@ -38,6 +38,10 @@ #define CANVAS_TEXT_AREA_FONT_SIZE 16.0f #define CANVAS_TEXT_AREA_SPACING 0.1f #define CANVAS_TEXT_AREA_LINE_HEIGHT 22.0f +#define CANVAS_CONVERSATION_EXPANDED_HEIGHT 300.0f +#define CANVAS_CONVERSATION_COLLAPSED_HEIGHT 58.0f +#define CANVAS_PARKING_LOT_X 2400.0f +#define CANVAS_PARKING_LOT_Y -600.0f typedef struct { int32 start; @@ -46,6 +50,13 @@ float width; } Canvas_Text_Line; +static int32 Canvas_Text_Build_Lines( + Font font, + const char *p_text, + float max_width, + Canvas_Text_Line *p_lines, + int32 line_capacity); + static float Canvas_Clamp(float value, float min_value, float max_value) { if (value < min_value) return min_value; @@ -743,6 +754,28 @@ }; } +static Rectangle Canvas_Conversation_Collapse_Bounds( + const Canvas_Entity *p_entity) +{ + return (Rectangle){ + p_entity->position.x + p_entity->size.x - 74.0f, + p_entity->position.y + 12.0f, + 28.0f, + 28.0f, + }; +} + +static Rectangle Canvas_Conversation_Park_Bounds( + const Canvas_Entity *p_entity) +{ + return (Rectangle){ + p_entity->position.x + p_entity->size.x - 40.0f, + p_entity->position.y + 12.0f, + 28.0f, + 28.0f, + }; +} + static Rectangle Canvas_Entity_Bounds(const Canvas_Entity *p_entity) { if (p_entity->type == CANVAS_ENTITY_CIRCLE) { @@ -753,6 +786,7 @@ p_entity->size.x * 2.0f, }; } + if (p_entity->type == CANVAS_ENTITY_LINE) { float end_x = p_entity->position.x + p_entity->size.x; float end_y = p_entity->position.y + p_entity->size.y; @@ -949,6 +983,16 @@ Canvas_Context_Append(p_writer, "url="); Canvas_Context_Append_Quoted(p_writer, p_entity->text); break; + case CANVAS_ENTITY_CONVERSATION: + Canvas_Context_Append(p_writer, "title="); + Canvas_Context_Append_Quoted(p_writer, p_entity->label); + Canvas_Context_Append(p_writer, " transcript="); + Canvas_Context_Append_Quoted(p_writer, p_entity->text); + Canvas_Context_Append( + p_writer, + " collapsed=%s", + p_entity->active ? "true" : "false"); + break; case CANVAS_ENTITY_LUCIDE_GALLERY: Canvas_Context_Append( p_writer, @@ -997,6 +1041,14 @@ p_entity->animation_amount <= 0.01f) { continue; } + if (p_entity->type == CANVAS_ENTITY_TEXT_AREA && + strcmp(p_entity->label, "Dictation") == 0) { + continue; + } + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + p_entity->parked) { + continue; + } if (!CheckCollisionRecs(viewport, Canvas_Entity_Bounds(p_entity))) continue; snapshot.visible_count++; @@ -1124,6 +1176,14 @@ entity.active = TRUE; snprintf(entity.text, sizeof(entity.text), "https://mrjunejune.com"); } + if (type == CANVAS_ENTITY_CONVERSATION) { + entity.size = (Vector2){ + 320.0f, + CANVAS_CONVERSATION_EXPANDED_HEIGHT, + }; + entity.animation_amount = 1.0f; + snprintf(entity.label, sizeof(entity.label), "Agent session"); + } if (type == CANVAS_ENTITY_LUCIDE_GALLERY) { entity.size = (Vector2){1440.0f, 1040.0f}; } @@ -1132,6 +1192,188 @@ return TRUE; } +uint32 Canvas_Scene_Add_Conversation( + Canvas_Scene *p_scene, + const char *p_title, + const char *p_prompt, + const char *p_response, + Vector2 position) +{ + if (!Canvas_Scene_Add( + p_scene, + CANVAS_ENTITY_CONVERSATION, + position)) { + return 0; + } + Canvas_Entity *p_entity = + &p_scene->p_entities[Dowa_Array_Length(p_scene->p_entities) - 1]; + p_entity->size = (Vector2){420.0f, 300.0f}; + snprintf( + p_entity->label, + sizeof(p_entity->label), + "%s", + p_title && p_title[0] ? p_title : "New agent session"); + snprintf( + p_entity->text, + sizeof(p_entity->text), + "You\n%s\n\nCopilot\n%s", + p_prompt, + p_response); + p_scene->selected_index = + (int32)Dowa_Array_Length(p_scene->p_entities) - 1; + return p_entity->id; +} + +boolean Canvas_Scene_Append_Conversation( + Canvas_Scene *p_scene, + uint32 entity_id, + const char *p_prompt, + const char *p_response) +{ + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + Canvas_Entity *p_entity = &p_scene->p_entities[index]; + if (p_entity->id != entity_id || + p_entity->type != CANVAS_ENTITY_CONVERSATION) { + continue; + } + size_t length = strlen(p_entity->text); + snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "\n\nYou\n%s\n\nCopilot\n%s", + p_prompt, + p_response); + p_scene->selected_index = (int32)index; + return TRUE; + } + return FALSE; +} + +static float Canvas_Conversation_Max_Scroll( + const Canvas_Entity *p_entity, + Font font) +{ + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 line_count = Canvas_Text_Build_Lines( + font, + p_entity->text, + p_entity->size.x - 32.0f, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + return fmaxf( + 0.0f, + (float)line_count * 18.0f - + (CANVAS_CONVERSATION_EXPANDED_HEIGHT - 82.0f)); +} + +void Canvas_Conversation_Scroll_To_End( + Canvas_Entity *p_entity, + Font font) +{ + if (!p_entity || p_entity->type != CANVAS_ENTITY_CONVERSATION) return; + p_entity->value = (int32)Canvas_Conversation_Max_Scroll(p_entity, font); +} + +void Canvas_Conversation_Set_Collapsed( + Canvas_Entity *p_entity, + boolean collapsed) +{ + if (!p_entity || p_entity->type != CANVAS_ENTITY_CONVERSATION) return; + p_entity->active = collapsed; +} + +size_t Canvas_Scene_Parked_Conversation_Count( + const Canvas_Scene *p_scene) +{ + size_t count = 0; + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + const Canvas_Entity *p_entity = &p_scene->p_entities[index]; + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + p_entity->parked) { + count++; + } + } + return count; +} + +boolean Canvas_Scene_Set_Conversation_Parked( + Canvas_Scene *p_scene, + uint32 entity_id, + boolean parked) +{ + Canvas_Entity *p_entity = NULL; + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + if (p_scene->p_entities[index].id == entity_id && + p_scene->p_entities[index].type == CANVAS_ENTITY_CONVERSATION) { + p_entity = &p_scene->p_entities[index]; + break; + } + } + if (!p_entity || p_entity->parked == parked) return FALSE; + if (parked) { + size_t slot = Canvas_Scene_Parked_Conversation_Count(p_scene); + p_entity->parked_restore_position = p_entity->position; + p_entity->position = (Vector2){ + CANVAS_PARKING_LOT_X + (float)(slot % 3) * 460.0f, + CANVAS_PARKING_LOT_Y + (float)(slot / 3) * 340.0f, + }; + p_entity->parked = TRUE; + p_entity->pinned = FALSE; + } else { + p_entity->position = p_entity->parked_restore_position; + p_entity->parked = FALSE; + } + return TRUE; +} + +boolean Canvas_Scene_Show_Parking_Lot( + Canvas_Scene *p_scene, + Canvas_Camera *p_camera) +{ + Rectangle bounds = {0}; + boolean found = FALSE; + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + Canvas_Entity *p_entity = &p_scene->p_entities[index]; + if (p_entity->type != CANVAS_ENTITY_CONVERSATION || + !p_entity->parked) { + continue; + } + Rectangle entity_bounds = Canvas_Entity_Bounds(p_entity); + if (!found) { + bounds = entity_bounds; + found = TRUE; + } else { + float right = fmaxf( + bounds.x + bounds.width, + entity_bounds.x + entity_bounds.width); + float bottom = fmaxf( + bounds.y + bounds.height, + entity_bounds.y + entity_bounds.height); + bounds.x = fminf(bounds.x, entity_bounds.x); + bounds.y = fminf(bounds.y, entity_bounds.y); + bounds.width = right - bounds.x; + bounds.height = bottom - bounds.y; + } + } + if (!found) return FALSE; + Canvas_Camera_Fit_Bounds( + p_camera, + bounds, + 80.0f, + 80.0f, + 80.0f, + 80.0f); + return TRUE; +} + void Canvas_Scene_Clear(Canvas_Scene *p_scene) { Dowa_Array_Clear(p_scene->p_entities); @@ -1205,6 +1447,18 @@ Canvas_Scene_Add(p_scene, CANVAS_ENTITY_IMAGE, (Vector2){-550.0f, 220.0f}); Canvas_Scene_Add(p_scene, CANVAS_ENTITY_WEB_CONTENT, (Vector2){-150.0f, 120.0f}); + Canvas_Scene_Add(p_scene, CANVAS_ENTITY_CONVERSATION, (Vector2){610.0f, 220.0f}); + Canvas_Entity *p_conversation = + &p_scene->p_entities[Dowa_Array_Length(p_scene->p_entities) - 1]; + snprintf( + p_conversation->label, + sizeof(p_conversation->label), + "Spatial agent sessions"); + snprintf( + p_conversation->text, + sizeof(p_conversation->text), + "System\nPress M to dictate and Enter to send, or select the text " + "area and press Ctrl/Cmd+Enter to ask Copilot."); } void Canvas_Scene_Add_Browser_Grid(Canvas_Scene *p_scene) @@ -1322,7 +1576,8 @@ case CANVAS_ENTITY_SWITCH: case CANVAS_ENTITY_TABLE: case CANVAS_ENTITY_SCROLL_AREA: - case CANVAS_ENTITY_IMAGE: { + case CANVAS_ENTITY_IMAGE: + case CANVAS_ENTITY_CONVERSATION: { float height = p_entity->size.y; if (p_entity->type == CANVAS_ENTITY_DROPDOWN && p_entity->active) { height += 8.0f + 3.0f * 40.0f; @@ -1613,11 +1868,12 @@ static Rectangle Canvas_Text_Area_Content_Bounds( const Canvas_Entity *p_entity) { + float header_height = p_entity->label[0] ? 26.0f : 0.0f; return (Rectangle){ p_entity->position.x + CANVAS_TEXT_AREA_PADDING, - p_entity->position.y + CANVAS_TEXT_AREA_PADDING, + p_entity->position.y + CANVAS_TEXT_AREA_PADDING + header_height, p_entity->size.x - CANVAS_TEXT_AREA_PADDING * 2.0f, - p_entity->size.y - CANVAS_TEXT_AREA_PADDING * 2.0f, + p_entity->size.y - CANVAS_TEXT_AREA_PADDING * 2.0f - header_height, }; } @@ -1798,6 +2054,7 @@ p_entity->text_scroll_y + content.height) { p_entity->text_scroll_y = cursor_bottom - content.height; } + float max_scroll = fmaxf( 0.0f, (float)line_count * CANVAS_TEXT_AREA_LINE_HEIGHT - content.height); @@ -1807,6 +2064,31 @@ max_scroll); } +void Canvas_Text_Area_Set_Content( + Canvas_Entity *p_entity, + Font font, + const char *p_text, + boolean muted) +{ + if (!p_entity || p_entity->type != CANVAS_ENTITY_TEXT_AREA) return; + snprintf(p_entity->text, sizeof(p_entity->text), "%s", p_text ? p_text : ""); + p_entity->text_cursor = (int32)strlen(p_entity->text); + p_entity->text_selection_anchor = p_entity->text_cursor; + p_entity->text_muted = muted; + + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + Rectangle content = Canvas_Text_Area_Content_Bounds(p_entity); + int32 line_count = Canvas_Text_Build_Lines( + font, + p_entity->text, + content.width, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + p_entity->text_scroll_y = fmaxf( + 0.0f, + (float)line_count * CANVAS_TEXT_AREA_LINE_HEIGHT - content.height); +} + static void Canvas_Text_Update_Keyboard(Canvas_Entity *p_entity, Font font) { boolean shortcut = Canvas_Control_Is_Down(); @@ -2130,11 +2412,14 @@ for (size_t index = 0; index < Dowa_Array_Length(p_scene->p_entities); index++) { Canvas_Entity *p_entity = &p_scene->p_entities[index]; if (p_entity->type != CANVAS_ENTITY_ACCORDION && - p_entity->type != CANVAS_ENTITY_SWITCH) { + p_entity->type != CANVAS_ENTITY_SWITCH && + p_entity->type != CANVAS_ENTITY_CONVERSATION) { continue; } - float target = p_entity->active ? 1.0f : 0.0f; + float target = p_entity->type == CANVAS_ENTITY_CONVERSATION + ? (p_entity->active ? 0.0f : 1.0f) + : (p_entity->active ? 1.0f : 0.0f); float speed = p_entity->type == CANVAS_ENTITY_SWITCH ? 7.0f : 4.5f; float step = GetFrameTime() * speed; if (p_entity->animation_amount < target) { @@ -2149,6 +2434,21 @@ float amount = p_entity->animation_amount; float eased = amount * amount * (3.0f - 2.0f * amount); if (p_entity->type == CANVAS_ENTITY_SWITCH) continue; + if (p_entity->type == CANVAS_ENTITY_CONVERSATION) { + float height = CANVAS_CONVERSATION_COLLAPSED_HEIGHT + + (CANVAS_CONVERSATION_EXPANDED_HEIGHT - + CANVAS_CONVERSATION_COLLAPSED_HEIGHT) * eased; + if (p_entity->pinned) { + float pinned_scale = + p_entity->pinned_screen_size.x / p_entity->size.x; + p_entity->pinned_screen_size.y = height * pinned_scale; + p_entity->size.y = + p_entity->pinned_screen_size.y / p_camera->zoom; + } else { + p_entity->size.y = height; + } + continue; + } if (p_entity->pinned) { float pinned_scale = p_entity->pinned_screen_size.x / 300.0f; p_entity->pinned_screen_size.y = @@ -2217,6 +2517,27 @@ 240.0f); consumed_scroll = TRUE; } + } else if (p_hovered->type == CANVAS_ENTITY_CONVERSATION && + !control_down) { + float wheel = GetMouseWheelMove(); + if (wheel != 0.0f) { + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 line_count = Canvas_Text_Build_Lines( + font, + p_hovered->text, + p_hovered->size.x - 32.0f, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + float max_scroll = fmaxf( + 0.0f, + (float)line_count * 18.0f - + (p_hovered->size.y - 82.0f)); + p_hovered->value = (int32)Canvas_Clamp( + (float)p_hovered->value - wheel * 36.0f, + 0.0f, + max_scroll); + consumed_scroll = TRUE; + } } else if (p_hovered->type == CANVAS_ENTITY_LUCIDE_GALLERY && !control_down) { float wheel = GetMouseWheelMove(); @@ -2256,6 +2577,27 @@ if (p_scene->selected_index >= 0) { Canvas_Entity *p_entity = &p_scene->p_entities[p_scene->selected_index]; + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + CheckCollisionPointRec( + world_pointer, + Canvas_Conversation_Park_Bounds(p_entity))) { + Canvas_Scene_Set_Conversation_Parked( + p_scene, + p_entity->id, + p_entity->parked ? FALSE : TRUE); + p_scene->pressed_index = -1; + return TRUE; + } + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + CheckCollisionPointRec( + world_pointer, + Canvas_Conversation_Collapse_Bounds(p_entity))) { + Canvas_Conversation_Set_Collapsed( + p_entity, + p_entity->active ? FALSE : TRUE); + p_scene->pressed_index = -1; + return TRUE; + } if (p_entity->type == CANVAS_ENTITY_LUCIDE_GALLERY) { p_entity->active = CheckCollisionPointRec( world_pointer, @@ -2676,7 +3018,8 @@ case CANVAS_ENTITY_TABLE: case CANVAS_ENTITY_NOTIFICATION: case CANVAS_ENTITY_SCROLL_AREA: - case CANVAS_ENTITY_IMAGE: { + case CANVAS_ENTITY_IMAGE: + case CANVAS_ENTITY_CONVERSATION: { DrawRectangleRoundedLinesEx( (Rectangle){ p_entity->position.x - padding, @@ -2751,13 +3094,23 @@ { Rectangle content = Canvas_Text_Area_Content_Bounds(p_entity); if (!p_entity->text[0]) { + boolean dictation = strcmp(p_entity->label, "Dictation") == 0; DrawTextEx( font, - "Write something...", + dictation ? "Speak now..." : "Type a thought for Copilot...", (Vector2){content.x, content.y}, CANVAS_TEXT_AREA_FONT_SIZE, CANVAS_TEXT_AREA_SPACING, p_theme->text_muted); + DrawTextEx( + font, + dictation + ? "Enter to send | Backspace to clear" + : "Ctrl/Cmd+Enter to send", + (Vector2){content.x, content.y + 22.0f}, + 12.0f, + 0.0f, + Canvas_Color_Fade(p_theme->text_muted, 0.72f)); if (p_entity->active && fmod(GetTime(), 1.0) < 0.55) { DrawLineEx( (Vector2){content.x, content.y}, @@ -2840,7 +3193,7 @@ (Vector2){content.x, y}, CANVAS_TEXT_AREA_FONT_SIZE, CANVAS_TEXT_AREA_SPACING, - p_theme->text); + p_entity->text_muted ? p_theme->text_muted : p_theme->text); if (p_entity->active && line == cursor_line && @@ -3012,6 +3365,29 @@ 14, (p_entity->active ? 2.0f : 1.0f) / zoom, p_entity->active ? p_theme->accent : p_theme->border); + if (p_entity->label[0]) { + DrawTextEx( + font, + p_entity->label, + (Vector2){ + bounds.x + CANVAS_TEXT_AREA_PADDING, + bounds.y + 11.0f, + }, + 12.0f, + 0.0f, + p_theme->text_muted); + DrawLineEx( + (Vector2){ + bounds.x + bounds.width - 38.0f, + bounds.y + 17.0f, + }, + (Vector2){ + bounds.x + bounds.width - 16.0f, + bounds.y + 17.0f, + }, + 2.0f / zoom, + p_theme->border); + } Canvas_Draw_Text_Area_Content(p_entity, font, p_theme); break; } @@ -3505,6 +3881,122 @@ #endif break; } + case CANVAS_ENTITY_CONVERSATION: { + Rectangle bounds = { + p_entity->position.x, + p_entity->position.y, + p_entity->size.x, + p_entity->size.y, + }; + Canvas_Theme_Draw_Shadow( + p_theme, + bounds, + 0.06f, + 14, + zoom, + 1.0f); + DrawRectangleRounded(bounds, 0.06f, 14, p_theme->surface); + DrawRectangleRoundedLinesEx( + bounds, + 0.06f, + 14, + 1.0f / zoom, + p_theme->border); + DrawRectangleRounded( + (Rectangle){bounds.x, bounds.y, bounds.width, 54.0f}, + 0.06f, + 14, + p_theme->surface_muted); + Canvas_Lucide_Draw_Icon( + Canvas_Lucide_Find_Icon( + p_entity->agent_working ? "sparkles" : "messages-square"), + (Vector2){bounds.x + 16.0f, bounds.y + 15.0f}, + 0.82f, + 1.8f, + p_entity->agent_working ? + p_theme->accent : + p_theme->text); + DrawTextEx( + font, + p_entity->label, + (Vector2){bounds.x + 48.0f, bounds.y + 12.0f}, + 17.0f, + 0.0f, + p_theme->text); + DrawTextEx( + font, + p_entity->parked + ? "Parked session" + : (p_entity->agent_working + ? "Thinking..." + : "Agent session"), + (Vector2){bounds.x + 48.0f, bounds.y + 33.0f}, + 11.0f, + 0.0f, + p_theme->text_muted); + Rectangle collapse = + Canvas_Conversation_Collapse_Bounds(p_entity); + Rectangle park = Canvas_Conversation_Park_Bounds(p_entity); + Canvas_Lucide_Draw_Icon( + Canvas_Lucide_Find_Icon( + p_entity->active ? "chevron-down" : "chevron-up"), + (Vector2){collapse.x + 5.0f, collapse.y + 5.0f}, + 0.68f, + 1.6f, + p_theme->text_muted); + Canvas_Lucide_Draw_Icon( + Canvas_Lucide_Find_Icon( + p_entity->parked ? "archive-restore" : "archive"), + (Vector2){park.x + 5.0f, park.y + 5.0f}, + 0.68f, + 1.6f, + p_theme->text_muted); + + if (p_entity->size.y <= 82.0f) break; + Rectangle content = { + bounds.x + 16.0f, + bounds.y + 68.0f, + bounds.width - 32.0f, + bounds.height - 82.0f, + }; + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 line_count = Canvas_Text_Build_Lines( + font, + p_entity->text, + content.width, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + for (int32 line_index = 0; + line_index < line_count; + line_index++) { + float y = content.y + (float)line_index * 18.0f - + (float)p_entity->value; + if (y + 18.0f < content.y || + y > content.y + content.height) { + continue; + } + char line_text[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 length = + lines[line_index].end - lines[line_index].start; + memcpy( + line_text, + p_entity->text + lines[line_index].start, + (size_t)length); + line_text[length] = '\0'; + boolean speaker = + strcmp(line_text, "You") == 0 || + strcmp(line_text, "Copilot") == 0 || + strcmp(line_text, "System") == 0; + DrawTextEx( + font, + line_text, + (Vector2){content.x, y}, + 14.0f, + 0.0f, + speaker ? p_theme->accent : p_theme->text); + } + break; + } case CANVAS_ENTITY_LUCIDE_GALLERY: { Rectangle bounds = { p_entity->position.x, @@ -4046,6 +4538,7 @@ case CANVAS_ENTITY_SCROLL_AREA: return "Scrollable area"; case CANVAS_ENTITY_IMAGE: return "Image"; case CANVAS_ENTITY_WEB_CONTENT: return "Web content"; + case CANVAS_ENTITY_CONVERSATION: return "Agent conversation"; case CANVAS_ENTITY_LUCIDE_GALLERY: return "Lucide gallery"; default: return "Unknown"; }
--- a/infinite_canvas/canvas.h Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/canvas.h Tue Aug 18 19:14:53 2026 -0700 @@ -31,6 +31,7 @@ CANVAS_ENTITY_SCROLL_AREA, CANVAS_ENTITY_IMAGE, CANVAS_ENTITY_WEB_CONTENT, + CANVAS_ENTITY_CONVERSATION, CANVAS_ENTITY_LUCIDE_GALLERY, CANVAS_ENTITY_TYPE_COUNT, } Canvas_Entity_Type; @@ -41,6 +42,7 @@ Vector2 position; Vector2 size; Color color; + char label[128]; char text[CANVAS_ENTITY_TEXT_CAPACITY]; int32 value; int32 text_cursor; @@ -50,12 +52,16 @@ boolean active; boolean pinned; boolean removing; + boolean agent_working; + boolean text_muted; + boolean parked; boolean screen_space_initialized; float hover_amount; float animation_amount; float web_chrome_amount; Vector2 pinned_screen_position; Vector2 pinned_screen_size; + Vector2 parked_restore_position; } Canvas_Entity; typedef struct { @@ -133,6 +139,37 @@ void Canvas_Scene_Init(Canvas_Scene *p_scene, Dowa_Arena *p_arena); boolean Canvas_Scene_Add(Canvas_Scene *p_scene, Canvas_Entity_Type type, Vector2 position); +void Canvas_Text_Area_Set_Content( + Canvas_Entity *p_entity, + Font font, + const char *p_text, + boolean muted); +uint32 Canvas_Scene_Add_Conversation( + Canvas_Scene *p_scene, + const char *p_title, + const char *p_prompt, + const char *p_response, + Vector2 position); +boolean Canvas_Scene_Append_Conversation( + Canvas_Scene *p_scene, + uint32 entity_id, + const char *p_prompt, + const char *p_response); +void Canvas_Conversation_Scroll_To_End( + Canvas_Entity *p_entity, + Font font); +void Canvas_Conversation_Set_Collapsed( + Canvas_Entity *p_entity, + boolean collapsed); +size_t Canvas_Scene_Parked_Conversation_Count( + const Canvas_Scene *p_scene); +boolean Canvas_Scene_Set_Conversation_Parked( + Canvas_Scene *p_scene, + uint32 entity_id, + boolean parked); +boolean Canvas_Scene_Show_Parking_Lot( + Canvas_Scene *p_scene, + Canvas_Camera *p_camera); void Canvas_Scene_Clear(Canvas_Scene *p_scene); void Canvas_Scene_Fade_Out_All(Canvas_Scene *p_scene); void Canvas_Scene_Fade_Out_Selected(Canvas_Scene *p_scene);
--- a/infinite_canvas/canvas_test.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/canvas_test.c Tue Aug 18 19:14:53 2026 -0700 @@ -526,6 +526,133 @@ Dowa_Arena_Free(p_arena); } +static void Test_Dictation_Scratchpad_Is_Not_A_Routing_Target(void) +{ + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + + Canvas_Camera camera; + Canvas_Camera_Init(&camera, 800, 600); + Canvas_Scene scene; + Canvas_Scene_Init(&scene, p_arena); + assert(Canvas_Scene_Add( + &scene, + CANVAS_ENTITY_TEXT_AREA, + (Vector2){-260.0f, -120.0f})); + snprintf( + scene.p_entities[0].label, + sizeof(scene.p_entities[0].label), + "Dictation"); + snprintf( + scene.p_entities[0].text, + sizeof(scene.p_entities[0].text), + "Route this thought"); + uint32 conversation_id = Canvas_Scene_Add_Conversation( + &scene, + "Visible session", + "Earlier thought", + "Earlier response", + (Vector2){40.0f, -120.0f}); + assert(conversation_id != 0); + + char context[4096]; + Canvas_Context_Snapshot snapshot = Canvas_Scene_Build_Visible_Context( + &scene, + &camera, + context, + sizeof(context)); + assert(snapshot.visible_count == 1); + assert(!strstr(context, "Route this thought")); + assert(strstr(context, "Visible session")); + assert(strstr(context, TextFormat("id=%u", conversation_id))); + + Dowa_Arena_Free(p_arena); +} + +static void Test_Conversation_Create_And_Append(void) +{ + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + Canvas_Scene scene; + Canvas_Scene_Init(&scene, p_arena); + + uint32 id = Canvas_Scene_Add_Conversation( + &scene, + "Rendering architecture", + "How should images render?", + "Use a rich entity with typed resources.", + (Vector2){10.0f, 20.0f}); + assert(id != 0); + assert(Dowa_Array_Length(scene.p_entities) == 1); + assert(scene.p_entities[0].type == CANVAS_ENTITY_CONVERSATION); + assert(strcmp(scene.p_entities[0].label, "Rendering architecture") == 0); + assert(strstr(scene.p_entities[0].text, "How should images render?")); + + assert(Canvas_Scene_Append_Conversation( + &scene, + id, + "What about browser content?", + "Keep the resource typed and composited by its backend.")); + assert(strstr(scene.p_entities[0].text, "What about browser content?")); + assert(strstr(scene.p_entities[0].text, "composited by its backend")); + assert(!Canvas_Scene_Append_Conversation( + &scene, + id + 1, + "Missing", + "Missing")); + Dowa_Arena_Free(p_arena); +} + +static void Test_Conversation_Collapse_And_Parking(void) +{ + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + Canvas_Camera camera; + Canvas_Camera_Init(&camera, 800, 600); + Canvas_Scene scene; + Canvas_Scene_Init(&scene, p_arena); + + Vector2 original = {20.0f, 30.0f}; + uint32 id = Canvas_Scene_Add_Conversation( + &scene, + "Parkable session", + "Question", + "Answer", + original); + Canvas_Entity *p_conversation = &scene.p_entities[0]; + assert(Near(p_conversation->animation_amount, 1.0f)); + Canvas_Conversation_Set_Collapsed(p_conversation, TRUE); + assert(p_conversation->active); + Canvas_Conversation_Set_Collapsed(p_conversation, FALSE); + assert(!p_conversation->active); + + assert(Canvas_Scene_Set_Conversation_Parked(&scene, id, TRUE)); + assert(p_conversation->parked); + assert(Canvas_Scene_Parked_Conversation_Count(&scene) == 1); + assert(!Near(p_conversation->position.x, original.x)); + char context[4096]; + Canvas_Context_Snapshot snapshot = Canvas_Scene_Build_Visible_Context( + &scene, + &camera, + context, + sizeof(context)); + assert(snapshot.visible_count == 0); + assert(Canvas_Scene_Show_Parking_Lot(&scene, &camera)); + snapshot = Canvas_Scene_Build_Visible_Context( + &scene, + &camera, + context, + sizeof(context)); + assert(snapshot.visible_count == 0); + + assert(Canvas_Scene_Set_Conversation_Parked(&scene, id, FALSE)); + assert(!p_conversation->parked); + assert(Canvas_Scene_Parked_Conversation_Count(&scene) == 0); + assert(Near(p_conversation->position.x, original.x)); + assert(Near(p_conversation->position.y, original.y)); + Dowa_Arena_Free(p_arena); +} + int main(void) { Test_Theme_Resolution(); @@ -543,8 +670,11 @@ Test_Lucide_Gallery_Reuses_Entity_And_Fits_Camera(); Test_Lucide_Search_Is_Case_Insensitive(); Test_Demo_Contains_Every_Entity_Type(); + Test_Conversation_Create_And_Append(); Test_Pinned_Entity_Stays_In_Screen_Space(); Test_Visible_Context_Tracks_Camera_And_Values(); + Test_Dictation_Scratchpad_Is_Not_A_Routing_Target(); + Test_Conversation_Collapse_And_Parking(); printf("Infinite canvas tests passed.\n"); return 0; }
--- a/infinite_canvas/dev_ui.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/dev_ui.c Tue Aug 18 19:14:53 2026 -0700 @@ -5,7 +5,8 @@ #define RAYGUI_IMPLEMENTATION #include "third_party/raylib/include/raygui.h" -static const Rectangle DEV_PANEL = {18.0f, 18.0f, 264.0f, 478.0f}; +static const Rectangle DEV_PANEL = {18.0f, 18.0f, 264.0f, 520.0f}; +static const Rectangle DEV_PANEL_COLLAPSED = {18.0f, 18.0f, 44.0f, 44.0f}; static Rectangle Canvas_Dev_UI_Context_Bounds(void) { @@ -206,6 +207,7 @@ { *p_ui = (Canvas_Dev_UI){ .selected_type = CANVAS_ENTITY_RECTANGLE, + .collapsed = FALSE, .dropdown_open = FALSE, .context_open = FALSE, }; @@ -219,7 +221,11 @@ if (p_ui->dropdown_open) return TRUE; Vector2 pointer = GetMousePosition(); - if (CheckCollisionPointRec(pointer, DEV_PANEL)) return TRUE; + if (CheckCollisionPointRec( + pointer, + p_ui->collapsed ? DEV_PANEL_COLLAPSED : DEV_PANEL)) { + return TRUE; + } if (p_ui->context_open && CheckCollisionPointRec(pointer, Canvas_Dev_UI_Context_Bounds())) { return TRUE; @@ -234,6 +240,35 @@ Font font, Canvas_Theme *p_theme) { + if (p_ui->collapsed) { + Canvas_Theme_Draw_Shadow( + p_theme, + DEV_PANEL_COLLAPSED, + 0.24f, + 12, + 1.0f, + 1.0f); + DrawRectangleRounded( + DEV_PANEL_COLLAPSED, + 0.24f, + 12, + p_theme->surface); + DrawRectangleRoundedLinesEx( + DEV_PANEL_COLLAPSED, + 0.24f, + 12, + 1.0f, + p_theme->border); + if (GuiButtonRounded( + (Rectangle){24.0f, 24.0f, 32.0f, 32.0f}, + ">", + 0.30f, + 10)) { + p_ui->collapsed = FALSE; + } + return; + } + Canvas_Context_Snapshot context = Canvas_Scene_Build_Visible_Context( p_scene, p_camera, @@ -256,6 +291,16 @@ 12.0f, 0.1f, p_theme->text_muted); + if (GuiButtonRounded( + (Rectangle){230.0f, 30.0f, 36.0f, 30.0f}, + "<", + 0.30f, + 10)) { + p_ui->collapsed = TRUE; + p_ui->dropdown_open = FALSE; + p_ui->context_open = FALSE; + return; + } if (p_ui->dropdown_open) GuiLock(); @@ -322,19 +367,28 @@ Canvas_Scene_Show_Lucide_Gallery(p_scene, p_camera); } - DrawTextEx(font, TextFormat("%d FPS", GetFPS()), (Vector2){34.0f, 396.0f}, 12.0f, 0.0f, p_theme->text_muted); - DrawTextEx(font, TextFormat("%.2fx zoom", p_camera->zoom), (Vector2){104.0f, 396.0f}, 12.0f, 0.0f, p_theme->text_muted); + size_t parked_count = Canvas_Scene_Parked_Conversation_Count(p_scene); + if (GuiButtonRounded( + (Rectangle){34.0f, 378.0f, 232.0f, 34.0f}, + TextFormat("Parking lot (%d)", (int)parked_count), + 0.32f, + 10)) { + Canvas_Scene_Show_Parking_Lot(p_scene, p_camera); + } + + DrawTextEx(font, TextFormat("%d FPS", GetFPS()), (Vector2){34.0f, 438.0f}, 12.0f, 0.0f, p_theme->text_muted); + DrawTextEx(font, TextFormat("%.2fx zoom", p_camera->zoom), (Vector2){104.0f, 438.0f}, 12.0f, 0.0f, p_theme->text_muted); DrawTextEx( font, TextFormat("%d objects", (int)Dowa_Array_Length(p_scene->p_entities)), - (Vector2){196.0f, 396.0f}, + (Vector2){196.0f, 438.0f}, 12.0f, 0.0f, p_theme->text_muted); DrawTextEx( font, "Drag objects directly on the canvas", - (Vector2){34.0f, 442.0f}, + (Vector2){34.0f, 484.0f}, 12.0f, 0.0f, p_theme->text_muted);
--- a/infinite_canvas/dev_ui.h Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/dev_ui.h Tue Aug 18 19:14:53 2026 -0700 @@ -5,6 +5,7 @@ typedef struct { int32 selected_type; + boolean collapsed; boolean dropdown_open; boolean context_open; float context_scroll;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/dictation_policy_test.sh Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +set -euo pipefail + +native_source="$1" +main_source="$2" +dev_ui_source="$3" + +require_pattern() { + local pattern="$1" + local file="$2" + local message="$3" + if ! grep -Eq "$pattern" "$file"; then + echo "$message" >&2 + exit 1 + fi +} + +require_pattern \ + 'CefRequestContext::CreateContext' \ + "$native_source" \ + "Dictation must use an isolated CEF request context." +require_pattern \ + 'view_->is_dictation_transport' \ + "$native_source" \ + "Microphone permission must be scoped to the hidden dictation transport." +require_pattern \ + 'http://127\.0\.0\.1:8090' \ + "$native_source" \ + "Microphone permission must be scoped to the loopback dictation origin." +require_pattern \ + 'OnShowPermissionPrompt' \ + "$native_source" \ + "The hidden Chrome-style permission prompt must be handled natively." +require_pattern \ + 'CEF_PERMISSION_TYPE_MIC_STREAM' \ + "$native_source" \ + "The native prompt handler must identify microphone requests." +require_pattern \ + 'CEF_PERMISSION_RESULT_ACCEPT' \ + "$native_source" \ + "The native prompt handler must accept loopback microphone access." +require_pattern \ + 'requests_other_media' \ + "$native_source" \ + "The native permission handler must reject non-audio media requests." +require_pattern \ + 'callback->Continue\(audio\)' \ + "$native_source" \ + "The native permission handler must grant loopback audio directly." + +if grep -Eq 'AppendSwitch\("(enable-media-stream|use-fake-ui-for-media-stream)"' \ + "$native_source"; then + echo "Global media auto-grant switches are forbidden." >&2 + exit 1 +fi + +require_pattern \ + 'Canvas_App_Ensure_Dictation_Entity' \ + "$main_source" \ + "Dictation must create a retained canvas entity." +require_pattern \ + 'CANVAS_ENTITY_TEXT_AREA' \ + "$main_source" \ + "Dictation must use the wrapping text-area entity." +require_pattern \ + 'Canvas_Text_Area_Set_Content' \ + "$main_source" \ + "Live transcripts must update the retained text area." +require_pattern \ + '"Dictation"' \ + "$main_source" \ + "The live transcript entity must expose a draggable Dictation header." +submit_source="$( + sed -n \ + '/static boolean Canvas_App_Submit_Dictation(void)/,/^}/p' \ + "$main_source" +)" +if grep -Eq 'dictation_entity_id = 0' <<<"$submit_source" || + ! grep -Eq 'Canvas_App_Sync_Dictation_Entity' <<<"$submit_source"; then + echo "Submission must clear and reuse the one Dictation scratchpad." >&2 + exit 1 +fi +if grep -Eq 'Set_Dictation_Active' <<<"$submit_source"; then + echo "Sending a thought must keep live microphone capture active." >&2 + exit 1 +fi +if grep -Eq 'Canvas_App_Draw_Dictation_Overlay' "$main_source"; then + echo "Dictation must not also render a screen-space popup." >&2 + exit 1 +fi + +indicator_source="$( + sed -n \ + '/static void Canvas_App_Draw_Listening_Indicator(void)/,/^}/p' \ + "$main_source" +)" +for pattern in \ + 'if \(!g_app->dictation_listening\) return' \ + 'GetScreenHeight\(\) - 28\.0f' \ + 'g_app->theme\.danger' \ + 'g_app->theme\.text_muted' \ + '"Listening"'; do + if ! grep -Eq "$pattern" <<<"$indicator_source"; then + echo "Dictation must show a theme-aware bottom-left listening indicator." >&2 + exit 1 + fi +done + +input_source="$( + sed -n \ + '/static void Canvas_App_Update_Dictation_Overlay_Input(void)/,/^}/p' \ + "$main_source" +)" +if ! grep -Eq 'IsKeyPressed\(KEY_ENTER\)' <<<"$input_source"; then + echo "Dictation overlay must submit from Enter." >&2 + exit 1 +fi +if ! grep -Eq 'Canvas_Web_Surface_Commit_Dictation' <<<"$input_source"; then + echo "Enter must commit the utterance without stopping the microphone." >&2 + exit 1 +fi +if ! grep -Eq 'dictation_listening.*IsKeyPressed\(KEY_BACKSPACE\)' \ + <<<"$input_source"; then + echo "Backspace must clear active dictation without stopping capture." >&2 + exit 1 +fi +entity_source="$( + sed -n \ + '/static void Canvas_App_Sync_Dictation_Entity(void)/,/^}/p' \ + "$main_source" +)" +if ! grep -Eq "dictation_partial\\[0\\]" <<<"$entity_source" || + ! grep -Eq 'TRUE : FALSE' <<<"$entity_source"; then + echo "In-progress dictation text must use the muted theme color." >&2 + exit 1 +fi + +require_pattern \ + 'EnsureDictationView\(p_surface\);' \ + "$native_source" \ + "Native dictation transport must be prewarmed before the M hotkey." +require_pattern \ + 'Canvas_Web_Surface_Set_Dictation_Active' \ + "$main_source" \ + "Dictation must use deterministic start and stop commands." + +active_source="$( + sed -n \ + '/static void Canvas_App_Set_Dictation_Active(boolean active)/,/^}/p' \ + "$main_source" +)" +if grep -Eq "dictation_(partial|transcript)\\[0\\] = '\\\\0'" \ + <<<"$active_source"; then + echo "Resuming dictation must preserve the existing scratchpad text." >&2 + exit 1 +fi + +require_pattern \ + 'p_ui->collapsed' \ + "$dev_ui_source" \ + "Developer controls must expose a persistent collapsed state." +require_pattern \ + 'DEV_PANEL_COLLAPSED' \ + "$dev_ui_source" \ + "Collapsed developer controls must retain a visible restore button." + +final_source="$( + sed -n \ + '/CANVAS_DICTATION_EVENT_FINAL/,/if (!g_app->dictation_overlay_visible/p' \ + "$main_source" +)" +if ! grep -Eq 'dictation_send_pending' <<<"$final_source"; then + echo "Final dictation must wait for explicit Enter submission." >&2 + exit 1 +fi
--- a/infinite_canvas/docs/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/docs/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -17,6 +17,10 @@ │ ├── camera transforms │ ├── pinned entities │ └── notification stacking +├── Spatial agent sessions +│ ├── WebRTC dictation bridge +│ ├── asynchronous Copilot SDK orchestration +│ └── retained conversation entities └── Session lineage ├── original renderer └── scalability, images, pinning, context, and icons @@ -25,6 +29,7 @@ - [Rendering surfaces](rendering.md) - [Entities and components](entities-and-components.md) - [Pinning and visible context](pinning-and-context.md) +- [Dictation and spatial agent sessions](agent-sessions.md) - [Session lineage](session-lineage.md) ## Runtime frame order @@ -32,14 +37,14 @@ `main.c` owns the integration order: 1. Update viewport dimensions. -2. Let native web surfaces translate focused input. -3. Update scene interaction and keyboard focus. -4. Apply camera pan/zoom. -5. Reproject pinned entities and notification stacks. -6. Reconcile web/image resources with visible entities. -7. Draw the Raylib world. -8. Composite native browser textures or position web overlays. -9. Draw the developer UI. +2. Handle dictation and selected browser chrome. +3. Let native web surfaces translate focused input. +4. Update scene interaction and keyboard focus. +5. Submit or poll asynchronous agent work. +6. Apply camera pan/zoom and reproject pinned entities. +7. Reconcile web/image resources with visible entities. +8. Draw each Raylib entity and its owning native/DOM surface in scene order. +9. Draw world overlays and the developer UI. Preserve this order. In particular, pinned positions must be synchronized before surface bounds are calculated, and native surface textures must draw after the
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/docs/agent-sessions.md Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,84 @@ +# Dictation and Spatial Agent Sessions + +## Prototype flow + +The first orchestration slice connects three retained systems: + +1. `M` reuses the single retained `Dictation` text-area scratchpad and toggles + a hidden browser transport for `http://127.0.0.1:8090/?canvas=1`. +2. `Canvas_Web_Surface_Set_Dictation_Active()` deterministically starts or + stops capture on an already-negotiated WebRTC client. The hidden transport + acquires its stream, completes signaling/ICE, and opens the transcript data + channel during startup with its audio track disabled. + CEF grants microphone access only to the loopback dictation origin. +3. Status, partial, and final transcript events cross the platform boundary: + - native CEF encodes each event in the document title and + `OnTitleChange()` queues it for the caption; + - WebAssembly uses `window.postMessage()` from a hidden iframe. +4. `main.c` accumulates speech in the scratchpad and submits it to + `Canvas_Agent_Service` only when the user presses `Enter`. A selected text + area can enter the same path with `Ctrl/Cmd+Enter`. + Enter sends a `commit` command through the live transcript data channel, so + the current utterance finalizes without closing the microphone session. +5. `agent_service_copilot.c` sends the new thought plus + `Canvas_Scene_Build_Visible_Context()` through the shared asynchronous + `Inference_Bridge`. +6. The Bazel-managed Copilot SDK sidecar uses the `canvas_orchestrator` profile + and returns JSON choosing `create` or `append` and a conversation entity ID. + The canvas materializes that decision as a retained rich conversation card. + +The scratchpad is excluded from serialized camera context because the submitted +thought is already sent separately. Only conversation entities currently in +view are valid append targets; moving a conversation out of view removes it +from the orchestrator's routing choices. + +## Service boundary + +`agent_service_copilot.c` submits work to `Inference_Bridge`, whose worker +thread owns the JSON-line Python sidecar process so SDK latency never blocks +Raylib. The sidecar maintains the persistent +`infinite-canvas-orchestrator` SDK session and uses the loopback LiteLLM +gateway for authenticated GitHub Copilot inference. Only one orchestration +request is admitted at a time in this prototype. + +`agent_service_copilot.c` warms that orchestration session plus three reserved +worker sessions during initialization. They share the same +`canvas_orchestrator` profile and move Copilot SDK client/session setup ahead of +the first user turn. The persistent orchestrator is resumed when available; +reserved workers are created fresh because they have no conversation history +to recover, avoiding expected `session.resume` errors during startup. + +## Conversation entities + +`CANVAS_ENTITY_CONVERSATION` is intentionally more than an editable text area. +It retains: + +- a stable entity/session ID used by Copilot routing; +- a title; +- user and agent transcript content; +- asynchronous working state; +- independent scroll, selection, lifecycle, z-order, and pinning state. +- auto-follow to the latest appended turn and a collapsed header-only state; +- a recoverable parked state that is excluded from orchestration context. + +Images, browser views, and other components already appear in visible canvas +context as typed neighboring entities. A future resource list can attach those +entity IDs directly to a conversation without replacing the conversation card +or flattening rich content into one string. + +Archiving a conversation records its prior position and moves it to the +off-canvas parking lot. The Developer Controls parking-lot button fits the +camera to parked sessions. Restoring a parked card returns it to its prior +position and makes it eligible for routing again when visible. + +## Current constraints + +- `//infinite_canvas:orchestration_dev` owns the dictation server, LiteLLM + gateway, and canvas lifecycle. `//infinite_canvas:agent_dev` is an alias. +- One persistent Dictation scratchpad is reused and cleared after submission. + Submitted turns are retained in conversation entities after orchestration. +- One Copilot request runs at a time. +- Conversation transcript storage is currently bounded by + `CANVAS_ENTITY_TEXT_CAPACITY`. +- Linux uses the Copilot SDK bridge. Other platforms currently select an + explicit unavailable stub.
--- a/infinite_canvas/docs/entities-and-components.md Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/docs/entities-and-components.md Tue Aug 18 19:14:53 2026 -0700 @@ -7,6 +7,8 @@ - a stable `id` for external resources such as browser views; - `type`, world `position`, and world `size`; - generic semantic fields: `text`, `value`, and `active`; +- optional rich-context fields such as a conversation title and agent working + state; - interaction/animation state; - pinning state in both world and screen coordinates. @@ -19,6 +21,13 @@ them back to zero, then compact the retained array. Raylib colors, native CEF texture tints, and WebAssembly overlay opacity all consume the same eased value. +Treat animation as part of every entity state transition, not optional polish. +State changes set retained targets; frame updates ease visual geometry and +opacity toward those targets. Do not snap expandable, collapsible, selected, +hovered, created, removed, or otherwise stateful entities between layouts. +Conversation collapse/expand uses the same smoothstep height interpolation as +accordions, while preserving its final semantic `active` state for context. + ## Adding a component Wire every relevant surface, not only drawing: @@ -47,6 +56,7 @@ - developer UI; - focused native CEF content; +- the canvas URL editor or local dictation bridge; - hovered/pressed scene entity; - camera.
--- a/infinite_canvas/main.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/main.c Tue Aug 18 19:14:53 2026 -0700 @@ -1,3 +1,4 @@ +#include "infinite_canvas/agent_service.h" #include "infinite_canvas/canvas.h" #include "infinite_canvas/web_surface.h" @@ -9,6 +10,7 @@ #include <emscripten/emscripten.h> #endif +#include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -18,11 +20,25 @@ Canvas_Camera camera; Canvas_Scene scene; Canvas_Web_Surface web_surface; + Canvas_Agent_Service agent_service; Font font; Canvas_Theme theme; boolean owns_font; const char *p_screenshot_path; uint32 frame_count; + uint32 screenshot_frame; + uint32 pending_conversation_id; + uint32 dictation_entity_id; + boolean agent_initialized; + boolean dictation_overlay_visible; + boolean dictation_listening; + boolean dictation_requested_active; + boolean dictation_send_pending; + char dictation_status[128]; + char dictation_partial[CANVAS_ENTITY_TEXT_CAPACITY]; + char dictation_transcript[CANVAS_ENTITY_TEXT_CAPACITY]; + char pending_prompt[CANVAS_ENTITY_TEXT_CAPACITY]; + char agent_context[CANVAS_CONTEXT_MAX_LENGTH]; #if defined(INFINITE_CANVAS_DEV_UI) Canvas_Dev_UI dev_ui; #endif @@ -30,6 +46,381 @@ static Canvas_App *g_app = NULL; +static const char *Canvas_App_JSON_String( + const char *p_json, + const char *p_key, + char *p_output, + size_t output_capacity) +{ + char needle[128]; + snprintf(needle, sizeof(needle), "\"%s\"", p_key); + const char *p_cursor = strstr(p_json, needle); + if (!p_cursor) return NULL; + p_cursor = strchr(p_cursor + strlen(needle), ':'); + if (!p_cursor) return NULL; + while (*++p_cursor == ' ') {} + if (*p_cursor != '"') return NULL; + p_cursor++; + size_t output = 0; + while (*p_cursor && *p_cursor != '"' && output + 1 < output_capacity) { + if (*p_cursor == '\\' && p_cursor[1]) { + p_cursor++; + if (*p_cursor == 'n') p_output[output++] = '\n'; + else if (*p_cursor == 't') p_output[output++] = '\t'; + else if (*p_cursor != 'r') p_output[output++] = *p_cursor; + } else { + p_output[output++] = *p_cursor; + } + p_cursor++; + } + p_output[output] = '\0'; + return p_output; +} + +static uint32 Canvas_App_JSON_Uint(const char *p_json, const char *p_key) +{ + char needle[128]; + snprintf(needle, sizeof(needle), "\"%s\"", p_key); + const char *p_cursor = strstr(p_json, needle); + if (!p_cursor) return 0; + p_cursor = strchr(p_cursor + strlen(needle), ':'); + if (!p_cursor) return 0; + return (uint32)strtoul(p_cursor + 1, NULL, 10); +} + +static Canvas_Entity *Canvas_App_Find_Entity(uint32 entity_id) +{ + for (size_t index = 0; + index < Dowa_Array_Length(g_app->scene.p_entities); + index++) { + if (g_app->scene.p_entities[index].id == entity_id) { + return &g_app->scene.p_entities[index]; + } + } + return NULL; +} + +static void Canvas_App_Set_Conversation_Text( + Canvas_Entity *p_entity, + const char *p_prompt, + const char *p_speaker, + const char *p_response) +{ + size_t length = 0; + int32 written = snprintf( + p_entity->text, + sizeof(p_entity->text), + "You\n"); + if (written > 0) length = (size_t)written; + written = snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "%.*s\n\n%s\n", + 900, + p_prompt, + p_speaker); + if (written > 0) length += (size_t)written; + if (length >= sizeof(p_entity->text)) { + length = sizeof(p_entity->text) - 1; + } + snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "%.*s", + 900, + p_response); + Canvas_Conversation_Scroll_To_End(p_entity, g_app->font); +} + +static boolean Canvas_App_Submit_Prompt(const char *p_prompt) +{ + if (!p_prompt || !p_prompt[0]) return FALSE; + Canvas_Scene_Build_Visible_Context( + &g_app->scene, + &g_app->camera, + g_app->agent_context, + sizeof(g_app->agent_context)); + if (!Canvas_Agent_Service_Submit( + &g_app->agent_service, + p_prompt, + g_app->agent_context)) { + return FALSE; + } + snprintf( + g_app->pending_prompt, + sizeof(g_app->pending_prompt), + "%s", + p_prompt); + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + g_app->pending_conversation_id = Canvas_Scene_Add_Conversation( + &g_app->scene, + "Routing thought...", + p_prompt, + "Copilot is deciding whether to create or extend a session.", + position); + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (p_pending) p_pending->agent_working = TRUE; + return TRUE; +} + +static void Canvas_App_Apply_Agent_Response(const char *p_json) +{ + char action[32] = {0}; + char title[128] = {0}; + char response[CANVAS_ENTITY_TEXT_CAPACITY] = {0}; + Canvas_App_JSON_String(p_json, "action", action, sizeof(action)); + Canvas_App_JSON_String(p_json, "title", title, sizeof(title)); + if (!Canvas_App_JSON_String( + p_json, + "response", + response, + sizeof(response))) { + snprintf(response, sizeof(response), "%s", p_json); + } + uint32 conversation_id = + Canvas_App_JSON_Uint(p_json, "conversation_id"); + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (strcmp(action, "append") == 0 && + conversation_id != 0 && + Canvas_Scene_Append_Conversation( + &g_app->scene, + conversation_id, + g_app->pending_prompt, + response)) { + Canvas_Entity *p_conversation = + Canvas_App_Find_Entity(conversation_id); + Canvas_Conversation_Scroll_To_End( + p_conversation, + g_app->font); + if (p_pending) p_pending->removing = TRUE; + } else if (p_pending) { + snprintf( + p_pending->label, + sizeof(p_pending->label), + "%s", + title[0] ? title : "Copilot session"); + Canvas_App_Set_Conversation_Text( + p_pending, + g_app->pending_prompt, + "Copilot", + response); + p_pending->agent_working = FALSE; + } else { + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + uint32 created_id = Canvas_Scene_Add_Conversation( + &g_app->scene, + title[0] ? title : "Copilot session", + g_app->pending_prompt, + response, + position); + Canvas_Conversation_Scroll_To_End( + Canvas_App_Find_Entity(created_id), + g_app->font); + } + g_app->pending_conversation_id = 0; + g_app->pending_prompt[0] = '\0'; +} + +static Canvas_Entity *Canvas_App_Ensure_Dictation_Entity(void) +{ + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (p_entity) return p_entity; + + Vector2 center = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f, + (float)g_app->camera.viewport_height * 0.5f, + }); + Vector2 size = {520.0f, 240.0f}; + if (!Canvas_Scene_Add( + &g_app->scene, + CANVAS_ENTITY_TEXT_AREA, + (Vector2){ + center.x - size.x * 0.5f, + center.y - size.y * 0.5f, + })) { + return NULL; + } + size_t index = Dowa_Array_Length(g_app->scene.p_entities) - 1; + p_entity = &g_app->scene.p_entities[index]; + p_entity->size = size; + snprintf(p_entity->label, sizeof(p_entity->label), "Dictation"); + g_app->dictation_entity_id = p_entity->id; + g_app->scene.selected_index = (int32)index; + return p_entity; +} + +static void Canvas_App_Sync_Dictation_Entity(void) +{ + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (!p_entity) return; + + char text[CANVAS_ENTITY_TEXT_CAPACITY]; + snprintf(text, sizeof(text), "%s", g_app->dictation_transcript); + size_t length = strlen(text); + if (length > 0 && g_app->dictation_partial[0] && + length + 1 < sizeof(text)) { + text[length++] = ' '; + text[length] = '\0'; + } + snprintf( + text + length, + sizeof(text) - length, + "%s", + g_app->dictation_partial); + Canvas_Text_Area_Set_Content( + p_entity, + g_app->font, + text, + g_app->dictation_partial[0] ? TRUE : FALSE); +} + +static void Canvas_App_Set_Dictation_Active(boolean active) +{ + g_app->dictation_overlay_visible = TRUE; + g_app->dictation_requested_active = active; + g_app->dictation_listening = FALSE; + g_app->dictation_send_pending = FALSE; + if (active) { + Canvas_App_Ensure_Dictation_Entity(); + Canvas_App_Sync_Dictation_Entity(); + } + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "%s", + active ? "Starting microphone..." : "Stopping microphone..."); + Canvas_Web_Surface_Set_Dictation_Active( + &g_app->web_surface, + active); +} + +static boolean Canvas_App_Submit_Dictation(void) +{ + if (!g_app->dictation_transcript[0]) return FALSE; + if (!Canvas_App_Submit_Prompt(g_app->dictation_transcript)) { + g_app->dictation_send_pending = FALSE; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Copilot is busy; press Enter to retry"); + return FALSE; + } + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (p_entity) p_entity->text_muted = FALSE; + g_app->dictation_transcript[0] = '\0'; + g_app->dictation_partial[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + g_app->dictation_send_pending = FALSE; + g_app->dictation_overlay_visible = + g_app->dictation_listening; + return TRUE; +} + +static void Canvas_App_Append_Dictation(const char *p_text) +{ + if (!p_text || !p_text[0]) return; + size_t length = strlen(g_app->dictation_transcript); + if (length > 0 && length + 1 < sizeof(g_app->dictation_transcript)) { + g_app->dictation_transcript[length++] = ' '; + g_app->dictation_transcript[length] = '\0'; + } + size_t available = sizeof(g_app->dictation_transcript) - length - 1; + size_t text_length = strlen(p_text); + size_t copy_length = text_length < available ? text_length : available; + memcpy(g_app->dictation_transcript + length, p_text, copy_length); + g_app->dictation_transcript[length + copy_length] = '\0'; + Canvas_App_Sync_Dictation_Entity(); +} + +static void Canvas_App_Update_Dictation_Overlay_Input(void) +{ + if (!g_app->dictation_overlay_visible) return; + if (g_app->dictation_listening && IsKeyPressed(KEY_BACKSPACE)) { + g_app->dictation_send_pending = FALSE; + g_app->dictation_partial[0] = '\0'; + g_app->dictation_transcript[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Listening"); + return; + } + if (IsKeyPressed(KEY_ESCAPE)) { + if (g_app->dictation_listening) { + Canvas_Web_Surface_Set_Dictation_Active( + &g_app->web_surface, + FALSE); + g_app->dictation_requested_active = FALSE; + g_app->dictation_listening = FALSE; + } + g_app->dictation_send_pending = FALSE; + g_app->dictation_partial[0] = '\0'; + g_app->dictation_transcript[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + g_app->dictation_overlay_visible = FALSE; + return; + } + if (!IsKeyPressed(KEY_ENTER)) return; + + if (g_app->dictation_partial[0]) { + g_app->dictation_send_pending = TRUE; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Finishing dictation..."); + Canvas_Web_Surface_Commit_Dictation( + &g_app->web_surface); + } else if (g_app->dictation_transcript[0]) { + Canvas_App_Submit_Dictation(); + } +} + +static boolean Canvas_App_Dictation_Status_Is_Error(const char *p_status) +{ + return strstr(p_status, "failed") != NULL || + strstr(p_status, "Unable") != NULL || + strstr(p_status, "closed") != NULL || + strstr(p_status, "invalid") != NULL || + strstr(p_status, "error") != NULL; +} + +static void Canvas_App_Draw_Listening_Indicator(void) +{ + if (!g_app->dictation_listening) return; + + float pulse = 0.72f + 0.28f * sinf((float)GetTime() * 5.0f); + Vector2 dot = {28.0f, (float)GetScreenHeight() - 28.0f}; + DrawCircleV( + dot, + 5.0f + pulse * 2.0f, + Fade(g_app->theme.danger, 0.82f)); + DrawTextEx( + g_app->font, + "Listening", + (Vector2){44.0f, dot.y - 8.0f}, + 14.0f, + 0.0f, + g_app->theme.text_muted); +} + static void Canvas_App_Draw_Frame(void) { int32 width = GetScreenWidth(); @@ -43,6 +434,15 @@ Canvas_Web_Surface_Set_Dark_Mode( &g_app->web_surface, g_app->theme.resolved_mode == CANVAS_THEME_DARK ? TRUE : FALSE); + boolean dictation_hotkey = + !Canvas_Scene_Is_Text_Editing(&g_app->scene) && + !Canvas_Web_Chrome_Is_Editing(&g_app->scene) && + IsKeyPressed(KEY_M); + if (dictation_hotkey) { + Canvas_App_Set_Dictation_Active( + g_app->dictation_requested_active ? FALSE : TRUE); + } + Canvas_App_Update_Dictation_Overlay_Input(); Canvas_Web_Chrome_Action chrome_action = CANVAS_WEB_CHROME_NONE; if (!block_pointer_input) { const char *p_current_url = NULL; @@ -76,7 +476,9 @@ } } boolean web_input = FALSE; - if (!block_pointer_input && chrome_action == CANVAS_WEB_CHROME_NONE) { + if (!block_pointer_input && + !dictation_hotkey && + chrome_action == CANVAS_WEB_CHROME_NONE) { web_input = Canvas_Web_Surface_Update_Input( &g_app->web_surface, &g_app->camera, @@ -99,6 +501,7 @@ Canvas_Scene_Toggle_Selected_Pin(&g_app->scene, &g_app->camera); } if (!keyboard_focus && + !g_app->dictation_listening && (IsKeyPressed(KEY_DELETE) || IsKeyPressed(KEY_BACKSPACE))) { Canvas_Scene_Fade_Out_Selected(&g_app->scene); } @@ -109,6 +512,136 @@ Canvas_Scene_Sync_Pinned(&g_app->scene, &g_app->camera); Canvas_Scene_Update_Notification_Stack(&g_app->scene, &g_app->camera); + char dictation_text[CANVAS_ENTITY_TEXT_CAPACITY]; + Canvas_Dictation_Event dictation_event = + Canvas_Web_Surface_Consume_Dictation( + &g_app->web_surface, + dictation_text, + sizeof(dictation_text)); + if (dictation_event == CANVAS_DICTATION_EVENT_STATUS) { + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "%.127s", + dictation_text); + if (strcmp(dictation_text, "Idle") == 0) { + g_app->dictation_listening = FALSE; + if (g_app->dictation_send_pending) { + if (g_app->dictation_partial[0]) { + Canvas_App_Append_Dictation( + g_app->dictation_partial); + g_app->dictation_partial[0] = '\0'; + } + Canvas_App_Submit_Dictation(); + } else if (!g_app->dictation_transcript[0] && + !g_app->dictation_partial[0]) { + g_app->dictation_overlay_visible = FALSE; + } + } else if (strcmp(dictation_text, "Listening") == 0 || + strcmp(dictation_text, "Speech detected") == 0) { + g_app->dictation_listening = + g_app->dictation_requested_active; + if (strcmp(dictation_text, "Listening") == 0 && + g_app->dictation_send_pending) { + if (g_app->dictation_partial[0]) { + Canvas_App_Append_Dictation( + g_app->dictation_partial); + g_app->dictation_partial[0] = '\0'; + } + Canvas_App_Submit_Dictation(); + } + } else if (Canvas_App_Dictation_Status_Is_Error(dictation_text)) { + g_app->dictation_listening = FALSE; + g_app->dictation_overlay_visible = TRUE; + } + } else if (dictation_event == CANVAS_DICTATION_EVENT_PARTIAL) { + g_app->dictation_overlay_visible = TRUE; + snprintf( + g_app->dictation_partial, + sizeof(g_app->dictation_partial), + "%s", + dictation_text); + Canvas_App_Sync_Dictation_Entity(); + } else if (dictation_event == CANVAS_DICTATION_EVENT_FINAL) { + g_app->dictation_overlay_visible = TRUE; + Canvas_App_Append_Dictation(dictation_text); + g_app->dictation_partial[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + if (g_app->dictation_send_pending) { + Canvas_App_Submit_Dictation(); + } else { + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Ready to send"); + } + } + if (!g_app->dictation_overlay_visible && + g_app->scene.selected_index >= 0 && + g_app->scene.selected_index < + (int32)Dowa_Array_Length(g_app->scene.p_entities) && + (IsKeyDown(KEY_LEFT_CONTROL) || + IsKeyDown(KEY_RIGHT_CONTROL) || + IsKeyDown(KEY_LEFT_SUPER) || + IsKeyDown(KEY_RIGHT_SUPER)) && + IsKeyPressed(KEY_ENTER)) { + Canvas_Entity *p_selected = + &g_app->scene.p_entities[g_app->scene.selected_index]; + if (p_selected->type == CANVAS_ENTITY_TEXT_AREA && + p_selected->text[0]) { + size_t length = strlen(p_selected->text); + if (length > 0 && p_selected->text[length - 1] == '\n') { + p_selected->text[length - 1] = '\0'; + } + if (Canvas_App_Submit_Prompt(p_selected->text)) { + p_selected->text[0] = '\0'; + p_selected->text_cursor = 0; + p_selected->text_selection_anchor = 0; + } + } + } + + char agent_response[CANVAS_AGENT_RESPONSE_CAPACITY]; + char agent_error[512]; + Canvas_Agent_Status agent_status = g_app->agent_initialized ? + Canvas_Agent_Service_Poll( + &g_app->agent_service, + agent_response, + sizeof(agent_response), + agent_error, + sizeof(agent_error)) : + CANVAS_AGENT_IDLE; + if (agent_status == CANVAS_AGENT_READY) { + Canvas_App_Apply_Agent_Response(agent_response); + } else if (agent_status == CANVAS_AGENT_ERROR) { + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (p_pending) { + snprintf(p_pending->label, sizeof(p_pending->label), "Copilot unavailable"); + Canvas_App_Set_Conversation_Text( + p_pending, + g_app->pending_prompt, + "System", + agent_error); + p_pending->agent_working = FALSE; + } else if (g_app->pending_prompt[0]) { + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + Canvas_Scene_Add_Conversation( + &g_app->scene, + "Copilot unavailable", + g_app->pending_prompt, + agent_error, + position); + } + g_app->pending_conversation_id = 0; + g_app->pending_prompt[0] = '\0'; + } + Canvas_Web_Surface_Sync(&g_app->web_surface, &g_app->camera, &g_app->scene); BeginDrawing(); @@ -162,6 +695,7 @@ g_app->font, &g_app->theme); #endif + Canvas_App_Draw_Listening_Indicator(); EndDrawing(); g_app->frame_count++; #if !defined(PLATFORM_WEB) @@ -171,7 +705,7 @@ } #endif if (g_app->p_screenshot_path && - g_app->frame_count == 180) { + g_app->frame_count == g_app->screenshot_frame) { TakeScreenshot(g_app->p_screenshot_path); } } @@ -196,7 +730,21 @@ } memset(g_app, 0, sizeof(*g_app)); g_app->p_arena = p_arena; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Idle"); g_app->p_screenshot_path = getenv("INFINITE_CANVAS_SCREENSHOT"); + g_app->screenshot_frame = 180; + const char *p_screenshot_frame = + getenv("INFINITE_CANVAS_SCREENSHOT_FRAME"); + if (p_screenshot_frame) { + uint32 screenshot_frame = + (uint32)strtoul(p_screenshot_frame, NULL, 10); + if (screenshot_frame > 0) { + g_app->screenshot_frame = screenshot_frame; + } + } Canvas_Theme_Resolve( &g_app->theme, Canvas_Theme_Mode_From_String(getenv("INFINITE_CANVAS_THEME"))); @@ -219,6 +767,11 @@ g_app->theme.resolved_mode == CANVAS_THEME_DARK ? TRUE : FALSE)) { TraceLog(LOG_WARNING, "Web surface initialization failed"); } + g_app->agent_initialized = + Canvas_Agent_Service_Init(&g_app->agent_service, p_arena); + if (!g_app->agent_initialized) { + TraceLog(LOG_WARNING, "Agent service initialization failed"); + } g_app->font = LoadFontEx("infinite_canvas/assets/Inter-Variable.ttf", 48, NULL, 0); g_app->owns_font = g_app->font.texture.id != 0; @@ -231,6 +784,13 @@ } else { Canvas_Scene_Add_Demo(&g_app->scene); } + if (getenv("INFINITE_CANVAS_DICTATION_AUTOSTART")) { + Canvas_App_Set_Dictation_Active(TRUE); + } + const char *p_initial_prompt = getenv("INFINITE_CANVAS_AGENT_PROMPT"); + if (p_initial_prompt && p_initial_prompt[0]) { + Canvas_App_Submit_Prompt(p_initial_prompt); + } const char *p_image_source = getenv("INFINITE_CANVAS_IMAGE_SOURCE"); if (p_image_source) { for (size_t index = 0; index < Dowa_Array_Length(g_app->scene.p_entities); index++) { @@ -258,6 +818,9 @@ emscripten_set_main_loop(Canvas_App_Draw_Frame, 0, 1); #else while (!WindowShouldClose()) Canvas_App_Draw_Frame(); + if (g_app->agent_initialized) { + Canvas_Agent_Service_Shutdown(&g_app->agent_service); + } Canvas_Web_Surface_Shutdown(&g_app->web_surface); if (g_app->owns_font) UnloadFont(g_app->font); Dowa_Arena_Free(p_arena);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/run_orchestration_dev.sh Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNFILES_ROOT="${RUNFILES_DIR:-$0.runfiles}" +export RUNFILES_DIR="$RUNFILES_ROOT" + +find_runfile() { + local pattern="$1" + local path + path="$(find -L "$RUNFILES_ROOT" -path "$pattern" -print -quit)" + if [[ -z "$path" ]]; then + echo "Could not locate orchestration runtime file: $pattern" >&2 + exit 1 + fi + realpath "$path" +} + +find_config() { + local candidate + for candidate in \ + "${BUILD_WORKSPACE_DIRECTORY:-}/mrjunejune/.config" \ + "$PWD/mrjunejune/.config" \ + "${HOME:-}/.config/mrjunejune/.config"; do + [[ "$candidate" != /mrjunejune/.config ]] || continue + [[ "$candidate" != /.config/mrjunejune/.config ]] || continue + if [[ -f "$candidate" ]]; then + realpath "$candidate" + return 0 + fi + done + return 1 +} + +load_config() { + local config_path="$1" + local line key value + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%$'\r'}" + [[ "$line" =~ ^[[:space:]]*$ ]] && continue + [[ "$line" =~ ^[[:space:]]*# ]] && continue + if [[ "$line" != *=* ]]; then + echo "Invalid config line in $config_path" >&2 + exit 1 + fi + key="${line%%=*}" + value="${line#*=}" + key="${key#"${key%%[![:space:]]*}"}" + key="${key%"${key##*[![:space:]]}"}" + case "$key" in + LITELLM_MASTER_KEY|LITELLM_HOST|LITELLM_PORT|LITELLM_MODEL|\ + LITELLM_WIRE_API|GITHUB_COPILOT_TOKEN_DIR|\ + MRJUNEJUNE_INFERENCE_STATE|COPILOT_SESSION_IDLE_SECONDS|\ + COPILOT_MAX_SESSIONS) + printf -v "$key" '%s' "$value" + ;; + esac + done < "$config_path" +} + +dictation="$(find_runfile '*/dictation/server')" +dictation_bin="$(find_runfile '*/dictation/dictation_bin')" +canvas="$(find_runfile '*/infinite_canvas/dev_linux')" +sidecar_launcher="$(find_runfile '*/mrjunejune/inference_sidecar_launcher.sh')" +sidecar_zip="$(find_runfile '*/mrjunejune/inference/copilot_sidecar.zip')" +litellm_zip="$(find_runfile '*/mrjunejune/inference/litellm_proxy.zip')" +litellm_config="$(find_runfile '*/mrjunejune/inference/litellm_config.yaml')" +copilot_cli="$(find_runfile '*copilot_cli_linux_x86_64/copilot')" +python="$(find_runfile '*python_3_11_x86_64-unknown-linux-gnu/bin/python3')" + +config_file="$(find_config || true)" +if [[ -z "$config_file" ]]; then + echo "Missing mrjunejune/.config." >&2 + echo "Copy mrjunejune/.config.development to mrjunejune/.config." >&2 + exit 1 +fi +load_config "$config_file" + +state_root="${MRJUNEJUNE_INFERENCE_STATE:-${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference}" +token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}" +cache_root="$state_root/cache" +mkdir -p "$state_root/copilot" "$token_dir" "$cache_root" +chmod 700 "$state_root" "$state_root/copilot" "$token_dir" "$cache_root" + +if [[ ! -s "$token_dir/access-token" ]]; then + echo "GitHub Copilot is not authenticated." >&2 + echo "Run: bazel run //mrjunejune:run_inference_stack -- --authenticate" >&2 + exit 1 +fi + +if [[ -z "${LITELLM_MASTER_KEY:-}" ]]; then + umask 077 + LITELLM_MASTER_KEY="sk-$("$python" -c 'import secrets; print(secrets.token_hex(24))')" +fi + +litellm_host="${LITELLM_HOST:-127.0.0.1}" +litellm_port="${LITELLM_PORT:-4000}" +service_pids=() + +cleanup() { + local status=$? + trap - EXIT INT TERM + for pid in "${service_pids[@]}"; do + kill -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true + done + for pid in "${service_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + exit "$status" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +export GITHUB_COPILOT_TOKEN_DIR="$token_dir" +export XDG_CACHE_HOME="$cache_root" +export DICTATION_MODEL_DIR="${DICTATION_MODEL_DIR:-$HOME/.cache/zenbu/faster-whisper-small}" +export LITELLM_MASTER_KEY +setsid "$python" "$litellm_zip" \ + --config "$litellm_config" \ + --host "$litellm_host" \ + --port "$litellm_port" & +service_pids+=("$!") + +setsid "$dictation" "$dictation_bin" server & +service_pids+=("$!") + +for _ in {1..150}; do + if curl --fail --silent \ + --connect-timeout 1 \ + --max-time 2 \ + "http://${litellm_host}:${litellm_port}/health/readiness" \ + >/dev/null; then + break + fi + if ! kill -0 "${service_pids[0]}" 2>/dev/null; then + echo "Copilot LiteLLM gateway exited before becoming ready." >&2 + exit 1 + fi + sleep 0.1 +done +if ! curl --fail --silent \ + --connect-timeout 1 \ + --max-time 2 \ + "http://${litellm_host}:${litellm_port}/health/readiness" \ + >/dev/null; then + echo "Copilot LiteLLM readiness timed out." >&2 + exit 1 +fi + +for _ in {1..120}; do + if curl --silent --fail --output /dev/null http://127.0.0.1:8090/; then + break + fi + if ! kill -0 "${service_pids[1]}" 2>/dev/null; then + echo "The dictation service exited before becoming ready." >&2 + exit 1 + fi + sleep 0.25 +done +if ! curl --silent --fail --output /dev/null http://127.0.0.1:8090/; then + echo "Timed out waiting for dictation at http://127.0.0.1:8090." >&2 + exit 1 +fi + +export MRJUNEJUNE_PYTHON_PATH="$python" +export MRJUNEJUNE_SIDECAR_ZIP="$sidecar_zip" +export COPILOT_SIDECAR_HOME="$state_root/copilot" +export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1" +export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}" +export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}" +export LITELLM_API_KEY="$LITELLM_MASTER_KEY" +export INFINITE_CANVAS_COPILOT_SIDECAR_PATH="$sidecar_launcher" +export INFINITE_CANVAS_COPILOT_CLI_PATH="$copilot_cli" + +echo "Copilot orchestration is ready. Starting Infinite Canvas." +canvas_status=0 +"$canvas" "$@" || canvas_status=$? +exit "$canvas_status"
--- a/infinite_canvas/web_surface.h Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface.h Tue Aug 18 19:14:53 2026 -0700 @@ -5,6 +5,13 @@ #define CANVAS_MAX_WEB_VIEWS 12 +typedef enum { + CANVAS_DICTATION_EVENT_NONE = 0, + CANVAS_DICTATION_EVENT_STATUS, + CANVAS_DICTATION_EVENT_PARTIAL, + CANVAS_DICTATION_EVENT_FINAL, +} Canvas_Dictation_Event; + typedef struct { uint32 entity_id; Canvas_Entity_Type entity_type; @@ -16,6 +23,12 @@ size_t pixel_capacity; char source_url[CANVAS_ENTITY_TEXT_CAPACITY]; char url[CANVAS_ENTITY_TEXT_CAPACITY]; + char dictation_text[CANVAS_ENTITY_TEXT_CAPACITY]; + uint32 dictation_sequence; + uint32 consumed_dictation_sequence; + int32 dictation_command; + Canvas_Dictation_Event dictation_event; + boolean is_dictation_transport; boolean can_go_back; boolean can_go_forward; boolean color_scheme_applied; @@ -31,6 +44,7 @@ void *p_native; Canvas_Web_View views[CANVAS_MAX_WEB_VIEWS]; uint32 focused_entity_id; + int32 pending_dictation_command; boolean dark_mode; boolean initialized; } Canvas_Web_Surface; @@ -79,6 +93,15 @@ void Canvas_Web_Surface_Go_Forward( Canvas_Web_Surface *p_surface, uint32 entity_id); +void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active); +void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface); +Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity); boolean Canvas_Web_Surface_Is_Focused(const Canvas_Web_Surface *p_surface); void Canvas_Web_Surface_Shutdown(Canvas_Web_Surface *p_surface);
--- a/infinite_canvas/web_surface_native.cc Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface_native.cc Tue Aug 18 19:14:53 2026 -0700 @@ -6,16 +6,20 @@ #include "include/cef_browser.h" #include "include/cef_client.h" #include "include/cef_display_handler.h" +#include "include/cef_permission_handler.h" #include "include/cef_render_handler.h" +#include "include/cef_request_context.h" #include "include/capi/cef_app_capi.h" #include <algorithm> +#include <cctype> #include <cmath> #include <cstdio> #include <cstdlib> #include <cstring> #include <filesystem> #include <new> +#include <string> #if defined(OS_LINUX) #include <dlfcn.h> @@ -29,6 +33,8 @@ constexpr int32 kMinTextureHeight = 120; constexpr int32 kMaxTextureWidth = 1280; constexpr int32 kMaxTextureHeight = 800; +constexpr uint32 kDictationViewId = 0xffffffffu; +constexpr const char *kDictationUrl = "http://127.0.0.1:8090/?canvas=1"; int g_argc = 0; char **g_argv = nullptr; char *g_cef_argv[128] = {}; @@ -41,6 +47,26 @@ void FindResourcePaths(); +void PercentDecode(const char *source, char *target, size_t target_size) { + size_t output = 0; + for (size_t index = 0; + source[index] && output + 1 < target_size; + ++index) { + if (source[index] == '%' && + std::isxdigit(static_cast<unsigned char>(source[index + 1])) && + std::isxdigit(static_cast<unsigned char>(source[index + 2]))) { + char hex[3] = {source[index + 1], source[index + 2], '\0'}; + target[output++] = static_cast<char>(std::strtol(hex, nullptr, 16)); + index += 2; + } else if (source[index] == '+') { + target[output++] = ' '; + } else { + target[output++] = source[index]; + } + } + target[output] = '\0'; +} + void ApplyColorScheme(CefRefPtr<CefBrowser> browser, boolean dark_mode) { if (!browser) return; CefRefPtr<CefDictionaryValue> params = CefDictionaryValue::Create(); @@ -62,6 +88,12 @@ frame->ExecuteJavaScript(script, frame->GetURL(), 0); } +boolean IsLocalDictationOrigin(const CefString &requesting_origin) { + std::string origin = requesting_origin.ToString(); + return origin.rfind("http://127.0.0.1:8090", 0) == 0 || + origin.rfind("http://localhost:8090", 0) == 0; +} + const Canvas_Entity *FindSurfaceEntity( const Canvas_Scene *scene, uint32 entity_id) { @@ -167,12 +199,16 @@ class CanvasCefClient : public CefClient, public CefRenderHandler, public CefDisplayHandler, + public CefPermissionHandler, public CefLifeSpanHandler { public: explicit CanvasCefClient(Canvas_Web_View *view) : view_(view) {} CefRefPtr<CefRenderHandler> GetRenderHandler() override { return this; } CefRefPtr<CefDisplayHandler> GetDisplayHandler() override { return this; } + CefRefPtr<CefPermissionHandler> GetPermissionHandler() override { + return this; + } CefRefPtr<CefLifeSpanHandler> GetLifeSpanHandler() override { return this; } void OnAddressChange( @@ -189,6 +225,83 @@ view_->can_go_forward = browser->CanGoForward() ? TRUE : FALSE; } + void OnTitleChange( + CefRefPtr<CefBrowser> browser, + const CefString &title) override { + (void)browser; + std::string value = title.ToString(); + constexpr const char *prefix = "Zenbu Dictation|"; + if (value.rfind(prefix, 0) != 0) return; + const char *kind = std::strchr( + value.c_str() + std::strlen(prefix), + '|'); + if (!kind) return; + kind++; + const char *encoded = std::strchr(kind, '|'); + if (!encoded) return; + std::string event_kind(kind, static_cast<size_t>(encoded - kind)); + if (event_kind == "status") { + view_->dictation_event = CANVAS_DICTATION_EVENT_STATUS; + } else if (event_kind == "partial") { + view_->dictation_event = CANVAS_DICTATION_EVENT_PARTIAL; + } else if (event_kind == "final") { + view_->dictation_event = CANVAS_DICTATION_EVENT_FINAL; + } else { + return; + } + PercentDecode( + encoded + 1, + view_->dictation_text, + sizeof(view_->dictation_text)); + view_->dictation_sequence++; + } + + bool OnRequestMediaAccessPermission( + CefRefPtr<CefBrowser> browser, + CefRefPtr<CefFrame> frame, + const CefString &requesting_origin, + uint32_t requested_permissions, + CefRefPtr<CefMediaAccessCallback> callback) override { + (void)browser; + (void)frame; + boolean local_dictation = + view_->is_dictation_transport && + IsLocalDictationOrigin(requesting_origin); + uint32_t audio = CEF_MEDIA_PERMISSION_DEVICE_AUDIO_CAPTURE; + boolean requests_audio = (requested_permissions & audio) != 0; + boolean requests_other_media = (requested_permissions & ~audio) != 0; + if (local_dictation && requests_audio && !requests_other_media) { + callback->Continue(audio); + } else { + callback->Cancel(); + } + return true; + } + + bool OnShowPermissionPrompt( + CefRefPtr<CefBrowser> browser, + uint64_t prompt_id, + const CefString &requesting_origin, + uint32_t requested_permissions, + CefRefPtr<CefPermissionPromptCallback> callback) override { + (void)browser; + (void)prompt_id; + if (!view_->is_dictation_transport || + !IsLocalDictationOrigin(requesting_origin)) { + return false; + } + uint32_t microphone = CEF_PERMISSION_TYPE_MIC_STREAM; + boolean requests_microphone = + (requested_permissions & microphone) != 0; + boolean requests_other_permissions = + (requested_permissions & ~microphone) != 0; + callback->Continue( + requests_microphone && !requests_other_permissions + ? CEF_PERMISSION_RESULT_ACCEPT + : CEF_PERMISSION_RESULT_DENY); + return true; + } + void GetViewRect(CefRefPtr<CefBrowser> browser, CefRect &rect) override { (void)browser; rect = CefRect(0, 0, view_->texture_width, view_->texture_height); @@ -243,9 +356,17 @@ struct NativeState { CefRefPtr<CanvasCefClient> clients[CANVAS_MAX_WEB_VIEWS]; + CefRefPtr<CefRequestContext> dictation_context; boolean cef_initialized; }; +void EnsureDictationRequestContext(NativeState *state) { + if (state->dictation_context) return; + CefRequestContextSettings settings; + state->dictation_context = + CefRequestContext::CreateContext(settings, nullptr); +} + CefMainArgs MainArgs(int argc, char **argv) { #if defined(OS_WIN) (void)argc; @@ -396,13 +517,18 @@ : CefColorSetARGB(255, 255, 255, 255); const char *initial_url = view->entity_type == CANVAS_ENTITY_IMAGE ? "about:blank" : url; + CefRefPtr<CefRequestContext> request_context; + if (view->is_dictation_transport) { + EnsureDictationRequestContext(state); + request_context = state->dictation_context; + } if (!CefBrowserHost::CreateBrowser( window_info, state->clients[index], initial_url, browser_settings, nullptr, - nullptr)) { + request_context)) { state->clients[index] = nullptr; return FALSE; } @@ -410,6 +536,47 @@ return TRUE; } +Canvas_Web_View *EnsureDictationView(Canvas_Web_Surface *surface) { + Canvas_Web_View *view = FindView(surface, kDictationViewId); + if (view) { + view->active = TRUE; + return view; + } + view = FindInactiveView(surface); + if (!view) return nullptr; + boolean browser_created = view->browser_created; + view->active = TRUE; + view->entity_id = kDictationViewId; + view->entity_type = CANVAS_ENTITY_WEB_CONTENT; + view->is_dictation_transport = TRUE; + std::snprintf( + view->source_url, + sizeof(view->source_url), + "%s", + kDictationUrl); + std::snprintf(view->url, sizeof(view->url), "%s", kDictationUrl); + Rectangle bounds = {0.0f, 0.0f, 320.0f, 240.0f}; + if (!browser_created && + !CreateBrowser(surface, view, kDictationUrl, bounds)) { + view->active = FALSE; + view->entity_id = 0; + view->is_dictation_transport = FALSE; + view->source_url[0] = '\0'; + view->url[0] = '\0'; + return nullptr; + } + if (browser_created) { + EnsureViewResolution(surface, view, bounds); + NativeState *state = static_cast<NativeState *>(surface->p_native); + CefRefPtr<CanvasCefClient> client = + state->clients[ViewIndex(surface, view)]; + if (client && client->browser()) { + client->browser()->GetMainFrame()->LoadURL(kDictationUrl); + } + } + return view; +} + void PercentEncode( const char *source, char *target, @@ -528,8 +695,9 @@ const Canvas_Scene *scene) { for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { Canvas_Web_View *view = &surface->views[index]; - view->active = view->entity_id != 0 && - SceneHasVisibleEntity(scene, camera, view->entity_id); + view->active = view->is_dictation_transport || + (view->entity_id != 0 && + SceneHasVisibleEntity(scene, camera, view->entity_id)); } for (size_t index = 0; index < Dowa_Array_Length(scene->p_entities); ++index) { @@ -562,7 +730,7 @@ Canvas_Web_View *view = &surface->views[index]; CefRefPtr<CefBrowserHost> host = client->browser()->GetHost(); host->WasHidden(!view->active); - if (view->active) { + if (view->active && !view->is_dictation_transport) { const Canvas_Entity *entity = FindSurfaceEntity(scene, view->entity_id); if (entity) { host->SetWindowlessFrameRate( @@ -805,12 +973,22 @@ const Canvas_Scene *p_scene) { if (!p_surface->initialized) return; CefDoMessageLoopWork(); + EnsureDictationView(p_surface); ReconcileViews(p_surface, p_camera, p_scene); + if (p_surface->pending_dictation_command != 0) { + Canvas_Web_View *view = EnsureDictationView(p_surface); + if (view) { + view->dictation_command = p_surface->pending_dictation_command; + p_surface->pending_dictation_command = 0; + } + } for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { Canvas_Web_View *view = &p_surface->views[index]; if (!view->browser_created) continue; - if (!view->texture_ready && IsWindowReady()) { + if (!view->is_dictation_transport && + !view->texture_ready && + IsWindowReady()) { Image image = { .data = view->p_pixels, .width = view->texture_width, @@ -824,7 +1002,9 @@ SetTextureFilter(view->texture, TEXTURE_FILTER_BILINEAR); } } - if (view->texture_ready && view->pixels_dirty) { + if (!view->is_dictation_transport && + view->texture_ready && + view->pixels_dirty) { UpdateTexture(view->texture, view->p_pixels); view->pixels_dirty = FALSE; } @@ -840,6 +1020,29 @@ } view->can_go_back = client->browser()->CanGoBack() ? TRUE : FALSE; view->can_go_forward = client->browser()->CanGoForward() ? TRUE : FALSE; + if (view->dictation_command != 0) { + CefRefPtr<CefFrame> frame = client->browser()->GetMainFrame(); + const char *function_name = + view->dictation_command == 2 + ? "ZenbuDictationCommit" + : (view->dictation_command > 0 + ? "ZenbuDictationStart" + : "ZenbuDictationStop"); + char script[512]; + std::snprintf( + script, + sizeof(script), + "(()=>{let attempts=0;const invoke=()=>{" + "if(window.%s){window.%s();return;}" + "if(attempts++<40)setTimeout(invoke,250);};invoke();})()", + function_name, + function_name); + frame->ExecuteJavaScript( + script, + frame->GetURL(), + 0); + view->dictation_command = 0; + } } } } @@ -955,6 +1158,42 @@ } } +extern "C" void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active) { + int32 command = active ? 1 : -1; + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (view) { + view->dictation_command = command; + } else { + p_surface->pending_dictation_command = command; + } +} + +extern "C" void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface) { + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (view) { + view->dictation_command = 2; + } else { + p_surface->pending_dictation_command = 2; + } +} + +extern "C" Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity) { + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (!view || + view->dictation_sequence == view->consumed_dictation_sequence) { + return CANVAS_DICTATION_EVENT_NONE; + } + snprintf(p_text, text_capacity, "%s", view->dictation_text); + view->consumed_dictation_sequence = view->dictation_sequence; + return view->dictation_event; +} + extern "C" boolean Canvas_Web_Surface_Is_Focused( const Canvas_Web_Surface *p_surface) { return p_surface->focused_entity_id != 0;
--- a/infinite_canvas/web_surface_web.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface_web.c Tue Aug 18 19:14:53 2026 -0700 @@ -9,6 +9,69 @@ }); }); +EM_JS(void, Canvas_Web_Dictation_Init, (), { + if (Module.canvasDictationInitialized) return; + Module.canvasDictationInitialized = true; + Module.canvasDictationQueue = []; + window.addEventListener("message", event => { + if (event.data?.type !== "zenbu.dictation.event") return; + const frame = document.getElementById("canvas-dictation-transport"); + if (!frame || frame.contentWindow !== event.source) return; + Module.canvasDictationQueue.push({ + kind: String(event.data.kind || ""), + text: String(event.data.text || ""), + }); + }); +}); + +EM_JS(void, Canvas_Web_Dictation_Set_Active, (int active), { + let frame = document.getElementById("canvas-dictation-transport"); + if (!frame) { + frame = document.createElement("iframe"); + frame.id = "canvas-dictation-transport"; + frame.title = "Canvas dictation transport"; + frame.allow = "microphone"; + frame.sandbox = + "allow-same-origin allow-scripts"; + frame.style.position = "fixed"; + frame.style.left = "-2px"; + frame.style.top = "-2px"; + frame.style.width = "1px"; + frame.style.height = "1px"; + frame.style.opacity = "0"; + frame.style.pointerEvents = "none"; + frame.src = "http://127.0.0.1:8090/?canvas=1"; + document.body.appendChild(frame); + } + const send = () => frame.contentWindow.postMessage( + { + type: "zenbu.dictation.set-active", + active: active !== 0, + }, + "*"); + frame.addEventListener("load", send, {once: true}); + send(); +}); + +EM_JS(void, Canvas_Web_Dictation_Commit, (), { + const frame = document.getElementById("canvas-dictation-transport"); + frame?.contentWindow?.postMessage( + {type: "zenbu.dictation.commit"}, + "*"); +}); + +EM_JS(int, Canvas_Web_Dictation_Consume, ( + char *p_text, + int text_capacity), { + const event = Module.canvasDictationQueue?.shift(); + if (!event) return 0; + stringToUTF8(event.text, p_text, text_capacity); + if (event.kind === "status") return 1; + if (event.kind === "partial") return 2; + if (event.kind === "final") return 3; + return 0; +}); + EM_JS(void, Canvas_Web_Create_Or_Update, ( uint32 entity_id, const char *p_url, @@ -170,6 +233,7 @@ } surface.remove(); }); + document.getElementById("canvas-dictation-transport")?.remove(); }); static Rectangle Canvas_Web_Content_Bounds( @@ -236,6 +300,7 @@ p_surface->p_arena = p_arena; p_surface->dark_mode = dark_mode; p_surface->initialized = TRUE; + Canvas_Web_Dictation_Init(); return TRUE; } @@ -369,6 +434,32 @@ Canvas_Web_History_Go(entity_id, 1); } +void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active) +{ + (void)p_surface; + Canvas_Web_Dictation_Set_Active(active); +} + +void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface) +{ + (void)p_surface; + Canvas_Web_Dictation_Commit(); +} + +Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity) +{ + (void)p_surface; + return (Canvas_Dictation_Event)Canvas_Web_Dictation_Consume( + p_text, + (int)text_capacity); +} + boolean Canvas_Web_Surface_Is_Focused(const Canvas_Web_Surface *p_surface) { (void)p_surface;
--- a/mrjunejune/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -331,7 +331,10 @@ hdrs = ["inference_bridge.h"], deps = ["//dowa:dowa"], linkopts = ["-lpthread"], - visibility = ["//mrjunejune/test:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune/test:__pkg__", + ], ) filegroup( @@ -356,6 +359,12 @@ ], ) +filegroup( + name = "canvas_orchestration_launcher", + srcs = ["inference_sidecar_launcher.sh"], + visibility = ["//infinite_canvas:__pkg__"], +) + # Server binary cc_binary( name = "mrjunejune_server",
--- a/mrjunejune/inference/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -13,7 +13,10 @@ exports_files( ["litellm_config.yaml"], - visibility = ["//mrjunejune:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune:__pkg__", + ], ) py_binary( @@ -63,13 +66,19 @@ python_zip_file( name = "copilot_sidecar_zip", binary = ":copilot_sidecar", - visibility = ["//mrjunejune:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune:__pkg__", + ], ) python_zip_file( name = "litellm_proxy_zip", binary = ":litellm_proxy", - visibility = ["//mrjunejune:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune:__pkg__", + ], ) filegroup(
--- a/mrjunejune/inference/copilot_sidecar.py Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference/copilot_sidecar.py Tue Aug 18 19:14:53 2026 -0700 @@ -2,6 +2,7 @@ import argparse import asyncio +import hashlib import json import os import sys @@ -20,8 +21,27 @@ JsonObject = dict[str, Any] Emit = Callable[[JsonObject], Awaitable[None]] -_KNOWN_PROFILES: frozenset = frozenset({"public_visitor", "invited_friend", "june_admin"}) +_KNOWN_PROFILES: frozenset = frozenset({ + "public_visitor", + "invited_friend", + "june_admin", + "canvas_orchestrator", +}) _PROMPT_VERSION: int = 1 +_CANVAS_ORCHESTRATOR_PROMPT = """\ +You orchestrate spatial agent sessions on an infinite canvas. +There is one user input scratchpad and any number of conversation entities. +The user message includes only the visible canvas context plus one new thought. +Treat visible conversation entities as the only append candidates. Append when +the new thought clearly continues one of them; otherwise create a new +conversation. Never append to a conversation ID absent from visible context. +Return only one compact JSON object with these fields: +- action: "create" or "append" +- conversation_id: an existing numeric conversation ID for append, otherwise 0 +- title: a short session title +- response: the direct response that should appear on the canvas +Do not use Markdown fences, tools, or text outside the JSON object. +""" # Fixed namespace for uuid5 SDK session ID derivation. Must never change. _SDK_SESSION_NAMESPACE = uuid.UUID("3f7e8a1d-9b52-4c6f-a0d3-82e1f5c94b7a") @@ -254,7 +274,17 @@ def _compile_all_profiles(self) -> None: compiled: dict[str, CompiledProfile] = {} for profile in sorted(_KNOWN_PROFILES): - result = self._compile_fn(profile) + if profile == "canvas_orchestrator": + content = _CANVAS_ORCHESTRATOR_PROMPT.strip() + result = { + "content": content, + "version": 1, + "hash": hashlib.sha256( + content.encode("utf-8") + ).hexdigest(), + } + else: + result = self._compile_fn(profile) compiled[profile] = CompiledProfile( profile=profile, content=result["content"], @@ -327,7 +357,9 @@ await self.evict_idle_sessions( exclude=conversation_id, - reserve=1 if command_name == "turn.start" else 0, + reserve=1 + if command_name in {"turn.start", "conversation.warm"} + else 0, ) async with self._conversation_command_lock(conversation_id): if command_name == "turn.start": @@ -365,6 +397,45 @@ ) return await self._start_turn(request_id, conversation_id, prompt, compiled, history) + elif command_name == "conversation.warm": + compiled = self._validate_profile_fields( + command.get("prompt_profile"), + command.get("prompt_version"), + command.get("knowledge_version"), + ) + if compiled is None: + await self._fail( + request_id, + conversation_id, + "invalid_prompt_profile", + "a known prompt profile and current versions are required", + ) + return + resume_existing = command.get("resume_existing", True) + if not isinstance(resume_existing, bool): + await self._fail( + request_id, + conversation_id, + "invalid_request", + "resume_existing must be a boolean", + ) + return + await self._get_conversation( + conversation_id, + compiled, + [], + resume_existing=resume_existing, + ) + await self._send( + "session.warmed", + request_id, + conversation_id, + ) + await self._send( + "turn.done", + request_id, + conversation_id, + ) elif command_name == "turn.abort": await self._abort_turn(request_id, conversation_id) elif command_name == "conversation.delete": @@ -486,6 +557,8 @@ conversation_id: str, compiled: CompiledProfile, history: list[dict[str, str]], + *, + resume_existing: bool = True, ) -> Conversation: derived_id = _derive_sdk_session_id(conversation_id, compiled) async with self._conversations_lock: @@ -508,12 +581,13 @@ raise RuntimeError("Copilot session capacity exhausted") base_options = self._session_options(compiled) - # Try to resume a persisted derived session (no history injection). session: Any = None - try: - session = await self._client.resume_session(derived_id, **base_options) - except Exception: - pass + if resume_existing: + # Resume persisted user-facing sessions, then create if absent. + try: + session = await self._client.resume_session(derived_id, **base_options) + except Exception: + pass if session is None: # Fresh create — inject bounded transcript context into system message.
--- a/mrjunejune/inference/copilot_sidecar_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference/copilot_sidecar_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -10,6 +10,7 @@ Sidecar, SidecarConfig, _derive_sdk_session_id, + _CANVAS_ORCHESTRATOR_PROMPT, _SDK_SESSION_NAMESPACE, _validate_history, _HISTORY_MAX_ENTRIES, @@ -50,7 +51,15 @@ def _fake_sdk_id(conversation_id: str, profile: str) -> str: """Compute the deterministic SDK session ID for use in test assertions.""" - p = _FAKE_PROFILES[profile] + if profile == "canvas_orchestrator": + content = _CANVAS_ORCHESTRATOR_PROMPT.strip() + p = { + "content": content, + "version": 1, + "hash": hashlib.sha256(content.encode()).hexdigest(), + } + else: + p = _FAKE_PROFILES[profile] compiled = CompiledProfile( profile=profile, content=p["content"], @@ -210,6 +219,54 @@ ["turn.error", "turn.done"], ) + async def test_warm_creates_session_without_sending_prompt(self): + sidecar, client = await self.make_sidecar() + await sidecar.dispatch( + { + "command": "conversation.warm", + "request_id": "warm-1", + "conversation_id": "canvas-worker-1", + "prompt_profile": "canvas_orchestrator", + "prompt_version": 1, + "knowledge_version": 1, + "resume_existing": False, + } + ) + + self.assertEqual(len(client.create_calls), 1) + self.assertEqual(len(client.resume_calls), 0) + session = next(iter(client.sessions.values())) + self.assertEqual(session.prompts, []) + self.assertEqual( + [item["type"] for item in self.output[-2:]], + ["session.warmed", "turn.done"], + ) + + async def test_warm_rejects_non_boolean_resume_existing(self): + sidecar, client = await self.make_sidecar() + await sidecar.dispatch( + { + "command": "conversation.warm", + "request_id": "warm-invalid", + "conversation_id": "canvas-worker-1", + "prompt_profile": "canvas_orchestrator", + "prompt_version": 1, + "knowledge_version": 1, + "resume_existing": "false", + } + ) + + self.assertEqual(len(client.create_calls), 0) + self.assertEqual(len(client.resume_calls), 0) + self.assertEqual( + [item["type"] for item in self.output[-2:]], + ["turn.error", "turn.done"], + ) + self.assertEqual( + self.output[-2]["error"]["message"], + "resume_existing must be a boolean", + ) + async def test_multi_turn_reuses_one_session_and_configures_sdk(self): sidecar, client = await self.make_sidecar() for request_id, prompt in (("r1", "first"), ("r2", "second")): @@ -411,6 +468,7 @@ _fake_sdk_id("delete-me", "public_visitor"), _fake_sdk_id("delete-me", "invited_friend"), _fake_sdk_id("delete-me", "june_admin"), + _fake_sdk_id("delete-me", "canvas_orchestrator"), }, ) self.assertTrue(client.stopped) @@ -958,6 +1016,7 @@ _fake_sdk_id("conv-del-check", "public_visitor"), _fake_sdk_id("conv-del-check", "invited_friend"), _fake_sdk_id("conv-del-check", "june_admin"), + _fake_sdk_id("conv-del-check", "canvas_orchestrator"), }, ) self.assertNotIn("conv-del-check", client.delete_calls) @@ -977,6 +1036,7 @@ _fake_sdk_id("ghost-conv", "public_visitor"), _fake_sdk_id("ghost-conv", "invited_friend"), _fake_sdk_id("ghost-conv", "june_admin"), + _fake_sdk_id("ghost-conv", "canvas_orchestrator"), }, ) self.assertNotIn("ghost-conv", client.delete_calls)
--- a/mrjunejune/inference_bridge.c Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference_bridge.c Tue Aug 18 19:14:53 2026 -0700 @@ -198,6 +198,8 @@ return "invited_friend"; case INFERENCE_PROMPT_PROFILE_JUNE_ADMIN: return "june_admin"; + case INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR: + return "canvas_orchestrator"; } return NULL; } @@ -210,7 +212,9 @@ const char *prompt, const char *prompt_profile, uint32 prompt_version, - uint32 knowledge_version) + uint32 knowledge_version, + boolean include_resume_existing, + boolean resume_existing) { if (!command || !request_id) return FALSE; @@ -269,6 +273,27 @@ prompt_version, knowledge_version); } + else if (escaped_profile) + { + snprintf( + payload, + capacity, + "{\"command\":\"%s\",\"request_id\":\"%s\"," + "\"conversation_id\":\"%s\"," + "\"prompt_profile\":\"%s\",\"prompt_version\":%u," + "\"knowledge_version\":%u%s}", + escaped_command, + escaped_request, + escaped_conversation, + escaped_profile, + prompt_version, + knowledge_version, + include_resume_existing + ? (resume_existing + ? ",\"resume_existing\":true" + : ",\"resume_existing\":false") + : ""); + } else { snprintf( @@ -333,7 +358,8 @@ return; if (atomic_load(&p_bridge->running) && p_bridge->p_commands) Inference_Bridge_Command( - p_bridge, "shutdown", "server-shutdown", "", NULL, NULL, 0, 0); + p_bridge, "shutdown", "server-shutdown", "", NULL, NULL, 0, 0, + FALSE, FALSE); if (p_bridge->p_commands) { fclose(p_bridge->p_commands); @@ -642,13 +668,40 @@ return success; } +boolean Inference_Bridge_Warm_Conversation( + Inference_Bridge *p_bridge, + const char *request_id, + const char *conversation_id, + Inference_Prompt_Profile prompt_profile, + uint32 prompt_version, + uint32 knowledge_version, + boolean resume_existing) +{ + const char *profile = Inference_Bridge_Profile_Name(prompt_profile); + if (!p_bridge || !request_id || !request_id[0] || + !conversation_id || !conversation_id[0] || !profile) + return FALSE; + return Inference_Bridge_Command( + p_bridge, + "conversation.warm", + request_id, + conversation_id, + NULL, + profile, + prompt_version, + knowledge_version, + TRUE, + resume_existing); +} + boolean Inference_Bridge_Abort_Turn( Inference_Bridge *p_bridge, const char *request_id, const char *conversation_id) { return Inference_Bridge_Command( - p_bridge, "turn.abort", request_id, conversation_id, NULL, NULL, 0, 0); + p_bridge, "turn.abort", request_id, conversation_id, NULL, NULL, 0, 0, + FALSE, FALSE); } boolean Inference_Bridge_Delete_Conversation( @@ -664,7 +717,9 @@ NULL, NULL, 0, - 0); + 0, + FALSE, + FALSE); } void Inference_Bridge_Destroy(Inference_Bridge *p_bridge)
--- a/mrjunejune/inference_bridge.h Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference_bridge.h Tue Aug 18 19:14:53 2026 -0700 @@ -17,6 +17,7 @@ INFERENCE_PROMPT_PROFILE_PUBLIC_VISITOR = 0, INFERENCE_PROMPT_PROFILE_INVITED_FRIEND = 1, INFERENCE_PROMPT_PROFILE_JUNE_ADMIN = 2, + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR = 3, } Inference_Prompt_Profile; typedef struct { @@ -68,6 +69,15 @@ const Inference_Bridge_History_Message *p_history, uint32 history_count); +boolean Inference_Bridge_Warm_Conversation( + Inference_Bridge *p_bridge, + const char *request_id, + const char *conversation_id, + Inference_Prompt_Profile prompt_profile, + uint32 prompt_version, + uint32 knowledge_version, + boolean resume_existing); + boolean Inference_Bridge_Abort_Turn( Inference_Bridge *p_bridge, const char *request_id,
--- a/mrjunejune/test/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/test/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -77,6 +77,7 @@ name = "inference_bridge_fake_sidecar", srcs = ["inference_bridge_fake_sidecar.c"], deps = ["//dowa:dowa"], + visibility = ["//infinite_canvas:__pkg__"], ) cc_test(
--- a/mrjunejune/test/inference_bridge_fake_sidecar.c Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/test/inference_bridge_fake_sidecar.c Tue Aug 18 19:14:53 2026 -0700 @@ -100,6 +100,22 @@ Dowa_Arena_Free(p_arena); break; } + if (strcmp(command, "conversation.warm") == 0) + { + printf( + "{\"type\":\"session.warmed\",\"request_id\":\"%s\"," + "\"conversation_id\":\"%s\"}\n", + request_id, + conversation_id); + printf( + "{\"type\":\"turn.done\",\"request_id\":\"%s\"," + "\"conversation_id\":\"%s\"}\n", + request_id, + conversation_id); + fflush(stdout); + Dowa_Arena_Free(p_arena); + continue; + } if (strcmp(command, "turn.start") == 0) { /* Validate history before any other processing. */ @@ -131,7 +147,8 @@ boolean known_profile = strcmp(prompt_profile, "public_visitor") == 0 || strcmp(prompt_profile, "invited_friend") == 0 || - strcmp(prompt_profile, "june_admin") == 0; + strcmp(prompt_profile, "june_admin") == 0 || + strcmp(prompt_profile, "canvas_orchestrator") == 0; if (!known_profile || prompt_version != 1 || knowledge_version != 1) { printf(
--- a/mrjunejune/test/inference_bridge_test.c Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/test/inference_bridge_test.c Tue Aug 18 19:14:53 2026 -0700 @@ -13,6 +13,7 @@ int deltas; int completed; int usage; + int warmed; int custom; int done; int closed; @@ -32,6 +33,8 @@ pthread_mutex_lock(&p_state->mutex); if (strcmp(p_event->type, "turn.accepted") == 0) p_state->accepted++; + else if (strcmp(p_event->type, "session.warmed") == 0) + p_state->warmed++; else if (strcmp(p_event->type, "assistant.delta") == 0) { p_state->deltas++; @@ -87,6 +90,21 @@ pthread_mutex_unlock(&p_state->mutex); } +static void Wait_Warmed(Test_State *p_state, int target) +{ + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += 5; + pthread_mutex_lock(&p_state->mutex); + while (p_state->warmed < target) + { + int result = pthread_cond_timedwait( + &p_state->condition, &p_state->mutex, &deadline); + assert(result == 0); + } + pthread_mutex_unlock(&p_state->mutex); +} + int main(int argc, char **argv) { assert(argc == 2); @@ -100,6 +118,16 @@ assert(Inference_Bridge_Start(p_bridge)); assert(Inference_Bridge_Is_Ready(p_bridge)); + assert(Inference_Bridge_Warm_Conversation( + p_bridge, + "warm-1", + "warm-conversation", + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR, + 1, + 1, + FALSE)); + Wait_Warmed(&state, 1); + /* --- Validation: invalid profile --- */ assert(!Inference_Bridge_Start_Turn( p_bridge,
--- a/qwen3_vl/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/qwen3_vl/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -58,6 +58,17 @@ data = [":llama_cpp_cuda_runtime"], ) +sh_binary( + name = "serve_canvas", + srcs = ["qwen3_vl.sh"], + args = [ + "$(rootpath :llama_cpp_cuda_runtime)", + "serve_canvas", + ], + data = [":llama_cpp_cuda_runtime"], + visibility = ["//infinite_canvas:__pkg__"], +) + sh_test( name = "qwen3_vl_test", srcs = ["qwen3_vl_test.sh"],
--- a/qwen3_vl/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/qwen3_vl/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -40,6 +40,15 @@ curl.exe http://127.0.0.1:8080/health ``` +For the Linux Infinite Canvas process, start the server on the Windows WSL +adapter instead: + +```bash +bazel run //qwen3_vl:serve_canvas +``` + +The canvas detects the WSL gateway and uses that address by default. + The pinned CUDA executable runs on the Windows side of WSL interop. Windows browsers and clients can use `http://127.0.0.1:8080`; use `curl.exe`, rather than Linux `curl`, when checking it from a WSL shell.
--- a/qwen3_vl/qwen3_vl.sh Mon Aug 17 22:22:36 2026 -0700 +++ b/qwen3_vl/qwen3_vl.sh Tue Aug 18 19:14:53 2026 -0700 @@ -146,13 +146,21 @@ --top-p 0.8 \ "${NORMALIZED_ARGS[@]}" ;; - serve) + serve|serve_canvas) require_wsl require_model + host="${QWEN3_VL_HOST:-127.0.0.1}" + if [[ "$command" == "serve_canvas" && -z "${QWEN3_VL_HOST:-}" ]]; then + host="$(ip route show default | awk '{print $3; exit}')" + if [[ -z "$host" ]]; then + echo "Unable to resolve the Windows host address for WSL." >&2 + exit 1 + fi + fi exec "$runtime/llama-server.exe" \ --model "$(windows_path "$model_path")" \ --mmproj "$(windows_path "$mmproj_path")" \ - --host "${QWEN3_VL_HOST:-127.0.0.1}" \ + --host "$host" \ --port "${QWEN3_VL_PORT:-8080}" \ --ctx-size "${QWEN3_VL_CONTEXT_SIZE:-2048}" \ --parallel 1 \