# HG changeset patch # User MrJuneJune # Date 1787105693 25200 # Node ID 49e9e591c9bb3bf6416dab460e1f6080a2a5f322 # Parent b3b547563ec7371fe09181cf56b4476f6045e51f Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage. diff -r b3b547563ec7 -r 49e9e591c9bb dictation/BUILD --- a/dictation/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -70,6 +70,7 @@ requirement("nvidia-cublas-cu12"), requirement("nvidia-cudnn-cu12"), ], + visibility = ["//infinite_canvas:__pkg__"], ) for command in [ "server", diff -r b3b547563ec7 -r 49e9e591c9bb dictation/README.md --- a/dictation/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -36,11 +36,19 @@ ```bash DICTATION_COMPUTE_TYPE=int8_float16 \ DICTATION_MAX_SESSIONS=1 \ -DICTATION_PARTIAL_INTERVAL_MS=1200 \ -DICTATION_SILENCE_MS=700 \ +DICTATION_PARTIAL_INTERVAL_MS=500 \ +DICTATION_SILENCE_MS=400 \ bazel run //dictation:server ``` The first version intentionally binds to loopback and does not configure STUN or TURN. Remote WebRTC access requires HTTPS, authentication, origin policy, and usually a TURN service. + +Infinite Canvas hosts this page as a hidden CEF/iframe transport. Pressing `M` +shows partial and final transcript text in a centered Raylib caption; pressing +`Enter` sends the accumulated text to Qwen session orchestration: + +```bash +bazel run //infinite_canvas:agent_dev +``` diff -r b3b547563ec7 -r 49e9e591c9bb dictation/audio.py --- a/dictation/audio.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/audio.py Tue Aug 18 19:14:53 2026 -0700 @@ -24,8 +24,8 @@ *, sample_rate: int = 16000, speech_threshold: float = 0.012, - silence_ms: int = 700, - partial_interval_ms: int = 1200, + silence_ms: int = 400, + partial_interval_ms: int = 500, max_utterance_seconds: int = 30, pre_roll_ms: int = 200, ) -> None: diff -r b3b547563ec7 -r 49e9e591c9bb dictation/config.py --- a/dictation/config.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/config.py Tue Aug 18 19:14:53 2026 -0700 @@ -84,11 +84,11 @@ max_sessions=_integer("DICTATION_MAX_SESSIONS", 1, 1, 8), partial_interval_ms=_integer( "DICTATION_PARTIAL_INTERVAL_MS", - 1200, 500, + 250, 10000, ), - silence_ms=_integer("DICTATION_SILENCE_MS", 700, 200, 5000), + silence_ms=_integer("DICTATION_SILENCE_MS", 400, 200, 5000), max_utterance_seconds=_integer( "DICTATION_MAX_UTTERANCE_SECONDS", 30, diff -r b3b547563ec7 -r 49e9e591c9bb dictation/config_test.py --- a/dictation/config_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/config_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -14,6 +14,8 @@ self.assertEqual(config.port, 8090) self.assertEqual(config.max_sessions, 1) self.assertEqual(config.compute_type, "int8_float16") + self.assertEqual(config.partial_interval_ms, 500) + self.assertEqual(config.silence_ms, 400) self.assertTrue(str(config.model_dir).endswith("faster-whisper-small")) def test_rejects_invalid_session_limit(self): diff -r b3b547563ec7 -r 49e9e591c9bb dictation/server_test.py --- a/dictation/server_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/server_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -37,7 +37,21 @@ self.assertEqual(health.status_code, 200) self.assertEqual(health.json()["status"], "ok") self.assertIn("Zenbu Dictation", client.get("/").text) - self.assertIn("RTCPeerConnection", client.get("/dictation.js").text) + javascript = client.get("/dictation.js").text + self.assertIn("RTCPeerConnection", javascript) + self.assertIn("zenbu.dictation.event", javascript) + self.assertIn('publishDictation("partial"', javascript) + self.assertIn('publishDictation("final"', javascript) + self.assertIn('setStatus("Starting microphone..."', javascript) + self.assertIn("ZenbuDictationToggle", javascript) + self.assertIn("ZenbuDictationStart", javascript) + self.assertIn("ZenbuDictationStop", javascript) + self.assertIn("ZenbuDictationCommit", javascript) + self.assertIn("zenbu.dictation.set-active", javascript) + self.assertIn("zenbu.dictation.commit", javascript) + self.assertIn("prepareCanvasMicrophone", javascript) + self.assertIn("track.enabled = false", javascript) + self.assertIn("if (!stream)", javascript) def test_rejects_non_offer_sdp(self): with TestClient(create_app(test_config(), FakeTranscriber())) as client: diff -r b3b547563ec7 -r 49e9e591c9bb dictation/session.py --- a/dictation/session.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/session.py Tue Aug 18 19:14:53 2026 -0700 @@ -54,8 +54,11 @@ @channel.on("message") def on_message(message: Any) -> None: - if message == "stop": - self.stopping = True + if message in {"commit", "pause", "stop"}: + if message == "stop": + self.stopping = True + if self.partial_task and not self.partial_task.done(): + self.partial_task.cancel() event = self.segmenter.flush() if event and event.samples is not None: self._start_final(event.samples) @@ -155,6 +158,26 @@ ) ) + async def _cancel_pending_ice_transactions(self) -> None: + # aioice 0.10.2 can leave STUN retry timers armed after its datagram + # transport closes. Cancel their futures first so Transaction.run() + # clears each timer before RTCPeerConnection.close() drops sockets. + ice_transports = getattr( + self.peer, + "_RTCPeerConnection__iceTransports", + (), + ) + for ice_transport in ice_transports: + connection = getattr(ice_transport, "_connection", None) + for protocol in getattr(connection, "_protocols", ()): + for transaction in tuple( + getattr(protocol, "transactions", {}).values() + ): + future = getattr(transaction, "_Transaction__future", None) + if future is not None and not future.done(): + future.cancel() + await asyncio.sleep(0) + async def close(self) -> None: if self.closed: return @@ -174,4 +197,5 @@ await asyncio.gather(*tasks, return_exceptions=True) if self.channel and self.channel.readyState != "closed": self.channel.close() + await self._cancel_pending_ice_transactions() await self.peer.close() diff -r b3b547563ec7 -r 49e9e591c9bb dictation/transcriber.py --- a/dictation/transcriber.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/transcriber.py Tue Aug 18 19:14:53 2026 -0700 @@ -49,10 +49,10 @@ loop = asyncio.get_running_loop() self._model = await loop.run_in_executor( self._executor, - self._load, + self._load_and_warmup, ) - def _load(self): + def _load_and_warmup(self): from faster_whisper import WhisperModel if not (self._model_dir / "model.bin").is_file(): @@ -60,12 +60,25 @@ f"Model not found at {self._model_dir}. " "Run: bazel run //dictation:download_model" ) - return WhisperModel( + model = WhisperModel( str(self._model_dir), device="cuda", compute_type=self._compute_type, local_files_only=True, ) + # Loading weights does not initialize all CUDA kernels. Execute and + # consume one short silent inference now so the user's first utterance + # does not pay the one-time GPU setup cost. + segments, _ = model.transcribe( + np.zeros(8000, dtype=np.float32), + beam_size=1, + best_of=1, + condition_on_previous_text=False, + vad_filter=False, + without_timestamps=True, + ) + list(segments) + return model async def transcribe( self, @@ -89,10 +102,11 @@ ) -> Transcript: segments, info = self._model.transcribe( samples, - beam_size=5 if final else 1, - best_of=5 if final else 1, + beam_size=1, + best_of=1, condition_on_previous_text=False, vad_filter=False, + without_timestamps=True, ) text = "".join(segment.text for segment in segments).strip() return Transcript( diff -r b3b547563ec7 -r 49e9e591c9bb dictation/web/dictation.js --- a/dictation/web/dictation.js Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/web/dictation.js Tue Aug 18 19:14:53 2026 -0700 @@ -9,10 +9,26 @@ let stream = null; let sessionId = null; let stopResolver = null; +let captureActive = false; +let connectionPromise = null; +const canvasTransport = + new URLSearchParams(window.location.search).get("canvas") === "1"; + +function publishDictation(kind, text) { + const payload = { + type: "zenbu.dictation.event", + kind, + text, + }; + window.parent?.postMessage(payload, "*"); + document.title = + `Zenbu Dictation|${Date.now()}|${kind}|${encodeURIComponent(text)}`; +} function setStatus(message, state = "") { status.textContent = message; status.dataset.state = state; + publishDictation("status", message); } function waitForIceGathering(connection) { @@ -27,6 +43,24 @@ }); } +function waitForTranscriptChannel(transcriptChannel) { + if (transcriptChannel.readyState === "open") return Promise.resolve(); + return new Promise((resolve, reject) => { + const timeout = window.setTimeout( + () => reject(new Error("Transcript channel readiness timed out")), + 10000, + ); + transcriptChannel.addEventListener("open", () => { + window.clearTimeout(timeout); + resolve(); + }, { once: true }); + transcriptChannel.addEventListener("close", () => { + window.clearTimeout(timeout); + reject(new Error("Transcript channel closed during startup")); + }, { once: true }); + }); +} + function handleTranscript(message) { let event; try { @@ -37,20 +71,22 @@ } switch (event.type) { case "ready": - setStatus("Listening", "ready"); + setStatus(captureActive ? "Listening" : "Idle", "ready"); break; case "speech.started": setStatus("Speech detected", "working"); break; case "transcript.partial": partial.textContent = event.text; + publishDictation("partial", event.text); break; case "transcript.final": finalText.value += `${finalText.value ? " " : ""}${event.text}`; + publishDictation("final", event.text); partial.textContent = "Waiting for speech..."; break; case "speech.ended": - setStatus("Listening", "ready"); + setStatus(captureActive ? "Listening" : "Idle", "ready"); stopResolver?.(); stopResolver = null; break; @@ -60,11 +96,9 @@ } } -async function start() { - if (peer) return; - startButton.disabled = true; - setStatus("Requesting microphone access...", "working"); - try { +async function createConnection() { + if (peer && !["closed", "failed"].includes(peer.connectionState)) return; + if (!stream) { stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, @@ -73,6 +107,9 @@ }, video: false, }); + } + for (const track of stream.getTracks()) track.enabled = false; + try { peer = new RTCPeerConnection(); channel = peer.createDataChannel("transcripts", { ordered: true }); channel.addEventListener("message", handleTranscript); @@ -82,7 +119,7 @@ peer.addEventListener("connectionstatechange", () => { if (peer?.connectionState === "failed") { setStatus("WebRTC connection failed", "error"); - void stop(); + void stop(false, true); } }); for (const track of stream.getTracks()) peer.addTrack(track, stream); @@ -105,23 +142,50 @@ sdp: answer.sdp, type: answer.type, }); + await waitForTranscriptChannel(channel); stopButton.disabled = false; - setStatus("Connecting...", "working"); } catch (error) { - setStatus(error.message || "Unable to start dictation", "error"); - await stop(); + await stop(false, true); + throw error; } } -async function stop(waitForFinal = true) { +async function ensureConnection() { + if (peer && ["connected", "completed"].includes(peer.connectionState)) return; + if (!connectionPromise) { + connectionPromise = createConnection().finally(() => { + connectionPromise = null; + }); + } + await connectionPromise; +} + +async function start() { + captureActive = true; + startButton.disabled = true; + setStatus("Starting microphone...", "working"); + try { + await ensureConnection(); + for (const track of stream.getTracks()) track.enabled = true; + setStatus( + channel?.readyState === "open" ? "Listening" : "Connecting...", + channel?.readyState === "open" ? "ready" : "working", + ); + } catch (error) { + captureActive = false; + setStatus(error.message || "Unable to start dictation", "error"); + } +} + +async function stop(waitForFinal = true, releaseStream = !canvasTransport) { const closingSession = sessionId; - sessionId = null; const tracks = stream?.getTracks() || []; + captureActive = false; if (channel?.readyState === "open" && waitForFinal) { const ended = new Promise(resolve => { stopResolver = resolve; }); - channel.send("stop"); + channel.send(releaseStream ? "stop" : "pause"); for (const track of tracks) track.enabled = false; await Promise.race([ ended, @@ -129,8 +193,18 @@ ]); stopResolver = null; } - for (const track of tracks) track.stop(); - stream = null; + for (const track of tracks) { + track.enabled = false; + if (releaseStream) track.stop(); + } + if (!releaseStream) { + startButton.disabled = false; + stopButton.disabled = true; + setStatus("Idle"); + return; + } + sessionId = null; + if (releaseStream) stream = null; channel?.close(); channel = null; peer?.close(); @@ -148,4 +222,30 @@ startButton.addEventListener("click", () => void start()); stopButton.addEventListener("click", () => void stop()); -window.addEventListener("pagehide", () => void stop(false)); +window.ZenbuDictationStart = () => start(); +window.ZenbuDictationStop = () => stop(); +window.ZenbuDictationCommit = () => { + if (channel?.readyState === "open") channel.send("commit"); +}; +window.ZenbuDictationToggle = () => captureActive ? stop() : start(); +window.addEventListener("message", event => { + if (event.data?.type === "zenbu.dictation.set-active") { + void (event.data.active ? start() : stop()); + } else if (event.data?.type === "zenbu.dictation.commit") { + window.ZenbuDictationCommit(); + } +}); +window.addEventListener("pagehide", () => void stop(false, true)); + +async function prepareCanvasMicrophone() { + if (!canvasTransport || peer) return; + try { + await ensureConnection(); + for (const track of stream.getTracks()) track.enabled = false; + setStatus("Idle"); + } catch (error) { + setStatus(error.message || "Unable to prepare microphone", "error"); + } +} + +void prepareCanvasMicrophone(); diff -r b3b547563ec7 -r 49e9e591c9bb dictation/webrtc_test.py --- a/dictation/webrtc_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/webrtc_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -60,6 +60,16 @@ class WebRtcTest(unittest.IsolatedAsyncioTestCase): async def test_audio_track_returns_transcript_events(self): + loop_errors = [] + loop = asyncio.get_running_loop() + previous_exception_handler = loop.get_exception_handler() + loop.set_exception_handler( + lambda _loop, context: loop_errors.append(context) + ) + self.addCleanup( + loop.set_exception_handler, + previous_exception_handler, + ) service = DictationService(test_config(), FakeTranscriber()) await service.start() client = RTCPeerConnection() @@ -131,11 +141,17 @@ self.assertIn("speech.started", event_types) self.assertIn("transcript.partial", event_types) self.assertIn("transcript.final", event_types) + channel.send("commit") + await asyncio.sleep(0.2) + self.assertNotEqual(client.connectionState, "closed") + self.assertIn(answer["sessionId"], service.sessions) finally: await client.close() await service.close() + await asyncio.sleep(0.6) Path(audio_file.name).unlink(missing_ok=True) self.assertEqual(service.sessions, {}) + self.assertEqual(loop_errors, []) if __name__ == "__main__": diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/BUILD --- a/infinite_canvas/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -2,6 +2,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//gui_ze:gui_ze.bzl", "move_files_into_dir") load("//third_party/raylib:raylib.bzl", "raylib_binary") @@ -75,6 +76,28 @@ }), ) +cc_library( + name = "agent_service", + srcs = select({ + "//config:linux": ["agent_service_copilot.c"], + "//config:macos": ["agent_service_stub.c"], + "//config:windows": ["agent_service_stub.c"], + "//conditions:default": ["agent_service_stub.c"], + }), + hdrs = ["agent_service.h"], + deps = [ + ":canvas", + "//dowa:dowa", + ] + select({ + "//config:linux": ["//mrjunejune:inference_bridge"], + "//conditions:default": [], + }), + linkopts = select({ + "//config:linux": ["-lpthread"], + "//conditions:default": [], + }), +) + filegroup( name = "inter_font", srcs = ["assets/Inter-Variable.ttf"], @@ -124,6 +147,7 @@ "//conditions:default": [], }), deps = [ + ":agent_service", ":canvas", ":web_surface", "//dowa:dowa", @@ -153,6 +177,7 @@ }), defines = ["INFINITE_CANVAS_DEV_UI"], deps = [ + ":agent_service", ":canvas", ":dev_ui", ":theme", @@ -174,6 +199,31 @@ data = [":dev_bin"], ) +sh_binary( + name = "orchestration_dev", + srcs = ["run_orchestration_dev.sh"], + data = [ + ":dev", + "//dictation:server", + "//mrjunejune:canvas_orchestration_launcher", + "//mrjunejune/inference:copilot_sidecar_zip", + "//mrjunejune/inference:litellm_config.yaml", + "//mrjunejune/inference:litellm_proxy_zip", + "@copilot_cli_linux_x86_64//:copilot", + "@python_3_11//:files", + "@python_3_11//:python3", + ], + target_compatible_with = select({ + "//config:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +alias( + name = "agent_dev", + actual = ":orchestration_dev", +) + alias( name = "app", actual = select({ @@ -208,6 +258,7 @@ ], defines = ["INFINITE_CANVAS_DEV_UI"], deps = [ + ":agent_service", ":canvas", ":dev_ui", ":web_surface", @@ -245,6 +296,41 @@ deps = [":canvas"], ) +cc_test( + name = "agent_service_test", + srcs = ["agent_service_test.c"], + args = [ + "$(rootpath //mrjunejune/test:inference_bridge_fake_sidecar)", + ], + data = [ + "//mrjunejune/test:inference_bridge_fake_sidecar", + ], + deps = [ + ":agent_service", + "//dowa:dowa", + ], + linkopts = ["-lpthread"], + target_compatible_with = select({ + "//config:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +sh_test( + name = "dictation_policy_test", + srcs = ["dictation_policy_test.sh"], + args = [ + "$(rootpath :web_surface_native.cc)", + "$(rootpath :main.c)", + "$(rootpath :dev_ui.c)", + ], + data = [ + "dev_ui.c", + "main.c", + "web_surface_native.cc", + ], +) + sh_binary( name = "serve", srcs = ["serve.sh"], diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/README.md --- a/infinite_canvas/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -41,10 +41,14 @@ - Left-mouse drag on an entity: select and move it - `P`: pin or unpin the selected entity in screen space so it remains visible while the camera pans and zooms; pinned entities show a Lucide pin glyph +- `M`: show centered live dictation text and toggle microphone capture +- `Enter`: send visible dictation text to the spatial agent +- `Ctrl/Cmd+Enter`: submit the selected text area's thought to Copilot - `Delete`/`Backspace`: fade out and remove the selected entity - Development toolbar: spawn/clear entities, reset the camera, or restore the deterministic demo scene; spawned entities fade in and Clear fades all - entities out before removal + entities out before removal. Use the header arrow to collapse the toolbar + into a small restore button. - `View camera context`: open a live, scrollable snapshot of entities intersecting the camera viewport and their current semantic values - `Lucide icon gallery`: create or revisit a dedicated canvas region containing @@ -87,6 +91,66 @@ ISC-licensed vector data is generated from Lucide 1.31.0 and the visible grid rows are drawn directly with Raylib primitives. +## Spatial agent sessions + +Authenticate the Copilot CLI once: + +```bash +bazel run //mrjunejune:run_inference_stack -- --authenticate +``` + +Then start dictation, the Copilot LiteLLM gateway, and the canvas with one +target: + +```bash +bazel run //infinite_canvas:orchestration_dev +``` + +`//infinite_canvas:agent_dev` remains an alias. Closing the canvas stops the +supervised local services. + +Press `M` to start or stop the local WebRTC microphone session. The browser +transport is prewarmed and remains hidden: it acquires the native microphone +permission, device, signaling, ICE connection, and transcript data channel +ahead of the first hotkey while keeping the audio track disabled. `M` therefore +only unmutes an already-ready channel, so speech from the beginning of the +utterance is retained. Partial and final speech appear directly as minimal centered text +inside a retained `Dictation` text-area entity spawned near the camera center. +The entity word-wraps, scrolls to the latest line, and can be dragged by its +header to any position on the board. The same scratchpad is reused for every +thought and clears after submission; it does not create duplicate dictation +windows. A small pulsing `Listening` indicator +remains in the bottom-left while capture is active. In-progress speech uses +muted text; `Backspace` clears the entire current thought without stopping +capture. Pausing with `M` and resuming appends to the same thought rather than +replacing it. Press `Enter` to commit and send the current utterance while the +microphone stays live for the next thought; press `M` or `Escape` to stop it. +For typed input, select a text area and press `Ctrl/Cmd+Enter`. +Native CEF grants audio only to the fixed loopback dictation transport; it does +not show an inaccessible browser permission prompt or grant microphone access +to normal canvas web entities. + +The Copilot SDK receives the visible camera context, including existing +conversation entity IDs, through the shared `canvas_orchestrator` prompt +profile. It returns a routing decision to append to an existing session or +create a new one. Requests run off the render thread through the shared +`Inference_Bridge`. Conversation cards retain +their own title, transcript, working state, scroll position, selection, and +pinning state, forming the first rich-context container for future typed image +and component resources. Runtime configuration and Copilot token paths come +from the ignored `mrjunejune/.config`; no separate canvas environment file is +required. The Dictation scratchpad is excluded from camera context because its +text is submitted separately, and only visible conversation cards are eligible +append targets. Conversation cards follow the latest appended message and can +be collapsed from their header. The archive icon moves a session to the +off-canvas parking lot, excluding it from context; `Parking lot (N)` in +Developer Controls jumps there, and the restore icon returns the session to +its previous board position. Collapse and expansion use the same eased retained +animation model as accordion entities rather than snapping between heights. +Canvas startup also pre-creates the persistent orchestration SDK session and +three worker sessions, moving SDK session creation out of the first submitted +thought. + Tables support row selection, and notifications behave as bottom-right toast cards: multiple toasts overlap compactly, fan into a spaced stack on hover, and slide/fade away from either Undo or close. Position, size, stack expansion, diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/agent_service.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service.h Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,39 @@ +#ifndef INFINITE_CANVAS_AGENT_SERVICE_H +#define INFINITE_CANVAS_AGENT_SERVICE_H + +#include "dowa/dowa.h" +#include "infinite_canvas/canvas.h" + +#define CANVAS_AGENT_RESPONSE_CAPACITY 16384 + +typedef enum { + CANVAS_AGENT_IDLE = 0, + CANVAS_AGENT_WORKING, + CANVAS_AGENT_READY, + CANVAS_AGENT_ERROR, +} Canvas_Agent_Status; + +typedef struct { + Dowa_Arena *p_arena; + void *p_native; + Canvas_Agent_Status status; + char response[CANVAS_AGENT_RESPONSE_CAPACITY]; + char error[512]; +} Canvas_Agent_Service; + +boolean Canvas_Agent_Service_Init( + Canvas_Agent_Service *p_service, + Dowa_Arena *p_arena); +boolean Canvas_Agent_Service_Submit( + Canvas_Agent_Service *p_service, + const char *p_prompt, + const char *p_context); +Canvas_Agent_Status Canvas_Agent_Service_Poll( + Canvas_Agent_Service *p_service, + char *p_response, + size_t response_capacity, + char *p_error, + size_t error_capacity); +void Canvas_Agent_Service_Shutdown(Canvas_Agent_Service *p_service); + +#endif diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/agent_service_copilot.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service_copilot.c Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,276 @@ +#include "infinite_canvas/agent_service.h" + +#include "mrjunejune/inference_bridge.h" + +#include +#include +#include +#include +#include + +#define CANVAS_AGENT_WARM_SESSION_COUNT 4 + +typedef struct { + pthread_mutex_t mutex; + Canvas_Agent_Service *p_service; + Inference_Bridge *p_bridge; + uint32 request_sequence; + uint32 warmed_sessions; + char request_id[64]; + char turn_content[CANVAS_AGENT_RESPONSE_CAPACITY]; + char turn_error[512]; + char turn_prompt[ + CANVAS_CONTEXT_MAX_LENGTH + CANVAS_ENTITY_TEXT_CAPACITY + 512]; +} Canvas_Agent_Copilot; + +static boolean Canvas_Agent_Event_Matches( + const Canvas_Agent_Copilot *p_copilot, + const Inference_Event *p_event) +{ + return p_event->request_id && + strcmp(p_copilot->request_id, p_event->request_id) == 0; +} + +static void Canvas_Agent_Handle_Event( + const Inference_Event *p_event, + void *p_user_data) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_user_data; + if (!p_copilot || !p_event) return; + + pthread_mutex_lock(&p_copilot->mutex); + if (strcmp(p_event->type, "bridge.closed") == 0) { + if (p_copilot->p_service->status == CANVAS_AGENT_WORKING) { + snprintf( + p_copilot->p_service->error, + sizeof(p_copilot->p_service->error), + "%s", + p_event->error_message && p_event->error_message[0] + ? p_event->error_message + : "Copilot orchestration sidecar closed"); + p_copilot->p_service->status = CANVAS_AGENT_ERROR; + } + pthread_mutex_unlock(&p_copilot->mutex); + return; + } + if (strcmp(p_event->type, "session.warmed") == 0 && + p_event->request_id && + strncmp(p_event->request_id, "canvas-warm-", 12) == 0) { + p_copilot->warmed_sessions++; + pthread_mutex_unlock(&p_copilot->mutex); + return; + } + if (!Canvas_Agent_Event_Matches(p_copilot, p_event)) { + pthread_mutex_unlock(&p_copilot->mutex); + return; + } + + if (strcmp(p_event->type, "assistant.completed") == 0) { + snprintf( + p_copilot->turn_content, + sizeof(p_copilot->turn_content), + "%s", + p_event->content ? p_event->content : ""); + } else if (strcmp(p_event->type, "turn.error") == 0) { + snprintf( + p_copilot->turn_error, + sizeof(p_copilot->turn_error), + "%s", + p_event->error_message && p_event->error_message[0] + ? p_event->error_message + : "Copilot orchestration failed"); + } else if (strcmp(p_event->type, "turn.done") == 0) { + if (p_event->failed || p_copilot->turn_error[0]) { + snprintf( + p_copilot->p_service->error, + sizeof(p_copilot->p_service->error), + "%s", + p_copilot->turn_error[0] + ? p_copilot->turn_error + : "Copilot orchestration failed"); + p_copilot->p_service->status = CANVAS_AGENT_ERROR; + } else if (p_copilot->turn_content[0]) { + snprintf( + p_copilot->p_service->response, + sizeof(p_copilot->p_service->response), + "%s", + p_copilot->turn_content); + p_copilot->p_service->status = CANVAS_AGENT_READY; + } else { + snprintf( + p_copilot->p_service->error, + sizeof(p_copilot->p_service->error), + "Copilot returned an empty response"); + p_copilot->p_service->status = CANVAS_AGENT_ERROR; + } + } + pthread_mutex_unlock(&p_copilot->mutex); +} + +boolean Canvas_Agent_Service_Init( + Canvas_Agent_Service *p_service, + Dowa_Arena *p_arena) +{ + memset(p_service, 0, sizeof(*p_service)); + p_service->p_arena = p_arena; + const char *p_sidecar = + getenv("INFINITE_CANVAS_COPILOT_SIDECAR_PATH"); + const char *p_cli = + getenv("INFINITE_CANVAS_COPILOT_CLI_PATH"); + if (!p_sidecar || !p_sidecar[0] || !p_cli || !p_cli[0]) { + return FALSE; + } + + Canvas_Agent_Copilot *p_copilot = (Canvas_Agent_Copilot *) + Dowa_Arena_Allocate(p_arena, sizeof(*p_copilot)); + if (!p_copilot) return FALSE; + memset(p_copilot, 0, sizeof(*p_copilot)); + if (pthread_mutex_init(&p_copilot->mutex, NULL) != 0) { + return FALSE; + } + p_copilot->p_service = p_service; + p_copilot->p_bridge = Inference_Bridge_Create( + p_sidecar, + p_cli, + Canvas_Agent_Handle_Event, + p_copilot); + if (!p_copilot->p_bridge || + !Inference_Bridge_Start(p_copilot->p_bridge)) { + if (p_copilot->p_bridge) { + Inference_Bridge_Destroy(p_copilot->p_bridge); + } + pthread_mutex_destroy(&p_copilot->mutex); + return FALSE; + } + const char *warm_sessions[CANVAS_AGENT_WARM_SESSION_COUNT] = { + "infinite-canvas-orchestrator", + "infinite-canvas-worker-1", + "infinite-canvas-worker-2", + "infinite-canvas-worker-3", + }; + for (uint32 index = 0; + index < CANVAS_AGENT_WARM_SESSION_COUNT; + index++) { + char request_id[32]; + snprintf( + request_id, + sizeof(request_id), + "canvas-warm-%u", + index); + if (!Inference_Bridge_Warm_Conversation( + p_copilot->p_bridge, + request_id, + warm_sessions[index], + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR, + 1, + 1, + index == 0 ? TRUE : FALSE)) { + Inference_Bridge_Destroy(p_copilot->p_bridge); + pthread_mutex_destroy(&p_copilot->mutex); + return FALSE; + } + } + for (int32 attempt = 0; attempt < 2000; attempt++) { + pthread_mutex_lock(&p_copilot->mutex); + boolean ready = + p_copilot->warmed_sessions == CANVAS_AGENT_WARM_SESSION_COUNT; + pthread_mutex_unlock(&p_copilot->mutex); + if (ready) break; + usleep(10000); + } + p_service->p_native = p_copilot; + p_service->status = CANVAS_AGENT_IDLE; + return TRUE; +} + +boolean Canvas_Agent_Service_Submit( + Canvas_Agent_Service *p_service, + const char *p_prompt, + const char *p_context) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_service->p_native; + if (!p_copilot || !p_prompt || !p_prompt[0]) return FALSE; + + pthread_mutex_lock(&p_copilot->mutex); + if (p_service->status == CANVAS_AGENT_WORKING) { + pthread_mutex_unlock(&p_copilot->mutex); + return FALSE; + } + p_copilot->request_sequence++; + snprintf( + p_copilot->request_id, + sizeof(p_copilot->request_id), + "canvas-%u", + p_copilot->request_sequence); + snprintf( + p_copilot->turn_prompt, + sizeof(p_copilot->turn_prompt), + "Visible canvas context:\n%s\n\nNew user thought:\n%s", + p_context ? p_context : "", + p_prompt); + p_copilot->turn_content[0] = '\0'; + p_copilot->turn_error[0] = '\0'; + p_service->response[0] = '\0'; + p_service->error[0] = '\0'; + p_service->status = CANVAS_AGENT_WORKING; + char request_id[sizeof(p_copilot->request_id)]; + snprintf(request_id, sizeof(request_id), "%s", p_copilot->request_id); + pthread_mutex_unlock(&p_copilot->mutex); + + if (!Inference_Bridge_Start_Turn( + p_copilot->p_bridge, + request_id, + "infinite-canvas-orchestrator", + p_copilot->turn_prompt, + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR, + 1, + 1, + NULL, + 0)) { + pthread_mutex_lock(&p_copilot->mutex); + snprintf( + p_service->error, + sizeof(p_service->error), + "Unable to start the Copilot orchestration turn"); + p_service->status = CANVAS_AGENT_ERROR; + pthread_mutex_unlock(&p_copilot->mutex); + return FALSE; + } + return TRUE; +} + +Canvas_Agent_Status Canvas_Agent_Service_Poll( + Canvas_Agent_Service *p_service, + char *p_response, + size_t response_capacity, + char *p_error, + size_t error_capacity) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_service->p_native; + if (!p_copilot) return CANVAS_AGENT_ERROR; + pthread_mutex_lock(&p_copilot->mutex); + Canvas_Agent_Status status = p_service->status; + if (status == CANVAS_AGENT_READY) { + snprintf(p_response, response_capacity, "%s", p_service->response); + p_service->status = CANVAS_AGENT_IDLE; + } else if (status == CANVAS_AGENT_ERROR) { + snprintf(p_error, error_capacity, "%s", p_service->error); + p_service->status = CANVAS_AGENT_IDLE; + } + pthread_mutex_unlock(&p_copilot->mutex); + return status; +} + +void Canvas_Agent_Service_Shutdown(Canvas_Agent_Service *p_service) +{ + Canvas_Agent_Copilot *p_copilot = + (Canvas_Agent_Copilot *)p_service->p_native; + if (!p_copilot) return; + Inference_Bridge_Destroy(p_copilot->p_bridge); + p_copilot->p_bridge = NULL; + pthread_mutex_destroy(&p_copilot->mutex); + p_service->p_native = NULL; +} diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/agent_service_stub.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service_stub.c Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,51 @@ +#include "infinite_canvas/agent_service.h" + +#include +#include + +boolean Canvas_Agent_Service_Init( + Canvas_Agent_Service *p_service, + Dowa_Arena *p_arena) +{ + memset(p_service, 0, sizeof(*p_service)); + p_service->p_arena = p_arena; + p_service->status = CANVAS_AGENT_IDLE; + return TRUE; +} + +boolean Canvas_Agent_Service_Submit( + Canvas_Agent_Service *p_service, + const char *p_prompt, + const char *p_context) +{ + (void)p_prompt; + (void)p_context; + snprintf( + p_service->error, + sizeof(p_service->error), + "Copilot SDK orchestration is not available on this platform yet"); + p_service->status = CANVAS_AGENT_ERROR; + return TRUE; +} + +Canvas_Agent_Status Canvas_Agent_Service_Poll( + Canvas_Agent_Service *p_service, + char *p_response, + size_t response_capacity, + char *p_error, + size_t error_capacity) +{ + (void)p_response; + (void)response_capacity; + Canvas_Agent_Status status = p_service->status; + if (status == CANVAS_AGENT_ERROR) { + snprintf(p_error, error_capacity, "%s", p_service->error); + p_service->status = CANVAS_AGENT_IDLE; + } + return status; +} + +void Canvas_Agent_Service_Shutdown(Canvas_Agent_Service *p_service) +{ + (void)p_service; +} diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/agent_service_test.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/agent_service_test.c Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,49 @@ +#include "infinite_canvas/agent_service.h" + +#include +#include +#include +#include + +int main(int argc, char **p_argv) +{ + assert(argc == 2); + assert(setenv( + "INFINITE_CANVAS_COPILOT_SIDECAR_PATH", + p_argv[1], + 1) == 0); + assert(setenv( + "INFINITE_CANVAS_COPILOT_CLI_PATH", + p_argv[1], + 1) == 0); + + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + Canvas_Agent_Service service; + assert(Canvas_Agent_Service_Init(&service, p_arena)); + assert(Canvas_Agent_Service_Submit( + &service, + "Create a rendering session", + "entity id=42 type=Agent conversation")); + + Canvas_Agent_Status status = CANVAS_AGENT_WORKING; + char response[CANVAS_AGENT_RESPONSE_CAPACITY] = {0}; + char error[512] = {0}; + for (int32 attempt = 0; + attempt < 200 && status == CANVAS_AGENT_WORKING; + attempt++) { + usleep(10000); + status = Canvas_Agent_Service_Poll( + &service, + response, + sizeof(response), + error, + sizeof(error)); + } + assert(status == CANVAS_AGENT_READY); + assert(strstr(response, "hello traveler")); + + Canvas_Agent_Service_Shutdown(&service); + Dowa_Arena_Free(p_arena); + return 0; +} diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/canvas.c --- a/infinite_canvas/canvas.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/canvas.c Tue Aug 18 19:14:53 2026 -0700 @@ -38,6 +38,10 @@ #define CANVAS_TEXT_AREA_FONT_SIZE 16.0f #define CANVAS_TEXT_AREA_SPACING 0.1f #define CANVAS_TEXT_AREA_LINE_HEIGHT 22.0f +#define CANVAS_CONVERSATION_EXPANDED_HEIGHT 300.0f +#define CANVAS_CONVERSATION_COLLAPSED_HEIGHT 58.0f +#define CANVAS_PARKING_LOT_X 2400.0f +#define CANVAS_PARKING_LOT_Y -600.0f typedef struct { int32 start; @@ -46,6 +50,13 @@ float width; } Canvas_Text_Line; +static int32 Canvas_Text_Build_Lines( + Font font, + const char *p_text, + float max_width, + Canvas_Text_Line *p_lines, + int32 line_capacity); + static float Canvas_Clamp(float value, float min_value, float max_value) { if (value < min_value) return min_value; @@ -743,6 +754,28 @@ }; } +static Rectangle Canvas_Conversation_Collapse_Bounds( + const Canvas_Entity *p_entity) +{ + return (Rectangle){ + p_entity->position.x + p_entity->size.x - 74.0f, + p_entity->position.y + 12.0f, + 28.0f, + 28.0f, + }; +} + +static Rectangle Canvas_Conversation_Park_Bounds( + const Canvas_Entity *p_entity) +{ + return (Rectangle){ + p_entity->position.x + p_entity->size.x - 40.0f, + p_entity->position.y + 12.0f, + 28.0f, + 28.0f, + }; +} + static Rectangle Canvas_Entity_Bounds(const Canvas_Entity *p_entity) { if (p_entity->type == CANVAS_ENTITY_CIRCLE) { @@ -753,6 +786,7 @@ p_entity->size.x * 2.0f, }; } + if (p_entity->type == CANVAS_ENTITY_LINE) { float end_x = p_entity->position.x + p_entity->size.x; float end_y = p_entity->position.y + p_entity->size.y; @@ -949,6 +983,16 @@ Canvas_Context_Append(p_writer, "url="); Canvas_Context_Append_Quoted(p_writer, p_entity->text); break; + case CANVAS_ENTITY_CONVERSATION: + Canvas_Context_Append(p_writer, "title="); + Canvas_Context_Append_Quoted(p_writer, p_entity->label); + Canvas_Context_Append(p_writer, " transcript="); + Canvas_Context_Append_Quoted(p_writer, p_entity->text); + Canvas_Context_Append( + p_writer, + " collapsed=%s", + p_entity->active ? "true" : "false"); + break; case CANVAS_ENTITY_LUCIDE_GALLERY: Canvas_Context_Append( p_writer, @@ -997,6 +1041,14 @@ p_entity->animation_amount <= 0.01f) { continue; } + if (p_entity->type == CANVAS_ENTITY_TEXT_AREA && + strcmp(p_entity->label, "Dictation") == 0) { + continue; + } + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + p_entity->parked) { + continue; + } if (!CheckCollisionRecs(viewport, Canvas_Entity_Bounds(p_entity))) continue; snapshot.visible_count++; @@ -1124,6 +1176,14 @@ entity.active = TRUE; snprintf(entity.text, sizeof(entity.text), "https://mrjunejune.com"); } + if (type == CANVAS_ENTITY_CONVERSATION) { + entity.size = (Vector2){ + 320.0f, + CANVAS_CONVERSATION_EXPANDED_HEIGHT, + }; + entity.animation_amount = 1.0f; + snprintf(entity.label, sizeof(entity.label), "Agent session"); + } if (type == CANVAS_ENTITY_LUCIDE_GALLERY) { entity.size = (Vector2){1440.0f, 1040.0f}; } @@ -1132,6 +1192,188 @@ return TRUE; } +uint32 Canvas_Scene_Add_Conversation( + Canvas_Scene *p_scene, + const char *p_title, + const char *p_prompt, + const char *p_response, + Vector2 position) +{ + if (!Canvas_Scene_Add( + p_scene, + CANVAS_ENTITY_CONVERSATION, + position)) { + return 0; + } + Canvas_Entity *p_entity = + &p_scene->p_entities[Dowa_Array_Length(p_scene->p_entities) - 1]; + p_entity->size = (Vector2){420.0f, 300.0f}; + snprintf( + p_entity->label, + sizeof(p_entity->label), + "%s", + p_title && p_title[0] ? p_title : "New agent session"); + snprintf( + p_entity->text, + sizeof(p_entity->text), + "You\n%s\n\nCopilot\n%s", + p_prompt, + p_response); + p_scene->selected_index = + (int32)Dowa_Array_Length(p_scene->p_entities) - 1; + return p_entity->id; +} + +boolean Canvas_Scene_Append_Conversation( + Canvas_Scene *p_scene, + uint32 entity_id, + const char *p_prompt, + const char *p_response) +{ + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + Canvas_Entity *p_entity = &p_scene->p_entities[index]; + if (p_entity->id != entity_id || + p_entity->type != CANVAS_ENTITY_CONVERSATION) { + continue; + } + size_t length = strlen(p_entity->text); + snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "\n\nYou\n%s\n\nCopilot\n%s", + p_prompt, + p_response); + p_scene->selected_index = (int32)index; + return TRUE; + } + return FALSE; +} + +static float Canvas_Conversation_Max_Scroll( + const Canvas_Entity *p_entity, + Font font) +{ + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 line_count = Canvas_Text_Build_Lines( + font, + p_entity->text, + p_entity->size.x - 32.0f, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + return fmaxf( + 0.0f, + (float)line_count * 18.0f - + (CANVAS_CONVERSATION_EXPANDED_HEIGHT - 82.0f)); +} + +void Canvas_Conversation_Scroll_To_End( + Canvas_Entity *p_entity, + Font font) +{ + if (!p_entity || p_entity->type != CANVAS_ENTITY_CONVERSATION) return; + p_entity->value = (int32)Canvas_Conversation_Max_Scroll(p_entity, font); +} + +void Canvas_Conversation_Set_Collapsed( + Canvas_Entity *p_entity, + boolean collapsed) +{ + if (!p_entity || p_entity->type != CANVAS_ENTITY_CONVERSATION) return; + p_entity->active = collapsed; +} + +size_t Canvas_Scene_Parked_Conversation_Count( + const Canvas_Scene *p_scene) +{ + size_t count = 0; + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + const Canvas_Entity *p_entity = &p_scene->p_entities[index]; + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + p_entity->parked) { + count++; + } + } + return count; +} + +boolean Canvas_Scene_Set_Conversation_Parked( + Canvas_Scene *p_scene, + uint32 entity_id, + boolean parked) +{ + Canvas_Entity *p_entity = NULL; + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + if (p_scene->p_entities[index].id == entity_id && + p_scene->p_entities[index].type == CANVAS_ENTITY_CONVERSATION) { + p_entity = &p_scene->p_entities[index]; + break; + } + } + if (!p_entity || p_entity->parked == parked) return FALSE; + if (parked) { + size_t slot = Canvas_Scene_Parked_Conversation_Count(p_scene); + p_entity->parked_restore_position = p_entity->position; + p_entity->position = (Vector2){ + CANVAS_PARKING_LOT_X + (float)(slot % 3) * 460.0f, + CANVAS_PARKING_LOT_Y + (float)(slot / 3) * 340.0f, + }; + p_entity->parked = TRUE; + p_entity->pinned = FALSE; + } else { + p_entity->position = p_entity->parked_restore_position; + p_entity->parked = FALSE; + } + return TRUE; +} + +boolean Canvas_Scene_Show_Parking_Lot( + Canvas_Scene *p_scene, + Canvas_Camera *p_camera) +{ + Rectangle bounds = {0}; + boolean found = FALSE; + for (size_t index = 0; + index < Dowa_Array_Length(p_scene->p_entities); + index++) { + Canvas_Entity *p_entity = &p_scene->p_entities[index]; + if (p_entity->type != CANVAS_ENTITY_CONVERSATION || + !p_entity->parked) { + continue; + } + Rectangle entity_bounds = Canvas_Entity_Bounds(p_entity); + if (!found) { + bounds = entity_bounds; + found = TRUE; + } else { + float right = fmaxf( + bounds.x + bounds.width, + entity_bounds.x + entity_bounds.width); + float bottom = fmaxf( + bounds.y + bounds.height, + entity_bounds.y + entity_bounds.height); + bounds.x = fminf(bounds.x, entity_bounds.x); + bounds.y = fminf(bounds.y, entity_bounds.y); + bounds.width = right - bounds.x; + bounds.height = bottom - bounds.y; + } + } + if (!found) return FALSE; + Canvas_Camera_Fit_Bounds( + p_camera, + bounds, + 80.0f, + 80.0f, + 80.0f, + 80.0f); + return TRUE; +} + void Canvas_Scene_Clear(Canvas_Scene *p_scene) { Dowa_Array_Clear(p_scene->p_entities); @@ -1205,6 +1447,18 @@ Canvas_Scene_Add(p_scene, CANVAS_ENTITY_IMAGE, (Vector2){-550.0f, 220.0f}); Canvas_Scene_Add(p_scene, CANVAS_ENTITY_WEB_CONTENT, (Vector2){-150.0f, 120.0f}); + Canvas_Scene_Add(p_scene, CANVAS_ENTITY_CONVERSATION, (Vector2){610.0f, 220.0f}); + Canvas_Entity *p_conversation = + &p_scene->p_entities[Dowa_Array_Length(p_scene->p_entities) - 1]; + snprintf( + p_conversation->label, + sizeof(p_conversation->label), + "Spatial agent sessions"); + snprintf( + p_conversation->text, + sizeof(p_conversation->text), + "System\nPress M to dictate and Enter to send, or select the text " + "area and press Ctrl/Cmd+Enter to ask Copilot."); } void Canvas_Scene_Add_Browser_Grid(Canvas_Scene *p_scene) @@ -1322,7 +1576,8 @@ case CANVAS_ENTITY_SWITCH: case CANVAS_ENTITY_TABLE: case CANVAS_ENTITY_SCROLL_AREA: - case CANVAS_ENTITY_IMAGE: { + case CANVAS_ENTITY_IMAGE: + case CANVAS_ENTITY_CONVERSATION: { float height = p_entity->size.y; if (p_entity->type == CANVAS_ENTITY_DROPDOWN && p_entity->active) { height += 8.0f + 3.0f * 40.0f; @@ -1613,11 +1868,12 @@ static Rectangle Canvas_Text_Area_Content_Bounds( const Canvas_Entity *p_entity) { + float header_height = p_entity->label[0] ? 26.0f : 0.0f; return (Rectangle){ p_entity->position.x + CANVAS_TEXT_AREA_PADDING, - p_entity->position.y + CANVAS_TEXT_AREA_PADDING, + p_entity->position.y + CANVAS_TEXT_AREA_PADDING + header_height, p_entity->size.x - CANVAS_TEXT_AREA_PADDING * 2.0f, - p_entity->size.y - CANVAS_TEXT_AREA_PADDING * 2.0f, + p_entity->size.y - CANVAS_TEXT_AREA_PADDING * 2.0f - header_height, }; } @@ -1798,6 +2054,7 @@ p_entity->text_scroll_y + content.height) { p_entity->text_scroll_y = cursor_bottom - content.height; } + float max_scroll = fmaxf( 0.0f, (float)line_count * CANVAS_TEXT_AREA_LINE_HEIGHT - content.height); @@ -1807,6 +2064,31 @@ max_scroll); } +void Canvas_Text_Area_Set_Content( + Canvas_Entity *p_entity, + Font font, + const char *p_text, + boolean muted) +{ + if (!p_entity || p_entity->type != CANVAS_ENTITY_TEXT_AREA) return; + snprintf(p_entity->text, sizeof(p_entity->text), "%s", p_text ? p_text : ""); + p_entity->text_cursor = (int32)strlen(p_entity->text); + p_entity->text_selection_anchor = p_entity->text_cursor; + p_entity->text_muted = muted; + + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + Rectangle content = Canvas_Text_Area_Content_Bounds(p_entity); + int32 line_count = Canvas_Text_Build_Lines( + font, + p_entity->text, + content.width, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + p_entity->text_scroll_y = fmaxf( + 0.0f, + (float)line_count * CANVAS_TEXT_AREA_LINE_HEIGHT - content.height); +} + static void Canvas_Text_Update_Keyboard(Canvas_Entity *p_entity, Font font) { boolean shortcut = Canvas_Control_Is_Down(); @@ -2130,11 +2412,14 @@ for (size_t index = 0; index < Dowa_Array_Length(p_scene->p_entities); index++) { Canvas_Entity *p_entity = &p_scene->p_entities[index]; if (p_entity->type != CANVAS_ENTITY_ACCORDION && - p_entity->type != CANVAS_ENTITY_SWITCH) { + p_entity->type != CANVAS_ENTITY_SWITCH && + p_entity->type != CANVAS_ENTITY_CONVERSATION) { continue; } - float target = p_entity->active ? 1.0f : 0.0f; + float target = p_entity->type == CANVAS_ENTITY_CONVERSATION + ? (p_entity->active ? 0.0f : 1.0f) + : (p_entity->active ? 1.0f : 0.0f); float speed = p_entity->type == CANVAS_ENTITY_SWITCH ? 7.0f : 4.5f; float step = GetFrameTime() * speed; if (p_entity->animation_amount < target) { @@ -2149,6 +2434,21 @@ float amount = p_entity->animation_amount; float eased = amount * amount * (3.0f - 2.0f * amount); if (p_entity->type == CANVAS_ENTITY_SWITCH) continue; + if (p_entity->type == CANVAS_ENTITY_CONVERSATION) { + float height = CANVAS_CONVERSATION_COLLAPSED_HEIGHT + + (CANVAS_CONVERSATION_EXPANDED_HEIGHT - + CANVAS_CONVERSATION_COLLAPSED_HEIGHT) * eased; + if (p_entity->pinned) { + float pinned_scale = + p_entity->pinned_screen_size.x / p_entity->size.x; + p_entity->pinned_screen_size.y = height * pinned_scale; + p_entity->size.y = + p_entity->pinned_screen_size.y / p_camera->zoom; + } else { + p_entity->size.y = height; + } + continue; + } if (p_entity->pinned) { float pinned_scale = p_entity->pinned_screen_size.x / 300.0f; p_entity->pinned_screen_size.y = @@ -2217,6 +2517,27 @@ 240.0f); consumed_scroll = TRUE; } + } else if (p_hovered->type == CANVAS_ENTITY_CONVERSATION && + !control_down) { + float wheel = GetMouseWheelMove(); + if (wheel != 0.0f) { + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 line_count = Canvas_Text_Build_Lines( + font, + p_hovered->text, + p_hovered->size.x - 32.0f, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + float max_scroll = fmaxf( + 0.0f, + (float)line_count * 18.0f - + (p_hovered->size.y - 82.0f)); + p_hovered->value = (int32)Canvas_Clamp( + (float)p_hovered->value - wheel * 36.0f, + 0.0f, + max_scroll); + consumed_scroll = TRUE; + } } else if (p_hovered->type == CANVAS_ENTITY_LUCIDE_GALLERY && !control_down) { float wheel = GetMouseWheelMove(); @@ -2256,6 +2577,27 @@ if (p_scene->selected_index >= 0) { Canvas_Entity *p_entity = &p_scene->p_entities[p_scene->selected_index]; + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + CheckCollisionPointRec( + world_pointer, + Canvas_Conversation_Park_Bounds(p_entity))) { + Canvas_Scene_Set_Conversation_Parked( + p_scene, + p_entity->id, + p_entity->parked ? FALSE : TRUE); + p_scene->pressed_index = -1; + return TRUE; + } + if (p_entity->type == CANVAS_ENTITY_CONVERSATION && + CheckCollisionPointRec( + world_pointer, + Canvas_Conversation_Collapse_Bounds(p_entity))) { + Canvas_Conversation_Set_Collapsed( + p_entity, + p_entity->active ? FALSE : TRUE); + p_scene->pressed_index = -1; + return TRUE; + } if (p_entity->type == CANVAS_ENTITY_LUCIDE_GALLERY) { p_entity->active = CheckCollisionPointRec( world_pointer, @@ -2676,7 +3018,8 @@ case CANVAS_ENTITY_TABLE: case CANVAS_ENTITY_NOTIFICATION: case CANVAS_ENTITY_SCROLL_AREA: - case CANVAS_ENTITY_IMAGE: { + case CANVAS_ENTITY_IMAGE: + case CANVAS_ENTITY_CONVERSATION: { DrawRectangleRoundedLinesEx( (Rectangle){ p_entity->position.x - padding, @@ -2751,13 +3094,23 @@ { Rectangle content = Canvas_Text_Area_Content_Bounds(p_entity); if (!p_entity->text[0]) { + boolean dictation = strcmp(p_entity->label, "Dictation") == 0; DrawTextEx( font, - "Write something...", + dictation ? "Speak now..." : "Type a thought for Copilot...", (Vector2){content.x, content.y}, CANVAS_TEXT_AREA_FONT_SIZE, CANVAS_TEXT_AREA_SPACING, p_theme->text_muted); + DrawTextEx( + font, + dictation + ? "Enter to send | Backspace to clear" + : "Ctrl/Cmd+Enter to send", + (Vector2){content.x, content.y + 22.0f}, + 12.0f, + 0.0f, + Canvas_Color_Fade(p_theme->text_muted, 0.72f)); if (p_entity->active && fmod(GetTime(), 1.0) < 0.55) { DrawLineEx( (Vector2){content.x, content.y}, @@ -2840,7 +3193,7 @@ (Vector2){content.x, y}, CANVAS_TEXT_AREA_FONT_SIZE, CANVAS_TEXT_AREA_SPACING, - p_theme->text); + p_entity->text_muted ? p_theme->text_muted : p_theme->text); if (p_entity->active && line == cursor_line && @@ -3012,6 +3365,29 @@ 14, (p_entity->active ? 2.0f : 1.0f) / zoom, p_entity->active ? p_theme->accent : p_theme->border); + if (p_entity->label[0]) { + DrawTextEx( + font, + p_entity->label, + (Vector2){ + bounds.x + CANVAS_TEXT_AREA_PADDING, + bounds.y + 11.0f, + }, + 12.0f, + 0.0f, + p_theme->text_muted); + DrawLineEx( + (Vector2){ + bounds.x + bounds.width - 38.0f, + bounds.y + 17.0f, + }, + (Vector2){ + bounds.x + bounds.width - 16.0f, + bounds.y + 17.0f, + }, + 2.0f / zoom, + p_theme->border); + } Canvas_Draw_Text_Area_Content(p_entity, font, p_theme); break; } @@ -3505,6 +3881,122 @@ #endif break; } + case CANVAS_ENTITY_CONVERSATION: { + Rectangle bounds = { + p_entity->position.x, + p_entity->position.y, + p_entity->size.x, + p_entity->size.y, + }; + Canvas_Theme_Draw_Shadow( + p_theme, + bounds, + 0.06f, + 14, + zoom, + 1.0f); + DrawRectangleRounded(bounds, 0.06f, 14, p_theme->surface); + DrawRectangleRoundedLinesEx( + bounds, + 0.06f, + 14, + 1.0f / zoom, + p_theme->border); + DrawRectangleRounded( + (Rectangle){bounds.x, bounds.y, bounds.width, 54.0f}, + 0.06f, + 14, + p_theme->surface_muted); + Canvas_Lucide_Draw_Icon( + Canvas_Lucide_Find_Icon( + p_entity->agent_working ? "sparkles" : "messages-square"), + (Vector2){bounds.x + 16.0f, bounds.y + 15.0f}, + 0.82f, + 1.8f, + p_entity->agent_working ? + p_theme->accent : + p_theme->text); + DrawTextEx( + font, + p_entity->label, + (Vector2){bounds.x + 48.0f, bounds.y + 12.0f}, + 17.0f, + 0.0f, + p_theme->text); + DrawTextEx( + font, + p_entity->parked + ? "Parked session" + : (p_entity->agent_working + ? "Thinking..." + : "Agent session"), + (Vector2){bounds.x + 48.0f, bounds.y + 33.0f}, + 11.0f, + 0.0f, + p_theme->text_muted); + Rectangle collapse = + Canvas_Conversation_Collapse_Bounds(p_entity); + Rectangle park = Canvas_Conversation_Park_Bounds(p_entity); + Canvas_Lucide_Draw_Icon( + Canvas_Lucide_Find_Icon( + p_entity->active ? "chevron-down" : "chevron-up"), + (Vector2){collapse.x + 5.0f, collapse.y + 5.0f}, + 0.68f, + 1.6f, + p_theme->text_muted); + Canvas_Lucide_Draw_Icon( + Canvas_Lucide_Find_Icon( + p_entity->parked ? "archive-restore" : "archive"), + (Vector2){park.x + 5.0f, park.y + 5.0f}, + 0.68f, + 1.6f, + p_theme->text_muted); + + if (p_entity->size.y <= 82.0f) break; + Rectangle content = { + bounds.x + 16.0f, + bounds.y + 68.0f, + bounds.width - 32.0f, + bounds.height - 82.0f, + }; + Canvas_Text_Line lines[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 line_count = Canvas_Text_Build_Lines( + font, + p_entity->text, + content.width, + lines, + CANVAS_ENTITY_TEXT_CAPACITY); + for (int32 line_index = 0; + line_index < line_count; + line_index++) { + float y = content.y + (float)line_index * 18.0f - + (float)p_entity->value; + if (y + 18.0f < content.y || + y > content.y + content.height) { + continue; + } + char line_text[CANVAS_ENTITY_TEXT_CAPACITY]; + int32 length = + lines[line_index].end - lines[line_index].start; + memcpy( + line_text, + p_entity->text + lines[line_index].start, + (size_t)length); + line_text[length] = '\0'; + boolean speaker = + strcmp(line_text, "You") == 0 || + strcmp(line_text, "Copilot") == 0 || + strcmp(line_text, "System") == 0; + DrawTextEx( + font, + line_text, + (Vector2){content.x, y}, + 14.0f, + 0.0f, + speaker ? p_theme->accent : p_theme->text); + } + break; + } case CANVAS_ENTITY_LUCIDE_GALLERY: { Rectangle bounds = { p_entity->position.x, @@ -4046,6 +4538,7 @@ case CANVAS_ENTITY_SCROLL_AREA: return "Scrollable area"; case CANVAS_ENTITY_IMAGE: return "Image"; case CANVAS_ENTITY_WEB_CONTENT: return "Web content"; + case CANVAS_ENTITY_CONVERSATION: return "Agent conversation"; case CANVAS_ENTITY_LUCIDE_GALLERY: return "Lucide gallery"; default: return "Unknown"; } diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/canvas.h --- a/infinite_canvas/canvas.h Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/canvas.h Tue Aug 18 19:14:53 2026 -0700 @@ -31,6 +31,7 @@ CANVAS_ENTITY_SCROLL_AREA, CANVAS_ENTITY_IMAGE, CANVAS_ENTITY_WEB_CONTENT, + CANVAS_ENTITY_CONVERSATION, CANVAS_ENTITY_LUCIDE_GALLERY, CANVAS_ENTITY_TYPE_COUNT, } Canvas_Entity_Type; @@ -41,6 +42,7 @@ Vector2 position; Vector2 size; Color color; + char label[128]; char text[CANVAS_ENTITY_TEXT_CAPACITY]; int32 value; int32 text_cursor; @@ -50,12 +52,16 @@ boolean active; boolean pinned; boolean removing; + boolean agent_working; + boolean text_muted; + boolean parked; boolean screen_space_initialized; float hover_amount; float animation_amount; float web_chrome_amount; Vector2 pinned_screen_position; Vector2 pinned_screen_size; + Vector2 parked_restore_position; } Canvas_Entity; typedef struct { @@ -133,6 +139,37 @@ void Canvas_Scene_Init(Canvas_Scene *p_scene, Dowa_Arena *p_arena); boolean Canvas_Scene_Add(Canvas_Scene *p_scene, Canvas_Entity_Type type, Vector2 position); +void Canvas_Text_Area_Set_Content( + Canvas_Entity *p_entity, + Font font, + const char *p_text, + boolean muted); +uint32 Canvas_Scene_Add_Conversation( + Canvas_Scene *p_scene, + const char *p_title, + const char *p_prompt, + const char *p_response, + Vector2 position); +boolean Canvas_Scene_Append_Conversation( + Canvas_Scene *p_scene, + uint32 entity_id, + const char *p_prompt, + const char *p_response); +void Canvas_Conversation_Scroll_To_End( + Canvas_Entity *p_entity, + Font font); +void Canvas_Conversation_Set_Collapsed( + Canvas_Entity *p_entity, + boolean collapsed); +size_t Canvas_Scene_Parked_Conversation_Count( + const Canvas_Scene *p_scene); +boolean Canvas_Scene_Set_Conversation_Parked( + Canvas_Scene *p_scene, + uint32 entity_id, + boolean parked); +boolean Canvas_Scene_Show_Parking_Lot( + Canvas_Scene *p_scene, + Canvas_Camera *p_camera); void Canvas_Scene_Clear(Canvas_Scene *p_scene); void Canvas_Scene_Fade_Out_All(Canvas_Scene *p_scene); void Canvas_Scene_Fade_Out_Selected(Canvas_Scene *p_scene); diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/canvas_test.c --- a/infinite_canvas/canvas_test.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/canvas_test.c Tue Aug 18 19:14:53 2026 -0700 @@ -526,6 +526,133 @@ Dowa_Arena_Free(p_arena); } +static void Test_Dictation_Scratchpad_Is_Not_A_Routing_Target(void) +{ + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + + Canvas_Camera camera; + Canvas_Camera_Init(&camera, 800, 600); + Canvas_Scene scene; + Canvas_Scene_Init(&scene, p_arena); + assert(Canvas_Scene_Add( + &scene, + CANVAS_ENTITY_TEXT_AREA, + (Vector2){-260.0f, -120.0f})); + snprintf( + scene.p_entities[0].label, + sizeof(scene.p_entities[0].label), + "Dictation"); + snprintf( + scene.p_entities[0].text, + sizeof(scene.p_entities[0].text), + "Route this thought"); + uint32 conversation_id = Canvas_Scene_Add_Conversation( + &scene, + "Visible session", + "Earlier thought", + "Earlier response", + (Vector2){40.0f, -120.0f}); + assert(conversation_id != 0); + + char context[4096]; + Canvas_Context_Snapshot snapshot = Canvas_Scene_Build_Visible_Context( + &scene, + &camera, + context, + sizeof(context)); + assert(snapshot.visible_count == 1); + assert(!strstr(context, "Route this thought")); + assert(strstr(context, "Visible session")); + assert(strstr(context, TextFormat("id=%u", conversation_id))); + + Dowa_Arena_Free(p_arena); +} + +static void Test_Conversation_Create_And_Append(void) +{ + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + Canvas_Scene scene; + Canvas_Scene_Init(&scene, p_arena); + + uint32 id = Canvas_Scene_Add_Conversation( + &scene, + "Rendering architecture", + "How should images render?", + "Use a rich entity with typed resources.", + (Vector2){10.0f, 20.0f}); + assert(id != 0); + assert(Dowa_Array_Length(scene.p_entities) == 1); + assert(scene.p_entities[0].type == CANVAS_ENTITY_CONVERSATION); + assert(strcmp(scene.p_entities[0].label, "Rendering architecture") == 0); + assert(strstr(scene.p_entities[0].text, "How should images render?")); + + assert(Canvas_Scene_Append_Conversation( + &scene, + id, + "What about browser content?", + "Keep the resource typed and composited by its backend.")); + assert(strstr(scene.p_entities[0].text, "What about browser content?")); + assert(strstr(scene.p_entities[0].text, "composited by its backend")); + assert(!Canvas_Scene_Append_Conversation( + &scene, + id + 1, + "Missing", + "Missing")); + Dowa_Arena_Free(p_arena); +} + +static void Test_Conversation_Collapse_And_Parking(void) +{ + Dowa_Arena *p_arena = Dowa_Arena_Create(ONE_MEGA_BYTE); + assert(p_arena); + Canvas_Camera camera; + Canvas_Camera_Init(&camera, 800, 600); + Canvas_Scene scene; + Canvas_Scene_Init(&scene, p_arena); + + Vector2 original = {20.0f, 30.0f}; + uint32 id = Canvas_Scene_Add_Conversation( + &scene, + "Parkable session", + "Question", + "Answer", + original); + Canvas_Entity *p_conversation = &scene.p_entities[0]; + assert(Near(p_conversation->animation_amount, 1.0f)); + Canvas_Conversation_Set_Collapsed(p_conversation, TRUE); + assert(p_conversation->active); + Canvas_Conversation_Set_Collapsed(p_conversation, FALSE); + assert(!p_conversation->active); + + assert(Canvas_Scene_Set_Conversation_Parked(&scene, id, TRUE)); + assert(p_conversation->parked); + assert(Canvas_Scene_Parked_Conversation_Count(&scene) == 1); + assert(!Near(p_conversation->position.x, original.x)); + char context[4096]; + Canvas_Context_Snapshot snapshot = Canvas_Scene_Build_Visible_Context( + &scene, + &camera, + context, + sizeof(context)); + assert(snapshot.visible_count == 0); + assert(Canvas_Scene_Show_Parking_Lot(&scene, &camera)); + snapshot = Canvas_Scene_Build_Visible_Context( + &scene, + &camera, + context, + sizeof(context)); + assert(snapshot.visible_count == 0); + + assert(Canvas_Scene_Set_Conversation_Parked(&scene, id, FALSE)); + assert(!p_conversation->parked); + assert(Canvas_Scene_Parked_Conversation_Count(&scene) == 0); + assert(Near(p_conversation->position.x, original.x)); + assert(Near(p_conversation->position.y, original.y)); + Dowa_Arena_Free(p_arena); +} + int main(void) { Test_Theme_Resolution(); @@ -543,8 +670,11 @@ Test_Lucide_Gallery_Reuses_Entity_And_Fits_Camera(); Test_Lucide_Search_Is_Case_Insensitive(); Test_Demo_Contains_Every_Entity_Type(); + Test_Conversation_Create_And_Append(); Test_Pinned_Entity_Stays_In_Screen_Space(); Test_Visible_Context_Tracks_Camera_And_Values(); + Test_Dictation_Scratchpad_Is_Not_A_Routing_Target(); + Test_Conversation_Collapse_And_Parking(); printf("Infinite canvas tests passed.\n"); return 0; } diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/dev_ui.c --- a/infinite_canvas/dev_ui.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/dev_ui.c Tue Aug 18 19:14:53 2026 -0700 @@ -5,7 +5,8 @@ #define RAYGUI_IMPLEMENTATION #include "third_party/raylib/include/raygui.h" -static const Rectangle DEV_PANEL = {18.0f, 18.0f, 264.0f, 478.0f}; +static const Rectangle DEV_PANEL = {18.0f, 18.0f, 264.0f, 520.0f}; +static const Rectangle DEV_PANEL_COLLAPSED = {18.0f, 18.0f, 44.0f, 44.0f}; static Rectangle Canvas_Dev_UI_Context_Bounds(void) { @@ -206,6 +207,7 @@ { *p_ui = (Canvas_Dev_UI){ .selected_type = CANVAS_ENTITY_RECTANGLE, + .collapsed = FALSE, .dropdown_open = FALSE, .context_open = FALSE, }; @@ -219,7 +221,11 @@ if (p_ui->dropdown_open) return TRUE; Vector2 pointer = GetMousePosition(); - if (CheckCollisionPointRec(pointer, DEV_PANEL)) return TRUE; + if (CheckCollisionPointRec( + pointer, + p_ui->collapsed ? DEV_PANEL_COLLAPSED : DEV_PANEL)) { + return TRUE; + } if (p_ui->context_open && CheckCollisionPointRec(pointer, Canvas_Dev_UI_Context_Bounds())) { return TRUE; @@ -234,6 +240,35 @@ Font font, Canvas_Theme *p_theme) { + if (p_ui->collapsed) { + Canvas_Theme_Draw_Shadow( + p_theme, + DEV_PANEL_COLLAPSED, + 0.24f, + 12, + 1.0f, + 1.0f); + DrawRectangleRounded( + DEV_PANEL_COLLAPSED, + 0.24f, + 12, + p_theme->surface); + DrawRectangleRoundedLinesEx( + DEV_PANEL_COLLAPSED, + 0.24f, + 12, + 1.0f, + p_theme->border); + if (GuiButtonRounded( + (Rectangle){24.0f, 24.0f, 32.0f, 32.0f}, + ">", + 0.30f, + 10)) { + p_ui->collapsed = FALSE; + } + return; + } + Canvas_Context_Snapshot context = Canvas_Scene_Build_Visible_Context( p_scene, p_camera, @@ -256,6 +291,16 @@ 12.0f, 0.1f, p_theme->text_muted); + if (GuiButtonRounded( + (Rectangle){230.0f, 30.0f, 36.0f, 30.0f}, + "<", + 0.30f, + 10)) { + p_ui->collapsed = TRUE; + p_ui->dropdown_open = FALSE; + p_ui->context_open = FALSE; + return; + } if (p_ui->dropdown_open) GuiLock(); @@ -322,19 +367,28 @@ Canvas_Scene_Show_Lucide_Gallery(p_scene, p_camera); } - DrawTextEx(font, TextFormat("%d FPS", GetFPS()), (Vector2){34.0f, 396.0f}, 12.0f, 0.0f, p_theme->text_muted); - DrawTextEx(font, TextFormat("%.2fx zoom", p_camera->zoom), (Vector2){104.0f, 396.0f}, 12.0f, 0.0f, p_theme->text_muted); + size_t parked_count = Canvas_Scene_Parked_Conversation_Count(p_scene); + if (GuiButtonRounded( + (Rectangle){34.0f, 378.0f, 232.0f, 34.0f}, + TextFormat("Parking lot (%d)", (int)parked_count), + 0.32f, + 10)) { + Canvas_Scene_Show_Parking_Lot(p_scene, p_camera); + } + + DrawTextEx(font, TextFormat("%d FPS", GetFPS()), (Vector2){34.0f, 438.0f}, 12.0f, 0.0f, p_theme->text_muted); + DrawTextEx(font, TextFormat("%.2fx zoom", p_camera->zoom), (Vector2){104.0f, 438.0f}, 12.0f, 0.0f, p_theme->text_muted); DrawTextEx( font, TextFormat("%d objects", (int)Dowa_Array_Length(p_scene->p_entities)), - (Vector2){196.0f, 396.0f}, + (Vector2){196.0f, 438.0f}, 12.0f, 0.0f, p_theme->text_muted); DrawTextEx( font, "Drag objects directly on the canvas", - (Vector2){34.0f, 442.0f}, + (Vector2){34.0f, 484.0f}, 12.0f, 0.0f, p_theme->text_muted); diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/dev_ui.h --- a/infinite_canvas/dev_ui.h Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/dev_ui.h Tue Aug 18 19:14:53 2026 -0700 @@ -5,6 +5,7 @@ typedef struct { int32 selected_type; + boolean collapsed; boolean dropdown_open; boolean context_open; float context_scroll; diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/dictation_policy_test.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/dictation_policy_test.sh Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +set -euo pipefail + +native_source="$1" +main_source="$2" +dev_ui_source="$3" + +require_pattern() { + local pattern="$1" + local file="$2" + local message="$3" + if ! grep -Eq "$pattern" "$file"; then + echo "$message" >&2 + exit 1 + fi +} + +require_pattern \ + 'CefRequestContext::CreateContext' \ + "$native_source" \ + "Dictation must use an isolated CEF request context." +require_pattern \ + 'view_->is_dictation_transport' \ + "$native_source" \ + "Microphone permission must be scoped to the hidden dictation transport." +require_pattern \ + 'http://127\.0\.0\.1:8090' \ + "$native_source" \ + "Microphone permission must be scoped to the loopback dictation origin." +require_pattern \ + 'OnShowPermissionPrompt' \ + "$native_source" \ + "The hidden Chrome-style permission prompt must be handled natively." +require_pattern \ + 'CEF_PERMISSION_TYPE_MIC_STREAM' \ + "$native_source" \ + "The native prompt handler must identify microphone requests." +require_pattern \ + 'CEF_PERMISSION_RESULT_ACCEPT' \ + "$native_source" \ + "The native prompt handler must accept loopback microphone access." +require_pattern \ + 'requests_other_media' \ + "$native_source" \ + "The native permission handler must reject non-audio media requests." +require_pattern \ + 'callback->Continue\(audio\)' \ + "$native_source" \ + "The native permission handler must grant loopback audio directly." + +if grep -Eq 'AppendSwitch\("(enable-media-stream|use-fake-ui-for-media-stream)"' \ + "$native_source"; then + echo "Global media auto-grant switches are forbidden." >&2 + exit 1 +fi + +require_pattern \ + 'Canvas_App_Ensure_Dictation_Entity' \ + "$main_source" \ + "Dictation must create a retained canvas entity." +require_pattern \ + 'CANVAS_ENTITY_TEXT_AREA' \ + "$main_source" \ + "Dictation must use the wrapping text-area entity." +require_pattern \ + 'Canvas_Text_Area_Set_Content' \ + "$main_source" \ + "Live transcripts must update the retained text area." +require_pattern \ + '"Dictation"' \ + "$main_source" \ + "The live transcript entity must expose a draggable Dictation header." +submit_source="$( + sed -n \ + '/static boolean Canvas_App_Submit_Dictation(void)/,/^}/p' \ + "$main_source" +)" +if grep -Eq 'dictation_entity_id = 0' <<<"$submit_source" || + ! grep -Eq 'Canvas_App_Sync_Dictation_Entity' <<<"$submit_source"; then + echo "Submission must clear and reuse the one Dictation scratchpad." >&2 + exit 1 +fi +if grep -Eq 'Set_Dictation_Active' <<<"$submit_source"; then + echo "Sending a thought must keep live microphone capture active." >&2 + exit 1 +fi +if grep -Eq 'Canvas_App_Draw_Dictation_Overlay' "$main_source"; then + echo "Dictation must not also render a screen-space popup." >&2 + exit 1 +fi + +indicator_source="$( + sed -n \ + '/static void Canvas_App_Draw_Listening_Indicator(void)/,/^}/p' \ + "$main_source" +)" +for pattern in \ + 'if \(!g_app->dictation_listening\) return' \ + 'GetScreenHeight\(\) - 28\.0f' \ + 'g_app->theme\.danger' \ + 'g_app->theme\.text_muted' \ + '"Listening"'; do + if ! grep -Eq "$pattern" <<<"$indicator_source"; then + echo "Dictation must show a theme-aware bottom-left listening indicator." >&2 + exit 1 + fi +done + +input_source="$( + sed -n \ + '/static void Canvas_App_Update_Dictation_Overlay_Input(void)/,/^}/p' \ + "$main_source" +)" +if ! grep -Eq 'IsKeyPressed\(KEY_ENTER\)' <<<"$input_source"; then + echo "Dictation overlay must submit from Enter." >&2 + exit 1 +fi +if ! grep -Eq 'Canvas_Web_Surface_Commit_Dictation' <<<"$input_source"; then + echo "Enter must commit the utterance without stopping the microphone." >&2 + exit 1 +fi +if ! grep -Eq 'dictation_listening.*IsKeyPressed\(KEY_BACKSPACE\)' \ + <<<"$input_source"; then + echo "Backspace must clear active dictation without stopping capture." >&2 + exit 1 +fi +entity_source="$( + sed -n \ + '/static void Canvas_App_Sync_Dictation_Entity(void)/,/^}/p' \ + "$main_source" +)" +if ! grep -Eq "dictation_partial\\[0\\]" <<<"$entity_source" || + ! grep -Eq 'TRUE : FALSE' <<<"$entity_source"; then + echo "In-progress dictation text must use the muted theme color." >&2 + exit 1 +fi + +require_pattern \ + 'EnsureDictationView\(p_surface\);' \ + "$native_source" \ + "Native dictation transport must be prewarmed before the M hotkey." +require_pattern \ + 'Canvas_Web_Surface_Set_Dictation_Active' \ + "$main_source" \ + "Dictation must use deterministic start and stop commands." + +active_source="$( + sed -n \ + '/static void Canvas_App_Set_Dictation_Active(boolean active)/,/^}/p' \ + "$main_source" +)" +if grep -Eq "dictation_(partial|transcript)\\[0\\] = '\\\\0'" \ + <<<"$active_source"; then + echo "Resuming dictation must preserve the existing scratchpad text." >&2 + exit 1 +fi + +require_pattern \ + 'p_ui->collapsed' \ + "$dev_ui_source" \ + "Developer controls must expose a persistent collapsed state." +require_pattern \ + 'DEV_PANEL_COLLAPSED' \ + "$dev_ui_source" \ + "Collapsed developer controls must retain a visible restore button." + +final_source="$( + sed -n \ + '/CANVAS_DICTATION_EVENT_FINAL/,/if (!g_app->dictation_overlay_visible/p' \ + "$main_source" +)" +if ! grep -Eq 'dictation_send_pending' <<<"$final_source"; then + echo "Final dictation must wait for explicit Enter submission." >&2 + exit 1 +fi diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/docs/README.md --- a/infinite_canvas/docs/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/docs/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -17,6 +17,10 @@ │ ├── camera transforms │ ├── pinned entities │ └── notification stacking +├── Spatial agent sessions +│ ├── WebRTC dictation bridge +│ ├── asynchronous Copilot SDK orchestration +│ └── retained conversation entities └── Session lineage ├── original renderer └── scalability, images, pinning, context, and icons @@ -25,6 +29,7 @@ - [Rendering surfaces](rendering.md) - [Entities and components](entities-and-components.md) - [Pinning and visible context](pinning-and-context.md) +- [Dictation and spatial agent sessions](agent-sessions.md) - [Session lineage](session-lineage.md) ## Runtime frame order @@ -32,14 +37,14 @@ `main.c` owns the integration order: 1. Update viewport dimensions. -2. Let native web surfaces translate focused input. -3. Update scene interaction and keyboard focus. -4. Apply camera pan/zoom. -5. Reproject pinned entities and notification stacks. -6. Reconcile web/image resources with visible entities. -7. Draw the Raylib world. -8. Composite native browser textures or position web overlays. -9. Draw the developer UI. +2. Handle dictation and selected browser chrome. +3. Let native web surfaces translate focused input. +4. Update scene interaction and keyboard focus. +5. Submit or poll asynchronous agent work. +6. Apply camera pan/zoom and reproject pinned entities. +7. Reconcile web/image resources with visible entities. +8. Draw each Raylib entity and its owning native/DOM surface in scene order. +9. Draw world overlays and the developer UI. Preserve this order. In particular, pinned positions must be synchronized before surface bounds are calculated, and native surface textures must draw after the diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/docs/agent-sessions.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/docs/agent-sessions.md Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,84 @@ +# Dictation and Spatial Agent Sessions + +## Prototype flow + +The first orchestration slice connects three retained systems: + +1. `M` reuses the single retained `Dictation` text-area scratchpad and toggles + a hidden browser transport for `http://127.0.0.1:8090/?canvas=1`. +2. `Canvas_Web_Surface_Set_Dictation_Active()` deterministically starts or + stops capture on an already-negotiated WebRTC client. The hidden transport + acquires its stream, completes signaling/ICE, and opens the transcript data + channel during startup with its audio track disabled. + CEF grants microphone access only to the loopback dictation origin. +3. Status, partial, and final transcript events cross the platform boundary: + - native CEF encodes each event in the document title and + `OnTitleChange()` queues it for the caption; + - WebAssembly uses `window.postMessage()` from a hidden iframe. +4. `main.c` accumulates speech in the scratchpad and submits it to + `Canvas_Agent_Service` only when the user presses `Enter`. A selected text + area can enter the same path with `Ctrl/Cmd+Enter`. + Enter sends a `commit` command through the live transcript data channel, so + the current utterance finalizes without closing the microphone session. +5. `agent_service_copilot.c` sends the new thought plus + `Canvas_Scene_Build_Visible_Context()` through the shared asynchronous + `Inference_Bridge`. +6. The Bazel-managed Copilot SDK sidecar uses the `canvas_orchestrator` profile + and returns JSON choosing `create` or `append` and a conversation entity ID. + The canvas materializes that decision as a retained rich conversation card. + +The scratchpad is excluded from serialized camera context because the submitted +thought is already sent separately. Only conversation entities currently in +view are valid append targets; moving a conversation out of view removes it +from the orchestrator's routing choices. + +## Service boundary + +`agent_service_copilot.c` submits work to `Inference_Bridge`, whose worker +thread owns the JSON-line Python sidecar process so SDK latency never blocks +Raylib. The sidecar maintains the persistent +`infinite-canvas-orchestrator` SDK session and uses the loopback LiteLLM +gateway for authenticated GitHub Copilot inference. Only one orchestration +request is admitted at a time in this prototype. + +`agent_service_copilot.c` warms that orchestration session plus three reserved +worker sessions during initialization. They share the same +`canvas_orchestrator` profile and move Copilot SDK client/session setup ahead of +the first user turn. The persistent orchestrator is resumed when available; +reserved workers are created fresh because they have no conversation history +to recover, avoiding expected `session.resume` errors during startup. + +## Conversation entities + +`CANVAS_ENTITY_CONVERSATION` is intentionally more than an editable text area. +It retains: + +- a stable entity/session ID used by Copilot routing; +- a title; +- user and agent transcript content; +- asynchronous working state; +- independent scroll, selection, lifecycle, z-order, and pinning state. +- auto-follow to the latest appended turn and a collapsed header-only state; +- a recoverable parked state that is excluded from orchestration context. + +Images, browser views, and other components already appear in visible canvas +context as typed neighboring entities. A future resource list can attach those +entity IDs directly to a conversation without replacing the conversation card +or flattening rich content into one string. + +Archiving a conversation records its prior position and moves it to the +off-canvas parking lot. The Developer Controls parking-lot button fits the +camera to parked sessions. Restoring a parked card returns it to its prior +position and makes it eligible for routing again when visible. + +## Current constraints + +- `//infinite_canvas:orchestration_dev` owns the dictation server, LiteLLM + gateway, and canvas lifecycle. `//infinite_canvas:agent_dev` is an alias. +- One persistent Dictation scratchpad is reused and cleared after submission. + Submitted turns are retained in conversation entities after orchestration. +- One Copilot request runs at a time. +- Conversation transcript storage is currently bounded by + `CANVAS_ENTITY_TEXT_CAPACITY`. +- Linux uses the Copilot SDK bridge. Other platforms currently select an + explicit unavailable stub. diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/docs/entities-and-components.md --- a/infinite_canvas/docs/entities-and-components.md Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/docs/entities-and-components.md Tue Aug 18 19:14:53 2026 -0700 @@ -7,6 +7,8 @@ - a stable `id` for external resources such as browser views; - `type`, world `position`, and world `size`; - generic semantic fields: `text`, `value`, and `active`; +- optional rich-context fields such as a conversation title and agent working + state; - interaction/animation state; - pinning state in both world and screen coordinates. @@ -19,6 +21,13 @@ them back to zero, then compact the retained array. Raylib colors, native CEF texture tints, and WebAssembly overlay opacity all consume the same eased value. +Treat animation as part of every entity state transition, not optional polish. +State changes set retained targets; frame updates ease visual geometry and +opacity toward those targets. Do not snap expandable, collapsible, selected, +hovered, created, removed, or otherwise stateful entities between layouts. +Conversation collapse/expand uses the same smoothstep height interpolation as +accordions, while preserving its final semantic `active` state for context. + ## Adding a component Wire every relevant surface, not only drawing: @@ -47,6 +56,7 @@ - developer UI; - focused native CEF content; +- the canvas URL editor or local dictation bridge; - hovered/pressed scene entity; - camera. diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/main.c --- a/infinite_canvas/main.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/main.c Tue Aug 18 19:14:53 2026 -0700 @@ -1,3 +1,4 @@ +#include "infinite_canvas/agent_service.h" #include "infinite_canvas/canvas.h" #include "infinite_canvas/web_surface.h" @@ -9,6 +10,7 @@ #include #endif +#include #include #include #include @@ -18,11 +20,25 @@ Canvas_Camera camera; Canvas_Scene scene; Canvas_Web_Surface web_surface; + Canvas_Agent_Service agent_service; Font font; Canvas_Theme theme; boolean owns_font; const char *p_screenshot_path; uint32 frame_count; + uint32 screenshot_frame; + uint32 pending_conversation_id; + uint32 dictation_entity_id; + boolean agent_initialized; + boolean dictation_overlay_visible; + boolean dictation_listening; + boolean dictation_requested_active; + boolean dictation_send_pending; + char dictation_status[128]; + char dictation_partial[CANVAS_ENTITY_TEXT_CAPACITY]; + char dictation_transcript[CANVAS_ENTITY_TEXT_CAPACITY]; + char pending_prompt[CANVAS_ENTITY_TEXT_CAPACITY]; + char agent_context[CANVAS_CONTEXT_MAX_LENGTH]; #if defined(INFINITE_CANVAS_DEV_UI) Canvas_Dev_UI dev_ui; #endif @@ -30,6 +46,381 @@ static Canvas_App *g_app = NULL; +static const char *Canvas_App_JSON_String( + const char *p_json, + const char *p_key, + char *p_output, + size_t output_capacity) +{ + char needle[128]; + snprintf(needle, sizeof(needle), "\"%s\"", p_key); + const char *p_cursor = strstr(p_json, needle); + if (!p_cursor) return NULL; + p_cursor = strchr(p_cursor + strlen(needle), ':'); + if (!p_cursor) return NULL; + while (*++p_cursor == ' ') {} + if (*p_cursor != '"') return NULL; + p_cursor++; + size_t output = 0; + while (*p_cursor && *p_cursor != '"' && output + 1 < output_capacity) { + if (*p_cursor == '\\' && p_cursor[1]) { + p_cursor++; + if (*p_cursor == 'n') p_output[output++] = '\n'; + else if (*p_cursor == 't') p_output[output++] = '\t'; + else if (*p_cursor != 'r') p_output[output++] = *p_cursor; + } else { + p_output[output++] = *p_cursor; + } + p_cursor++; + } + p_output[output] = '\0'; + return p_output; +} + +static uint32 Canvas_App_JSON_Uint(const char *p_json, const char *p_key) +{ + char needle[128]; + snprintf(needle, sizeof(needle), "\"%s\"", p_key); + const char *p_cursor = strstr(p_json, needle); + if (!p_cursor) return 0; + p_cursor = strchr(p_cursor + strlen(needle), ':'); + if (!p_cursor) return 0; + return (uint32)strtoul(p_cursor + 1, NULL, 10); +} + +static Canvas_Entity *Canvas_App_Find_Entity(uint32 entity_id) +{ + for (size_t index = 0; + index < Dowa_Array_Length(g_app->scene.p_entities); + index++) { + if (g_app->scene.p_entities[index].id == entity_id) { + return &g_app->scene.p_entities[index]; + } + } + return NULL; +} + +static void Canvas_App_Set_Conversation_Text( + Canvas_Entity *p_entity, + const char *p_prompt, + const char *p_speaker, + const char *p_response) +{ + size_t length = 0; + int32 written = snprintf( + p_entity->text, + sizeof(p_entity->text), + "You\n"); + if (written > 0) length = (size_t)written; + written = snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "%.*s\n\n%s\n", + 900, + p_prompt, + p_speaker); + if (written > 0) length += (size_t)written; + if (length >= sizeof(p_entity->text)) { + length = sizeof(p_entity->text) - 1; + } + snprintf( + p_entity->text + length, + sizeof(p_entity->text) - length, + "%.*s", + 900, + p_response); + Canvas_Conversation_Scroll_To_End(p_entity, g_app->font); +} + +static boolean Canvas_App_Submit_Prompt(const char *p_prompt) +{ + if (!p_prompt || !p_prompt[0]) return FALSE; + Canvas_Scene_Build_Visible_Context( + &g_app->scene, + &g_app->camera, + g_app->agent_context, + sizeof(g_app->agent_context)); + if (!Canvas_Agent_Service_Submit( + &g_app->agent_service, + p_prompt, + g_app->agent_context)) { + return FALSE; + } + snprintf( + g_app->pending_prompt, + sizeof(g_app->pending_prompt), + "%s", + p_prompt); + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + g_app->pending_conversation_id = Canvas_Scene_Add_Conversation( + &g_app->scene, + "Routing thought...", + p_prompt, + "Copilot is deciding whether to create or extend a session.", + position); + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (p_pending) p_pending->agent_working = TRUE; + return TRUE; +} + +static void Canvas_App_Apply_Agent_Response(const char *p_json) +{ + char action[32] = {0}; + char title[128] = {0}; + char response[CANVAS_ENTITY_TEXT_CAPACITY] = {0}; + Canvas_App_JSON_String(p_json, "action", action, sizeof(action)); + Canvas_App_JSON_String(p_json, "title", title, sizeof(title)); + if (!Canvas_App_JSON_String( + p_json, + "response", + response, + sizeof(response))) { + snprintf(response, sizeof(response), "%s", p_json); + } + uint32 conversation_id = + Canvas_App_JSON_Uint(p_json, "conversation_id"); + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (strcmp(action, "append") == 0 && + conversation_id != 0 && + Canvas_Scene_Append_Conversation( + &g_app->scene, + conversation_id, + g_app->pending_prompt, + response)) { + Canvas_Entity *p_conversation = + Canvas_App_Find_Entity(conversation_id); + Canvas_Conversation_Scroll_To_End( + p_conversation, + g_app->font); + if (p_pending) p_pending->removing = TRUE; + } else if (p_pending) { + snprintf( + p_pending->label, + sizeof(p_pending->label), + "%s", + title[0] ? title : "Copilot session"); + Canvas_App_Set_Conversation_Text( + p_pending, + g_app->pending_prompt, + "Copilot", + response); + p_pending->agent_working = FALSE; + } else { + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + uint32 created_id = Canvas_Scene_Add_Conversation( + &g_app->scene, + title[0] ? title : "Copilot session", + g_app->pending_prompt, + response, + position); + Canvas_Conversation_Scroll_To_End( + Canvas_App_Find_Entity(created_id), + g_app->font); + } + g_app->pending_conversation_id = 0; + g_app->pending_prompt[0] = '\0'; +} + +static Canvas_Entity *Canvas_App_Ensure_Dictation_Entity(void) +{ + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (p_entity) return p_entity; + + Vector2 center = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f, + (float)g_app->camera.viewport_height * 0.5f, + }); + Vector2 size = {520.0f, 240.0f}; + if (!Canvas_Scene_Add( + &g_app->scene, + CANVAS_ENTITY_TEXT_AREA, + (Vector2){ + center.x - size.x * 0.5f, + center.y - size.y * 0.5f, + })) { + return NULL; + } + size_t index = Dowa_Array_Length(g_app->scene.p_entities) - 1; + p_entity = &g_app->scene.p_entities[index]; + p_entity->size = size; + snprintf(p_entity->label, sizeof(p_entity->label), "Dictation"); + g_app->dictation_entity_id = p_entity->id; + g_app->scene.selected_index = (int32)index; + return p_entity; +} + +static void Canvas_App_Sync_Dictation_Entity(void) +{ + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (!p_entity) return; + + char text[CANVAS_ENTITY_TEXT_CAPACITY]; + snprintf(text, sizeof(text), "%s", g_app->dictation_transcript); + size_t length = strlen(text); + if (length > 0 && g_app->dictation_partial[0] && + length + 1 < sizeof(text)) { + text[length++] = ' '; + text[length] = '\0'; + } + snprintf( + text + length, + sizeof(text) - length, + "%s", + g_app->dictation_partial); + Canvas_Text_Area_Set_Content( + p_entity, + g_app->font, + text, + g_app->dictation_partial[0] ? TRUE : FALSE); +} + +static void Canvas_App_Set_Dictation_Active(boolean active) +{ + g_app->dictation_overlay_visible = TRUE; + g_app->dictation_requested_active = active; + g_app->dictation_listening = FALSE; + g_app->dictation_send_pending = FALSE; + if (active) { + Canvas_App_Ensure_Dictation_Entity(); + Canvas_App_Sync_Dictation_Entity(); + } + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "%s", + active ? "Starting microphone..." : "Stopping microphone..."); + Canvas_Web_Surface_Set_Dictation_Active( + &g_app->web_surface, + active); +} + +static boolean Canvas_App_Submit_Dictation(void) +{ + if (!g_app->dictation_transcript[0]) return FALSE; + if (!Canvas_App_Submit_Prompt(g_app->dictation_transcript)) { + g_app->dictation_send_pending = FALSE; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Copilot is busy; press Enter to retry"); + return FALSE; + } + Canvas_Entity *p_entity = + Canvas_App_Find_Entity(g_app->dictation_entity_id); + if (p_entity) p_entity->text_muted = FALSE; + g_app->dictation_transcript[0] = '\0'; + g_app->dictation_partial[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + g_app->dictation_send_pending = FALSE; + g_app->dictation_overlay_visible = + g_app->dictation_listening; + return TRUE; +} + +static void Canvas_App_Append_Dictation(const char *p_text) +{ + if (!p_text || !p_text[0]) return; + size_t length = strlen(g_app->dictation_transcript); + if (length > 0 && length + 1 < sizeof(g_app->dictation_transcript)) { + g_app->dictation_transcript[length++] = ' '; + g_app->dictation_transcript[length] = '\0'; + } + size_t available = sizeof(g_app->dictation_transcript) - length - 1; + size_t text_length = strlen(p_text); + size_t copy_length = text_length < available ? text_length : available; + memcpy(g_app->dictation_transcript + length, p_text, copy_length); + g_app->dictation_transcript[length + copy_length] = '\0'; + Canvas_App_Sync_Dictation_Entity(); +} + +static void Canvas_App_Update_Dictation_Overlay_Input(void) +{ + if (!g_app->dictation_overlay_visible) return; + if (g_app->dictation_listening && IsKeyPressed(KEY_BACKSPACE)) { + g_app->dictation_send_pending = FALSE; + g_app->dictation_partial[0] = '\0'; + g_app->dictation_transcript[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Listening"); + return; + } + if (IsKeyPressed(KEY_ESCAPE)) { + if (g_app->dictation_listening) { + Canvas_Web_Surface_Set_Dictation_Active( + &g_app->web_surface, + FALSE); + g_app->dictation_requested_active = FALSE; + g_app->dictation_listening = FALSE; + } + g_app->dictation_send_pending = FALSE; + g_app->dictation_partial[0] = '\0'; + g_app->dictation_transcript[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + g_app->dictation_overlay_visible = FALSE; + return; + } + if (!IsKeyPressed(KEY_ENTER)) return; + + if (g_app->dictation_partial[0]) { + g_app->dictation_send_pending = TRUE; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Finishing dictation..."); + Canvas_Web_Surface_Commit_Dictation( + &g_app->web_surface); + } else if (g_app->dictation_transcript[0]) { + Canvas_App_Submit_Dictation(); + } +} + +static boolean Canvas_App_Dictation_Status_Is_Error(const char *p_status) +{ + return strstr(p_status, "failed") != NULL || + strstr(p_status, "Unable") != NULL || + strstr(p_status, "closed") != NULL || + strstr(p_status, "invalid") != NULL || + strstr(p_status, "error") != NULL; +} + +static void Canvas_App_Draw_Listening_Indicator(void) +{ + if (!g_app->dictation_listening) return; + + float pulse = 0.72f + 0.28f * sinf((float)GetTime() * 5.0f); + Vector2 dot = {28.0f, (float)GetScreenHeight() - 28.0f}; + DrawCircleV( + dot, + 5.0f + pulse * 2.0f, + Fade(g_app->theme.danger, 0.82f)); + DrawTextEx( + g_app->font, + "Listening", + (Vector2){44.0f, dot.y - 8.0f}, + 14.0f, + 0.0f, + g_app->theme.text_muted); +} + static void Canvas_App_Draw_Frame(void) { int32 width = GetScreenWidth(); @@ -43,6 +434,15 @@ Canvas_Web_Surface_Set_Dark_Mode( &g_app->web_surface, g_app->theme.resolved_mode == CANVAS_THEME_DARK ? TRUE : FALSE); + boolean dictation_hotkey = + !Canvas_Scene_Is_Text_Editing(&g_app->scene) && + !Canvas_Web_Chrome_Is_Editing(&g_app->scene) && + IsKeyPressed(KEY_M); + if (dictation_hotkey) { + Canvas_App_Set_Dictation_Active( + g_app->dictation_requested_active ? FALSE : TRUE); + } + Canvas_App_Update_Dictation_Overlay_Input(); Canvas_Web_Chrome_Action chrome_action = CANVAS_WEB_CHROME_NONE; if (!block_pointer_input) { const char *p_current_url = NULL; @@ -76,7 +476,9 @@ } } boolean web_input = FALSE; - if (!block_pointer_input && chrome_action == CANVAS_WEB_CHROME_NONE) { + if (!block_pointer_input && + !dictation_hotkey && + chrome_action == CANVAS_WEB_CHROME_NONE) { web_input = Canvas_Web_Surface_Update_Input( &g_app->web_surface, &g_app->camera, @@ -99,6 +501,7 @@ Canvas_Scene_Toggle_Selected_Pin(&g_app->scene, &g_app->camera); } if (!keyboard_focus && + !g_app->dictation_listening && (IsKeyPressed(KEY_DELETE) || IsKeyPressed(KEY_BACKSPACE))) { Canvas_Scene_Fade_Out_Selected(&g_app->scene); } @@ -109,6 +512,136 @@ Canvas_Scene_Sync_Pinned(&g_app->scene, &g_app->camera); Canvas_Scene_Update_Notification_Stack(&g_app->scene, &g_app->camera); + char dictation_text[CANVAS_ENTITY_TEXT_CAPACITY]; + Canvas_Dictation_Event dictation_event = + Canvas_Web_Surface_Consume_Dictation( + &g_app->web_surface, + dictation_text, + sizeof(dictation_text)); + if (dictation_event == CANVAS_DICTATION_EVENT_STATUS) { + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "%.127s", + dictation_text); + if (strcmp(dictation_text, "Idle") == 0) { + g_app->dictation_listening = FALSE; + if (g_app->dictation_send_pending) { + if (g_app->dictation_partial[0]) { + Canvas_App_Append_Dictation( + g_app->dictation_partial); + g_app->dictation_partial[0] = '\0'; + } + Canvas_App_Submit_Dictation(); + } else if (!g_app->dictation_transcript[0] && + !g_app->dictation_partial[0]) { + g_app->dictation_overlay_visible = FALSE; + } + } else if (strcmp(dictation_text, "Listening") == 0 || + strcmp(dictation_text, "Speech detected") == 0) { + g_app->dictation_listening = + g_app->dictation_requested_active; + if (strcmp(dictation_text, "Listening") == 0 && + g_app->dictation_send_pending) { + if (g_app->dictation_partial[0]) { + Canvas_App_Append_Dictation( + g_app->dictation_partial); + g_app->dictation_partial[0] = '\0'; + } + Canvas_App_Submit_Dictation(); + } + } else if (Canvas_App_Dictation_Status_Is_Error(dictation_text)) { + g_app->dictation_listening = FALSE; + g_app->dictation_overlay_visible = TRUE; + } + } else if (dictation_event == CANVAS_DICTATION_EVENT_PARTIAL) { + g_app->dictation_overlay_visible = TRUE; + snprintf( + g_app->dictation_partial, + sizeof(g_app->dictation_partial), + "%s", + dictation_text); + Canvas_App_Sync_Dictation_Entity(); + } else if (dictation_event == CANVAS_DICTATION_EVENT_FINAL) { + g_app->dictation_overlay_visible = TRUE; + Canvas_App_Append_Dictation(dictation_text); + g_app->dictation_partial[0] = '\0'; + Canvas_App_Sync_Dictation_Entity(); + if (g_app->dictation_send_pending) { + Canvas_App_Submit_Dictation(); + } else { + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Ready to send"); + } + } + if (!g_app->dictation_overlay_visible && + g_app->scene.selected_index >= 0 && + g_app->scene.selected_index < + (int32)Dowa_Array_Length(g_app->scene.p_entities) && + (IsKeyDown(KEY_LEFT_CONTROL) || + IsKeyDown(KEY_RIGHT_CONTROL) || + IsKeyDown(KEY_LEFT_SUPER) || + IsKeyDown(KEY_RIGHT_SUPER)) && + IsKeyPressed(KEY_ENTER)) { + Canvas_Entity *p_selected = + &g_app->scene.p_entities[g_app->scene.selected_index]; + if (p_selected->type == CANVAS_ENTITY_TEXT_AREA && + p_selected->text[0]) { + size_t length = strlen(p_selected->text); + if (length > 0 && p_selected->text[length - 1] == '\n') { + p_selected->text[length - 1] = '\0'; + } + if (Canvas_App_Submit_Prompt(p_selected->text)) { + p_selected->text[0] = '\0'; + p_selected->text_cursor = 0; + p_selected->text_selection_anchor = 0; + } + } + } + + char agent_response[CANVAS_AGENT_RESPONSE_CAPACITY]; + char agent_error[512]; + Canvas_Agent_Status agent_status = g_app->agent_initialized ? + Canvas_Agent_Service_Poll( + &g_app->agent_service, + agent_response, + sizeof(agent_response), + agent_error, + sizeof(agent_error)) : + CANVAS_AGENT_IDLE; + if (agent_status == CANVAS_AGENT_READY) { + Canvas_App_Apply_Agent_Response(agent_response); + } else if (agent_status == CANVAS_AGENT_ERROR) { + Canvas_Entity *p_pending = + Canvas_App_Find_Entity(g_app->pending_conversation_id); + if (p_pending) { + snprintf(p_pending->label, sizeof(p_pending->label), "Copilot unavailable"); + Canvas_App_Set_Conversation_Text( + p_pending, + g_app->pending_prompt, + "System", + agent_error); + p_pending->agent_working = FALSE; + } else if (g_app->pending_prompt[0]) { + Vector2 position = Canvas_Camera_Screen_To_World( + &g_app->camera, + (Vector2){ + (float)g_app->camera.viewport_width * 0.5f - 210.0f, + (float)g_app->camera.viewport_height * 0.5f - 150.0f, + }); + Canvas_Scene_Add_Conversation( + &g_app->scene, + "Copilot unavailable", + g_app->pending_prompt, + agent_error, + position); + } + g_app->pending_conversation_id = 0; + g_app->pending_prompt[0] = '\0'; + } + Canvas_Web_Surface_Sync(&g_app->web_surface, &g_app->camera, &g_app->scene); BeginDrawing(); @@ -162,6 +695,7 @@ g_app->font, &g_app->theme); #endif + Canvas_App_Draw_Listening_Indicator(); EndDrawing(); g_app->frame_count++; #if !defined(PLATFORM_WEB) @@ -171,7 +705,7 @@ } #endif if (g_app->p_screenshot_path && - g_app->frame_count == 180) { + g_app->frame_count == g_app->screenshot_frame) { TakeScreenshot(g_app->p_screenshot_path); } } @@ -196,7 +730,21 @@ } memset(g_app, 0, sizeof(*g_app)); g_app->p_arena = p_arena; + snprintf( + g_app->dictation_status, + sizeof(g_app->dictation_status), + "Idle"); g_app->p_screenshot_path = getenv("INFINITE_CANVAS_SCREENSHOT"); + g_app->screenshot_frame = 180; + const char *p_screenshot_frame = + getenv("INFINITE_CANVAS_SCREENSHOT_FRAME"); + if (p_screenshot_frame) { + uint32 screenshot_frame = + (uint32)strtoul(p_screenshot_frame, NULL, 10); + if (screenshot_frame > 0) { + g_app->screenshot_frame = screenshot_frame; + } + } Canvas_Theme_Resolve( &g_app->theme, Canvas_Theme_Mode_From_String(getenv("INFINITE_CANVAS_THEME"))); @@ -219,6 +767,11 @@ g_app->theme.resolved_mode == CANVAS_THEME_DARK ? TRUE : FALSE)) { TraceLog(LOG_WARNING, "Web surface initialization failed"); } + g_app->agent_initialized = + Canvas_Agent_Service_Init(&g_app->agent_service, p_arena); + if (!g_app->agent_initialized) { + TraceLog(LOG_WARNING, "Agent service initialization failed"); + } g_app->font = LoadFontEx("infinite_canvas/assets/Inter-Variable.ttf", 48, NULL, 0); g_app->owns_font = g_app->font.texture.id != 0; @@ -231,6 +784,13 @@ } else { Canvas_Scene_Add_Demo(&g_app->scene); } + if (getenv("INFINITE_CANVAS_DICTATION_AUTOSTART")) { + Canvas_App_Set_Dictation_Active(TRUE); + } + const char *p_initial_prompt = getenv("INFINITE_CANVAS_AGENT_PROMPT"); + if (p_initial_prompt && p_initial_prompt[0]) { + Canvas_App_Submit_Prompt(p_initial_prompt); + } const char *p_image_source = getenv("INFINITE_CANVAS_IMAGE_SOURCE"); if (p_image_source) { for (size_t index = 0; index < Dowa_Array_Length(g_app->scene.p_entities); index++) { @@ -258,6 +818,9 @@ emscripten_set_main_loop(Canvas_App_Draw_Frame, 0, 1); #else while (!WindowShouldClose()) Canvas_App_Draw_Frame(); + if (g_app->agent_initialized) { + Canvas_Agent_Service_Shutdown(&g_app->agent_service); + } Canvas_Web_Surface_Shutdown(&g_app->web_surface); if (g_app->owns_font) UnloadFont(g_app->font); Dowa_Arena_Free(p_arena); diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/run_orchestration_dev.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/infinite_canvas/run_orchestration_dev.sh Tue Aug 18 19:14:53 2026 -0700 @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNFILES_ROOT="${RUNFILES_DIR:-$0.runfiles}" +export RUNFILES_DIR="$RUNFILES_ROOT" + +find_runfile() { + local pattern="$1" + local path + path="$(find -L "$RUNFILES_ROOT" -path "$pattern" -print -quit)" + if [[ -z "$path" ]]; then + echo "Could not locate orchestration runtime file: $pattern" >&2 + exit 1 + fi + realpath "$path" +} + +find_config() { + local candidate + for candidate in \ + "${BUILD_WORKSPACE_DIRECTORY:-}/mrjunejune/.config" \ + "$PWD/mrjunejune/.config" \ + "${HOME:-}/.config/mrjunejune/.config"; do + [[ "$candidate" != /mrjunejune/.config ]] || continue + [[ "$candidate" != /.config/mrjunejune/.config ]] || continue + if [[ -f "$candidate" ]]; then + realpath "$candidate" + return 0 + fi + done + return 1 +} + +load_config() { + local config_path="$1" + local line key value + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%$'\r'}" + [[ "$line" =~ ^[[:space:]]*$ ]] && continue + [[ "$line" =~ ^[[:space:]]*# ]] && continue + if [[ "$line" != *=* ]]; then + echo "Invalid config line in $config_path" >&2 + exit 1 + fi + key="${line%%=*}" + value="${line#*=}" + key="${key#"${key%%[![:space:]]*}"}" + key="${key%"${key##*[![:space:]]}"}" + case "$key" in + LITELLM_MASTER_KEY|LITELLM_HOST|LITELLM_PORT|LITELLM_MODEL|\ + LITELLM_WIRE_API|GITHUB_COPILOT_TOKEN_DIR|\ + MRJUNEJUNE_INFERENCE_STATE|COPILOT_SESSION_IDLE_SECONDS|\ + COPILOT_MAX_SESSIONS) + printf -v "$key" '%s' "$value" + ;; + esac + done < "$config_path" +} + +dictation="$(find_runfile '*/dictation/server')" +dictation_bin="$(find_runfile '*/dictation/dictation_bin')" +canvas="$(find_runfile '*/infinite_canvas/dev_linux')" +sidecar_launcher="$(find_runfile '*/mrjunejune/inference_sidecar_launcher.sh')" +sidecar_zip="$(find_runfile '*/mrjunejune/inference/copilot_sidecar.zip')" +litellm_zip="$(find_runfile '*/mrjunejune/inference/litellm_proxy.zip')" +litellm_config="$(find_runfile '*/mrjunejune/inference/litellm_config.yaml')" +copilot_cli="$(find_runfile '*copilot_cli_linux_x86_64/copilot')" +python="$(find_runfile '*python_3_11_x86_64-unknown-linux-gnu/bin/python3')" + +config_file="$(find_config || true)" +if [[ -z "$config_file" ]]; then + echo "Missing mrjunejune/.config." >&2 + echo "Copy mrjunejune/.config.development to mrjunejune/.config." >&2 + exit 1 +fi +load_config "$config_file" + +state_root="${MRJUNEJUNE_INFERENCE_STATE:-${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference}" +token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}" +cache_root="$state_root/cache" +mkdir -p "$state_root/copilot" "$token_dir" "$cache_root" +chmod 700 "$state_root" "$state_root/copilot" "$token_dir" "$cache_root" + +if [[ ! -s "$token_dir/access-token" ]]; then + echo "GitHub Copilot is not authenticated." >&2 + echo "Run: bazel run //mrjunejune:run_inference_stack -- --authenticate" >&2 + exit 1 +fi + +if [[ -z "${LITELLM_MASTER_KEY:-}" ]]; then + umask 077 + LITELLM_MASTER_KEY="sk-$("$python" -c 'import secrets; print(secrets.token_hex(24))')" +fi + +litellm_host="${LITELLM_HOST:-127.0.0.1}" +litellm_port="${LITELLM_PORT:-4000}" +service_pids=() + +cleanup() { + local status=$? + trap - EXIT INT TERM + for pid in "${service_pids[@]}"; do + kill -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true + done + for pid in "${service_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + exit "$status" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +export GITHUB_COPILOT_TOKEN_DIR="$token_dir" +export XDG_CACHE_HOME="$cache_root" +export DICTATION_MODEL_DIR="${DICTATION_MODEL_DIR:-$HOME/.cache/zenbu/faster-whisper-small}" +export LITELLM_MASTER_KEY +setsid "$python" "$litellm_zip" \ + --config "$litellm_config" \ + --host "$litellm_host" \ + --port "$litellm_port" & +service_pids+=("$!") + +setsid "$dictation" "$dictation_bin" server & +service_pids+=("$!") + +for _ in {1..150}; do + if curl --fail --silent \ + --connect-timeout 1 \ + --max-time 2 \ + "http://${litellm_host}:${litellm_port}/health/readiness" \ + >/dev/null; then + break + fi + if ! kill -0 "${service_pids[0]}" 2>/dev/null; then + echo "Copilot LiteLLM gateway exited before becoming ready." >&2 + exit 1 + fi + sleep 0.1 +done +if ! curl --fail --silent \ + --connect-timeout 1 \ + --max-time 2 \ + "http://${litellm_host}:${litellm_port}/health/readiness" \ + >/dev/null; then + echo "Copilot LiteLLM readiness timed out." >&2 + exit 1 +fi + +for _ in {1..120}; do + if curl --silent --fail --output /dev/null http://127.0.0.1:8090/; then + break + fi + if ! kill -0 "${service_pids[1]}" 2>/dev/null; then + echo "The dictation service exited before becoming ready." >&2 + exit 1 + fi + sleep 0.25 +done +if ! curl --silent --fail --output /dev/null http://127.0.0.1:8090/; then + echo "Timed out waiting for dictation at http://127.0.0.1:8090." >&2 + exit 1 +fi + +export MRJUNEJUNE_PYTHON_PATH="$python" +export MRJUNEJUNE_SIDECAR_ZIP="$sidecar_zip" +export COPILOT_SIDECAR_HOME="$state_root/copilot" +export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1" +export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}" +export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}" +export LITELLM_API_KEY="$LITELLM_MASTER_KEY" +export INFINITE_CANVAS_COPILOT_SIDECAR_PATH="$sidecar_launcher" +export INFINITE_CANVAS_COPILOT_CLI_PATH="$copilot_cli" + +echo "Copilot orchestration is ready. Starting Infinite Canvas." +canvas_status=0 +"$canvas" "$@" || canvas_status=$? +exit "$canvas_status" diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/web_surface.h --- a/infinite_canvas/web_surface.h Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface.h Tue Aug 18 19:14:53 2026 -0700 @@ -5,6 +5,13 @@ #define CANVAS_MAX_WEB_VIEWS 12 +typedef enum { + CANVAS_DICTATION_EVENT_NONE = 0, + CANVAS_DICTATION_EVENT_STATUS, + CANVAS_DICTATION_EVENT_PARTIAL, + CANVAS_DICTATION_EVENT_FINAL, +} Canvas_Dictation_Event; + typedef struct { uint32 entity_id; Canvas_Entity_Type entity_type; @@ -16,6 +23,12 @@ size_t pixel_capacity; char source_url[CANVAS_ENTITY_TEXT_CAPACITY]; char url[CANVAS_ENTITY_TEXT_CAPACITY]; + char dictation_text[CANVAS_ENTITY_TEXT_CAPACITY]; + uint32 dictation_sequence; + uint32 consumed_dictation_sequence; + int32 dictation_command; + Canvas_Dictation_Event dictation_event; + boolean is_dictation_transport; boolean can_go_back; boolean can_go_forward; boolean color_scheme_applied; @@ -31,6 +44,7 @@ void *p_native; Canvas_Web_View views[CANVAS_MAX_WEB_VIEWS]; uint32 focused_entity_id; + int32 pending_dictation_command; boolean dark_mode; boolean initialized; } Canvas_Web_Surface; @@ -79,6 +93,15 @@ void Canvas_Web_Surface_Go_Forward( Canvas_Web_Surface *p_surface, uint32 entity_id); +void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active); +void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface); +Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity); boolean Canvas_Web_Surface_Is_Focused(const Canvas_Web_Surface *p_surface); void Canvas_Web_Surface_Shutdown(Canvas_Web_Surface *p_surface); diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/web_surface_native.cc --- a/infinite_canvas/web_surface_native.cc Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface_native.cc Tue Aug 18 19:14:53 2026 -0700 @@ -6,16 +6,20 @@ #include "include/cef_browser.h" #include "include/cef_client.h" #include "include/cef_display_handler.h" +#include "include/cef_permission_handler.h" #include "include/cef_render_handler.h" +#include "include/cef_request_context.h" #include "include/capi/cef_app_capi.h" #include +#include #include #include #include #include #include #include +#include #if defined(OS_LINUX) #include @@ -29,6 +33,8 @@ constexpr int32 kMinTextureHeight = 120; constexpr int32 kMaxTextureWidth = 1280; constexpr int32 kMaxTextureHeight = 800; +constexpr uint32 kDictationViewId = 0xffffffffu; +constexpr const char *kDictationUrl = "http://127.0.0.1:8090/?canvas=1"; int g_argc = 0; char **g_argv = nullptr; char *g_cef_argv[128] = {}; @@ -41,6 +47,26 @@ void FindResourcePaths(); +void PercentDecode(const char *source, char *target, size_t target_size) { + size_t output = 0; + for (size_t index = 0; + source[index] && output + 1 < target_size; + ++index) { + if (source[index] == '%' && + std::isxdigit(static_cast(source[index + 1])) && + std::isxdigit(static_cast(source[index + 2]))) { + char hex[3] = {source[index + 1], source[index + 2], '\0'}; + target[output++] = static_cast(std::strtol(hex, nullptr, 16)); + index += 2; + } else if (source[index] == '+') { + target[output++] = ' '; + } else { + target[output++] = source[index]; + } + } + target[output] = '\0'; +} + void ApplyColorScheme(CefRefPtr browser, boolean dark_mode) { if (!browser) return; CefRefPtr params = CefDictionaryValue::Create(); @@ -62,6 +88,12 @@ frame->ExecuteJavaScript(script, frame->GetURL(), 0); } +boolean IsLocalDictationOrigin(const CefString &requesting_origin) { + std::string origin = requesting_origin.ToString(); + return origin.rfind("http://127.0.0.1:8090", 0) == 0 || + origin.rfind("http://localhost:8090", 0) == 0; +} + const Canvas_Entity *FindSurfaceEntity( const Canvas_Scene *scene, uint32 entity_id) { @@ -167,12 +199,16 @@ class CanvasCefClient : public CefClient, public CefRenderHandler, public CefDisplayHandler, + public CefPermissionHandler, public CefLifeSpanHandler { public: explicit CanvasCefClient(Canvas_Web_View *view) : view_(view) {} CefRefPtr GetRenderHandler() override { return this; } CefRefPtr GetDisplayHandler() override { return this; } + CefRefPtr GetPermissionHandler() override { + return this; + } CefRefPtr GetLifeSpanHandler() override { return this; } void OnAddressChange( @@ -189,6 +225,83 @@ view_->can_go_forward = browser->CanGoForward() ? TRUE : FALSE; } + void OnTitleChange( + CefRefPtr browser, + const CefString &title) override { + (void)browser; + std::string value = title.ToString(); + constexpr const char *prefix = "Zenbu Dictation|"; + if (value.rfind(prefix, 0) != 0) return; + const char *kind = std::strchr( + value.c_str() + std::strlen(prefix), + '|'); + if (!kind) return; + kind++; + const char *encoded = std::strchr(kind, '|'); + if (!encoded) return; + std::string event_kind(kind, static_cast(encoded - kind)); + if (event_kind == "status") { + view_->dictation_event = CANVAS_DICTATION_EVENT_STATUS; + } else if (event_kind == "partial") { + view_->dictation_event = CANVAS_DICTATION_EVENT_PARTIAL; + } else if (event_kind == "final") { + view_->dictation_event = CANVAS_DICTATION_EVENT_FINAL; + } else { + return; + } + PercentDecode( + encoded + 1, + view_->dictation_text, + sizeof(view_->dictation_text)); + view_->dictation_sequence++; + } + + bool OnRequestMediaAccessPermission( + CefRefPtr browser, + CefRefPtr frame, + const CefString &requesting_origin, + uint32_t requested_permissions, + CefRefPtr callback) override { + (void)browser; + (void)frame; + boolean local_dictation = + view_->is_dictation_transport && + IsLocalDictationOrigin(requesting_origin); + uint32_t audio = CEF_MEDIA_PERMISSION_DEVICE_AUDIO_CAPTURE; + boolean requests_audio = (requested_permissions & audio) != 0; + boolean requests_other_media = (requested_permissions & ~audio) != 0; + if (local_dictation && requests_audio && !requests_other_media) { + callback->Continue(audio); + } else { + callback->Cancel(); + } + return true; + } + + bool OnShowPermissionPrompt( + CefRefPtr browser, + uint64_t prompt_id, + const CefString &requesting_origin, + uint32_t requested_permissions, + CefRefPtr callback) override { + (void)browser; + (void)prompt_id; + if (!view_->is_dictation_transport || + !IsLocalDictationOrigin(requesting_origin)) { + return false; + } + uint32_t microphone = CEF_PERMISSION_TYPE_MIC_STREAM; + boolean requests_microphone = + (requested_permissions & microphone) != 0; + boolean requests_other_permissions = + (requested_permissions & ~microphone) != 0; + callback->Continue( + requests_microphone && !requests_other_permissions + ? CEF_PERMISSION_RESULT_ACCEPT + : CEF_PERMISSION_RESULT_DENY); + return true; + } + void GetViewRect(CefRefPtr browser, CefRect &rect) override { (void)browser; rect = CefRect(0, 0, view_->texture_width, view_->texture_height); @@ -243,9 +356,17 @@ struct NativeState { CefRefPtr clients[CANVAS_MAX_WEB_VIEWS]; + CefRefPtr dictation_context; boolean cef_initialized; }; +void EnsureDictationRequestContext(NativeState *state) { + if (state->dictation_context) return; + CefRequestContextSettings settings; + state->dictation_context = + CefRequestContext::CreateContext(settings, nullptr); +} + CefMainArgs MainArgs(int argc, char **argv) { #if defined(OS_WIN) (void)argc; @@ -396,13 +517,18 @@ : CefColorSetARGB(255, 255, 255, 255); const char *initial_url = view->entity_type == CANVAS_ENTITY_IMAGE ? "about:blank" : url; + CefRefPtr request_context; + if (view->is_dictation_transport) { + EnsureDictationRequestContext(state); + request_context = state->dictation_context; + } if (!CefBrowserHost::CreateBrowser( window_info, state->clients[index], initial_url, browser_settings, nullptr, - nullptr)) { + request_context)) { state->clients[index] = nullptr; return FALSE; } @@ -410,6 +536,47 @@ return TRUE; } +Canvas_Web_View *EnsureDictationView(Canvas_Web_Surface *surface) { + Canvas_Web_View *view = FindView(surface, kDictationViewId); + if (view) { + view->active = TRUE; + return view; + } + view = FindInactiveView(surface); + if (!view) return nullptr; + boolean browser_created = view->browser_created; + view->active = TRUE; + view->entity_id = kDictationViewId; + view->entity_type = CANVAS_ENTITY_WEB_CONTENT; + view->is_dictation_transport = TRUE; + std::snprintf( + view->source_url, + sizeof(view->source_url), + "%s", + kDictationUrl); + std::snprintf(view->url, sizeof(view->url), "%s", kDictationUrl); + Rectangle bounds = {0.0f, 0.0f, 320.0f, 240.0f}; + if (!browser_created && + !CreateBrowser(surface, view, kDictationUrl, bounds)) { + view->active = FALSE; + view->entity_id = 0; + view->is_dictation_transport = FALSE; + view->source_url[0] = '\0'; + view->url[0] = '\0'; + return nullptr; + } + if (browser_created) { + EnsureViewResolution(surface, view, bounds); + NativeState *state = static_cast(surface->p_native); + CefRefPtr client = + state->clients[ViewIndex(surface, view)]; + if (client && client->browser()) { + client->browser()->GetMainFrame()->LoadURL(kDictationUrl); + } + } + return view; +} + void PercentEncode( const char *source, char *target, @@ -528,8 +695,9 @@ const Canvas_Scene *scene) { for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { Canvas_Web_View *view = &surface->views[index]; - view->active = view->entity_id != 0 && - SceneHasVisibleEntity(scene, camera, view->entity_id); + view->active = view->is_dictation_transport || + (view->entity_id != 0 && + SceneHasVisibleEntity(scene, camera, view->entity_id)); } for (size_t index = 0; index < Dowa_Array_Length(scene->p_entities); ++index) { @@ -562,7 +730,7 @@ Canvas_Web_View *view = &surface->views[index]; CefRefPtr host = client->browser()->GetHost(); host->WasHidden(!view->active); - if (view->active) { + if (view->active && !view->is_dictation_transport) { const Canvas_Entity *entity = FindSurfaceEntity(scene, view->entity_id); if (entity) { host->SetWindowlessFrameRate( @@ -805,12 +973,22 @@ const Canvas_Scene *p_scene) { if (!p_surface->initialized) return; CefDoMessageLoopWork(); + EnsureDictationView(p_surface); ReconcileViews(p_surface, p_camera, p_scene); + if (p_surface->pending_dictation_command != 0) { + Canvas_Web_View *view = EnsureDictationView(p_surface); + if (view) { + view->dictation_command = p_surface->pending_dictation_command; + p_surface->pending_dictation_command = 0; + } + } for (int32 index = 0; index < CANVAS_MAX_WEB_VIEWS; ++index) { Canvas_Web_View *view = &p_surface->views[index]; if (!view->browser_created) continue; - if (!view->texture_ready && IsWindowReady()) { + if (!view->is_dictation_transport && + !view->texture_ready && + IsWindowReady()) { Image image = { .data = view->p_pixels, .width = view->texture_width, @@ -824,7 +1002,9 @@ SetTextureFilter(view->texture, TEXTURE_FILTER_BILINEAR); } } - if (view->texture_ready && view->pixels_dirty) { + if (!view->is_dictation_transport && + view->texture_ready && + view->pixels_dirty) { UpdateTexture(view->texture, view->p_pixels); view->pixels_dirty = FALSE; } @@ -840,6 +1020,29 @@ } view->can_go_back = client->browser()->CanGoBack() ? TRUE : FALSE; view->can_go_forward = client->browser()->CanGoForward() ? TRUE : FALSE; + if (view->dictation_command != 0) { + CefRefPtr frame = client->browser()->GetMainFrame(); + const char *function_name = + view->dictation_command == 2 + ? "ZenbuDictationCommit" + : (view->dictation_command > 0 + ? "ZenbuDictationStart" + : "ZenbuDictationStop"); + char script[512]; + std::snprintf( + script, + sizeof(script), + "(()=>{let attempts=0;const invoke=()=>{" + "if(window.%s){window.%s();return;}" + "if(attempts++<40)setTimeout(invoke,250);};invoke();})()", + function_name, + function_name); + frame->ExecuteJavaScript( + script, + frame->GetURL(), + 0); + view->dictation_command = 0; + } } } } @@ -955,6 +1158,42 @@ } } +extern "C" void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active) { + int32 command = active ? 1 : -1; + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (view) { + view->dictation_command = command; + } else { + p_surface->pending_dictation_command = command; + } +} + +extern "C" void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface) { + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (view) { + view->dictation_command = 2; + } else { + p_surface->pending_dictation_command = 2; + } +} + +extern "C" Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity) { + Canvas_Web_View *view = FindView(p_surface, kDictationViewId); + if (!view || + view->dictation_sequence == view->consumed_dictation_sequence) { + return CANVAS_DICTATION_EVENT_NONE; + } + snprintf(p_text, text_capacity, "%s", view->dictation_text); + view->consumed_dictation_sequence = view->dictation_sequence; + return view->dictation_event; +} + extern "C" boolean Canvas_Web_Surface_Is_Focused( const Canvas_Web_Surface *p_surface) { return p_surface->focused_entity_id != 0; diff -r b3b547563ec7 -r 49e9e591c9bb infinite_canvas/web_surface_web.c --- a/infinite_canvas/web_surface_web.c Mon Aug 17 22:22:36 2026 -0700 +++ b/infinite_canvas/web_surface_web.c Tue Aug 18 19:14:53 2026 -0700 @@ -9,6 +9,69 @@ }); }); +EM_JS(void, Canvas_Web_Dictation_Init, (), { + if (Module.canvasDictationInitialized) return; + Module.canvasDictationInitialized = true; + Module.canvasDictationQueue = []; + window.addEventListener("message", event => { + if (event.data?.type !== "zenbu.dictation.event") return; + const frame = document.getElementById("canvas-dictation-transport"); + if (!frame || frame.contentWindow !== event.source) return; + Module.canvasDictationQueue.push({ + kind: String(event.data.kind || ""), + text: String(event.data.text || ""), + }); + }); +}); + +EM_JS(void, Canvas_Web_Dictation_Set_Active, (int active), { + let frame = document.getElementById("canvas-dictation-transport"); + if (!frame) { + frame = document.createElement("iframe"); + frame.id = "canvas-dictation-transport"; + frame.title = "Canvas dictation transport"; + frame.allow = "microphone"; + frame.sandbox = + "allow-same-origin allow-scripts"; + frame.style.position = "fixed"; + frame.style.left = "-2px"; + frame.style.top = "-2px"; + frame.style.width = "1px"; + frame.style.height = "1px"; + frame.style.opacity = "0"; + frame.style.pointerEvents = "none"; + frame.src = "http://127.0.0.1:8090/?canvas=1"; + document.body.appendChild(frame); + } + const send = () => frame.contentWindow.postMessage( + { + type: "zenbu.dictation.set-active", + active: active !== 0, + }, + "*"); + frame.addEventListener("load", send, {once: true}); + send(); +}); + +EM_JS(void, Canvas_Web_Dictation_Commit, (), { + const frame = document.getElementById("canvas-dictation-transport"); + frame?.contentWindow?.postMessage( + {type: "zenbu.dictation.commit"}, + "*"); +}); + +EM_JS(int, Canvas_Web_Dictation_Consume, ( + char *p_text, + int text_capacity), { + const event = Module.canvasDictationQueue?.shift(); + if (!event) return 0; + stringToUTF8(event.text, p_text, text_capacity); + if (event.kind === "status") return 1; + if (event.kind === "partial") return 2; + if (event.kind === "final") return 3; + return 0; +}); + EM_JS(void, Canvas_Web_Create_Or_Update, ( uint32 entity_id, const char *p_url, @@ -170,6 +233,7 @@ } surface.remove(); }); + document.getElementById("canvas-dictation-transport")?.remove(); }); static Rectangle Canvas_Web_Content_Bounds( @@ -236,6 +300,7 @@ p_surface->p_arena = p_arena; p_surface->dark_mode = dark_mode; p_surface->initialized = TRUE; + Canvas_Web_Dictation_Init(); return TRUE; } @@ -369,6 +434,32 @@ Canvas_Web_History_Go(entity_id, 1); } +void Canvas_Web_Surface_Set_Dictation_Active( + Canvas_Web_Surface *p_surface, + boolean active) +{ + (void)p_surface; + Canvas_Web_Dictation_Set_Active(active); +} + +void Canvas_Web_Surface_Commit_Dictation( + Canvas_Web_Surface *p_surface) +{ + (void)p_surface; + Canvas_Web_Dictation_Commit(); +} + +Canvas_Dictation_Event Canvas_Web_Surface_Consume_Dictation( + Canvas_Web_Surface *p_surface, + char *p_text, + size_t text_capacity) +{ + (void)p_surface; + return (Canvas_Dictation_Event)Canvas_Web_Dictation_Consume( + p_text, + (int)text_capacity); +} + boolean Canvas_Web_Surface_Is_Focused(const Canvas_Web_Surface *p_surface) { (void)p_surface; diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/BUILD --- a/mrjunejune/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -331,7 +331,10 @@ hdrs = ["inference_bridge.h"], deps = ["//dowa:dowa"], linkopts = ["-lpthread"], - visibility = ["//mrjunejune/test:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune/test:__pkg__", + ], ) filegroup( @@ -356,6 +359,12 @@ ], ) +filegroup( + name = "canvas_orchestration_launcher", + srcs = ["inference_sidecar_launcher.sh"], + visibility = ["//infinite_canvas:__pkg__"], +) + # Server binary cc_binary( name = "mrjunejune_server", diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/inference/BUILD --- a/mrjunejune/inference/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -13,7 +13,10 @@ exports_files( ["litellm_config.yaml"], - visibility = ["//mrjunejune:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune:__pkg__", + ], ) py_binary( @@ -63,13 +66,19 @@ python_zip_file( name = "copilot_sidecar_zip", binary = ":copilot_sidecar", - visibility = ["//mrjunejune:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune:__pkg__", + ], ) python_zip_file( name = "litellm_proxy_zip", binary = ":litellm_proxy", - visibility = ["//mrjunejune:__pkg__"], + visibility = [ + "//infinite_canvas:__pkg__", + "//mrjunejune:__pkg__", + ], ) filegroup( diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/inference/copilot_sidecar.py --- a/mrjunejune/inference/copilot_sidecar.py Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference/copilot_sidecar.py Tue Aug 18 19:14:53 2026 -0700 @@ -2,6 +2,7 @@ import argparse import asyncio +import hashlib import json import os import sys @@ -20,8 +21,27 @@ JsonObject = dict[str, Any] Emit = Callable[[JsonObject], Awaitable[None]] -_KNOWN_PROFILES: frozenset = frozenset({"public_visitor", "invited_friend", "june_admin"}) +_KNOWN_PROFILES: frozenset = frozenset({ + "public_visitor", + "invited_friend", + "june_admin", + "canvas_orchestrator", +}) _PROMPT_VERSION: int = 1 +_CANVAS_ORCHESTRATOR_PROMPT = """\ +You orchestrate spatial agent sessions on an infinite canvas. +There is one user input scratchpad and any number of conversation entities. +The user message includes only the visible canvas context plus one new thought. +Treat visible conversation entities as the only append candidates. Append when +the new thought clearly continues one of them; otherwise create a new +conversation. Never append to a conversation ID absent from visible context. +Return only one compact JSON object with these fields: +- action: "create" or "append" +- conversation_id: an existing numeric conversation ID for append, otherwise 0 +- title: a short session title +- response: the direct response that should appear on the canvas +Do not use Markdown fences, tools, or text outside the JSON object. +""" # Fixed namespace for uuid5 SDK session ID derivation. Must never change. _SDK_SESSION_NAMESPACE = uuid.UUID("3f7e8a1d-9b52-4c6f-a0d3-82e1f5c94b7a") @@ -254,7 +274,17 @@ def _compile_all_profiles(self) -> None: compiled: dict[str, CompiledProfile] = {} for profile in sorted(_KNOWN_PROFILES): - result = self._compile_fn(profile) + if profile == "canvas_orchestrator": + content = _CANVAS_ORCHESTRATOR_PROMPT.strip() + result = { + "content": content, + "version": 1, + "hash": hashlib.sha256( + content.encode("utf-8") + ).hexdigest(), + } + else: + result = self._compile_fn(profile) compiled[profile] = CompiledProfile( profile=profile, content=result["content"], @@ -327,7 +357,9 @@ await self.evict_idle_sessions( exclude=conversation_id, - reserve=1 if command_name == "turn.start" else 0, + reserve=1 + if command_name in {"turn.start", "conversation.warm"} + else 0, ) async with self._conversation_command_lock(conversation_id): if command_name == "turn.start": @@ -365,6 +397,45 @@ ) return await self._start_turn(request_id, conversation_id, prompt, compiled, history) + elif command_name == "conversation.warm": + compiled = self._validate_profile_fields( + command.get("prompt_profile"), + command.get("prompt_version"), + command.get("knowledge_version"), + ) + if compiled is None: + await self._fail( + request_id, + conversation_id, + "invalid_prompt_profile", + "a known prompt profile and current versions are required", + ) + return + resume_existing = command.get("resume_existing", True) + if not isinstance(resume_existing, bool): + await self._fail( + request_id, + conversation_id, + "invalid_request", + "resume_existing must be a boolean", + ) + return + await self._get_conversation( + conversation_id, + compiled, + [], + resume_existing=resume_existing, + ) + await self._send( + "session.warmed", + request_id, + conversation_id, + ) + await self._send( + "turn.done", + request_id, + conversation_id, + ) elif command_name == "turn.abort": await self._abort_turn(request_id, conversation_id) elif command_name == "conversation.delete": @@ -486,6 +557,8 @@ conversation_id: str, compiled: CompiledProfile, history: list[dict[str, str]], + *, + resume_existing: bool = True, ) -> Conversation: derived_id = _derive_sdk_session_id(conversation_id, compiled) async with self._conversations_lock: @@ -508,12 +581,13 @@ raise RuntimeError("Copilot session capacity exhausted") base_options = self._session_options(compiled) - # Try to resume a persisted derived session (no history injection). session: Any = None - try: - session = await self._client.resume_session(derived_id, **base_options) - except Exception: - pass + if resume_existing: + # Resume persisted user-facing sessions, then create if absent. + try: + session = await self._client.resume_session(derived_id, **base_options) + except Exception: + pass if session is None: # Fresh create — inject bounded transcript context into system message. diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/inference/copilot_sidecar_test.py --- a/mrjunejune/inference/copilot_sidecar_test.py Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference/copilot_sidecar_test.py Tue Aug 18 19:14:53 2026 -0700 @@ -10,6 +10,7 @@ Sidecar, SidecarConfig, _derive_sdk_session_id, + _CANVAS_ORCHESTRATOR_PROMPT, _SDK_SESSION_NAMESPACE, _validate_history, _HISTORY_MAX_ENTRIES, @@ -50,7 +51,15 @@ def _fake_sdk_id(conversation_id: str, profile: str) -> str: """Compute the deterministic SDK session ID for use in test assertions.""" - p = _FAKE_PROFILES[profile] + if profile == "canvas_orchestrator": + content = _CANVAS_ORCHESTRATOR_PROMPT.strip() + p = { + "content": content, + "version": 1, + "hash": hashlib.sha256(content.encode()).hexdigest(), + } + else: + p = _FAKE_PROFILES[profile] compiled = CompiledProfile( profile=profile, content=p["content"], @@ -210,6 +219,54 @@ ["turn.error", "turn.done"], ) + async def test_warm_creates_session_without_sending_prompt(self): + sidecar, client = await self.make_sidecar() + await sidecar.dispatch( + { + "command": "conversation.warm", + "request_id": "warm-1", + "conversation_id": "canvas-worker-1", + "prompt_profile": "canvas_orchestrator", + "prompt_version": 1, + "knowledge_version": 1, + "resume_existing": False, + } + ) + + self.assertEqual(len(client.create_calls), 1) + self.assertEqual(len(client.resume_calls), 0) + session = next(iter(client.sessions.values())) + self.assertEqual(session.prompts, []) + self.assertEqual( + [item["type"] for item in self.output[-2:]], + ["session.warmed", "turn.done"], + ) + + async def test_warm_rejects_non_boolean_resume_existing(self): + sidecar, client = await self.make_sidecar() + await sidecar.dispatch( + { + "command": "conversation.warm", + "request_id": "warm-invalid", + "conversation_id": "canvas-worker-1", + "prompt_profile": "canvas_orchestrator", + "prompt_version": 1, + "knowledge_version": 1, + "resume_existing": "false", + } + ) + + self.assertEqual(len(client.create_calls), 0) + self.assertEqual(len(client.resume_calls), 0) + self.assertEqual( + [item["type"] for item in self.output[-2:]], + ["turn.error", "turn.done"], + ) + self.assertEqual( + self.output[-2]["error"]["message"], + "resume_existing must be a boolean", + ) + async def test_multi_turn_reuses_one_session_and_configures_sdk(self): sidecar, client = await self.make_sidecar() for request_id, prompt in (("r1", "first"), ("r2", "second")): @@ -411,6 +468,7 @@ _fake_sdk_id("delete-me", "public_visitor"), _fake_sdk_id("delete-me", "invited_friend"), _fake_sdk_id("delete-me", "june_admin"), + _fake_sdk_id("delete-me", "canvas_orchestrator"), }, ) self.assertTrue(client.stopped) @@ -958,6 +1016,7 @@ _fake_sdk_id("conv-del-check", "public_visitor"), _fake_sdk_id("conv-del-check", "invited_friend"), _fake_sdk_id("conv-del-check", "june_admin"), + _fake_sdk_id("conv-del-check", "canvas_orchestrator"), }, ) self.assertNotIn("conv-del-check", client.delete_calls) @@ -977,6 +1036,7 @@ _fake_sdk_id("ghost-conv", "public_visitor"), _fake_sdk_id("ghost-conv", "invited_friend"), _fake_sdk_id("ghost-conv", "june_admin"), + _fake_sdk_id("ghost-conv", "canvas_orchestrator"), }, ) self.assertNotIn("ghost-conv", client.delete_calls) diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/inference_bridge.c --- a/mrjunejune/inference_bridge.c Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference_bridge.c Tue Aug 18 19:14:53 2026 -0700 @@ -198,6 +198,8 @@ return "invited_friend"; case INFERENCE_PROMPT_PROFILE_JUNE_ADMIN: return "june_admin"; + case INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR: + return "canvas_orchestrator"; } return NULL; } @@ -210,7 +212,9 @@ const char *prompt, const char *prompt_profile, uint32 prompt_version, - uint32 knowledge_version) + uint32 knowledge_version, + boolean include_resume_existing, + boolean resume_existing) { if (!command || !request_id) return FALSE; @@ -269,6 +273,27 @@ prompt_version, knowledge_version); } + else if (escaped_profile) + { + snprintf( + payload, + capacity, + "{\"command\":\"%s\",\"request_id\":\"%s\"," + "\"conversation_id\":\"%s\"," + "\"prompt_profile\":\"%s\",\"prompt_version\":%u," + "\"knowledge_version\":%u%s}", + escaped_command, + escaped_request, + escaped_conversation, + escaped_profile, + prompt_version, + knowledge_version, + include_resume_existing + ? (resume_existing + ? ",\"resume_existing\":true" + : ",\"resume_existing\":false") + : ""); + } else { snprintf( @@ -333,7 +358,8 @@ return; if (atomic_load(&p_bridge->running) && p_bridge->p_commands) Inference_Bridge_Command( - p_bridge, "shutdown", "server-shutdown", "", NULL, NULL, 0, 0); + p_bridge, "shutdown", "server-shutdown", "", NULL, NULL, 0, 0, + FALSE, FALSE); if (p_bridge->p_commands) { fclose(p_bridge->p_commands); @@ -642,13 +668,40 @@ return success; } +boolean Inference_Bridge_Warm_Conversation( + Inference_Bridge *p_bridge, + const char *request_id, + const char *conversation_id, + Inference_Prompt_Profile prompt_profile, + uint32 prompt_version, + uint32 knowledge_version, + boolean resume_existing) +{ + const char *profile = Inference_Bridge_Profile_Name(prompt_profile); + if (!p_bridge || !request_id || !request_id[0] || + !conversation_id || !conversation_id[0] || !profile) + return FALSE; + return Inference_Bridge_Command( + p_bridge, + "conversation.warm", + request_id, + conversation_id, + NULL, + profile, + prompt_version, + knowledge_version, + TRUE, + resume_existing); +} + boolean Inference_Bridge_Abort_Turn( Inference_Bridge *p_bridge, const char *request_id, const char *conversation_id) { return Inference_Bridge_Command( - p_bridge, "turn.abort", request_id, conversation_id, NULL, NULL, 0, 0); + p_bridge, "turn.abort", request_id, conversation_id, NULL, NULL, 0, 0, + FALSE, FALSE); } boolean Inference_Bridge_Delete_Conversation( @@ -664,7 +717,9 @@ NULL, NULL, 0, - 0); + 0, + FALSE, + FALSE); } void Inference_Bridge_Destroy(Inference_Bridge *p_bridge) diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/inference_bridge.h --- a/mrjunejune/inference_bridge.h Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/inference_bridge.h Tue Aug 18 19:14:53 2026 -0700 @@ -17,6 +17,7 @@ INFERENCE_PROMPT_PROFILE_PUBLIC_VISITOR = 0, INFERENCE_PROMPT_PROFILE_INVITED_FRIEND = 1, INFERENCE_PROMPT_PROFILE_JUNE_ADMIN = 2, + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR = 3, } Inference_Prompt_Profile; typedef struct { @@ -68,6 +69,15 @@ const Inference_Bridge_History_Message *p_history, uint32 history_count); +boolean Inference_Bridge_Warm_Conversation( + Inference_Bridge *p_bridge, + const char *request_id, + const char *conversation_id, + Inference_Prompt_Profile prompt_profile, + uint32 prompt_version, + uint32 knowledge_version, + boolean resume_existing); + boolean Inference_Bridge_Abort_Turn( Inference_Bridge *p_bridge, const char *request_id, diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/test/BUILD --- a/mrjunejune/test/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/test/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -77,6 +77,7 @@ name = "inference_bridge_fake_sidecar", srcs = ["inference_bridge_fake_sidecar.c"], deps = ["//dowa:dowa"], + visibility = ["//infinite_canvas:__pkg__"], ) cc_test( diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/test/inference_bridge_fake_sidecar.c --- a/mrjunejune/test/inference_bridge_fake_sidecar.c Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/test/inference_bridge_fake_sidecar.c Tue Aug 18 19:14:53 2026 -0700 @@ -100,6 +100,22 @@ Dowa_Arena_Free(p_arena); break; } + if (strcmp(command, "conversation.warm") == 0) + { + printf( + "{\"type\":\"session.warmed\",\"request_id\":\"%s\"," + "\"conversation_id\":\"%s\"}\n", + request_id, + conversation_id); + printf( + "{\"type\":\"turn.done\",\"request_id\":\"%s\"," + "\"conversation_id\":\"%s\"}\n", + request_id, + conversation_id); + fflush(stdout); + Dowa_Arena_Free(p_arena); + continue; + } if (strcmp(command, "turn.start") == 0) { /* Validate history before any other processing. */ @@ -131,7 +147,8 @@ boolean known_profile = strcmp(prompt_profile, "public_visitor") == 0 || strcmp(prompt_profile, "invited_friend") == 0 || - strcmp(prompt_profile, "june_admin") == 0; + strcmp(prompt_profile, "june_admin") == 0 || + strcmp(prompt_profile, "canvas_orchestrator") == 0; if (!known_profile || prompt_version != 1 || knowledge_version != 1) { printf( diff -r b3b547563ec7 -r 49e9e591c9bb mrjunejune/test/inference_bridge_test.c --- a/mrjunejune/test/inference_bridge_test.c Mon Aug 17 22:22:36 2026 -0700 +++ b/mrjunejune/test/inference_bridge_test.c Tue Aug 18 19:14:53 2026 -0700 @@ -13,6 +13,7 @@ int deltas; int completed; int usage; + int warmed; int custom; int done; int closed; @@ -32,6 +33,8 @@ pthread_mutex_lock(&p_state->mutex); if (strcmp(p_event->type, "turn.accepted") == 0) p_state->accepted++; + else if (strcmp(p_event->type, "session.warmed") == 0) + p_state->warmed++; else if (strcmp(p_event->type, "assistant.delta") == 0) { p_state->deltas++; @@ -87,6 +90,21 @@ pthread_mutex_unlock(&p_state->mutex); } +static void Wait_Warmed(Test_State *p_state, int target) +{ + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += 5; + pthread_mutex_lock(&p_state->mutex); + while (p_state->warmed < target) + { + int result = pthread_cond_timedwait( + &p_state->condition, &p_state->mutex, &deadline); + assert(result == 0); + } + pthread_mutex_unlock(&p_state->mutex); +} + int main(int argc, char **argv) { assert(argc == 2); @@ -100,6 +118,16 @@ assert(Inference_Bridge_Start(p_bridge)); assert(Inference_Bridge_Is_Ready(p_bridge)); + assert(Inference_Bridge_Warm_Conversation( + p_bridge, + "warm-1", + "warm-conversation", + INFERENCE_PROMPT_PROFILE_CANVAS_ORCHESTRATOR, + 1, + 1, + FALSE)); + Wait_Warmed(&state, 1); + /* --- Validation: invalid profile --- */ assert(!Inference_Bridge_Start_Turn( p_bridge, diff -r b3b547563ec7 -r 49e9e591c9bb qwen3_vl/BUILD --- a/qwen3_vl/BUILD Mon Aug 17 22:22:36 2026 -0700 +++ b/qwen3_vl/BUILD Tue Aug 18 19:14:53 2026 -0700 @@ -58,6 +58,17 @@ data = [":llama_cpp_cuda_runtime"], ) +sh_binary( + name = "serve_canvas", + srcs = ["qwen3_vl.sh"], + args = [ + "$(rootpath :llama_cpp_cuda_runtime)", + "serve_canvas", + ], + data = [":llama_cpp_cuda_runtime"], + visibility = ["//infinite_canvas:__pkg__"], +) + sh_test( name = "qwen3_vl_test", srcs = ["qwen3_vl_test.sh"], diff -r b3b547563ec7 -r 49e9e591c9bb qwen3_vl/README.md --- a/qwen3_vl/README.md Mon Aug 17 22:22:36 2026 -0700 +++ b/qwen3_vl/README.md Tue Aug 18 19:14:53 2026 -0700 @@ -40,6 +40,15 @@ curl.exe http://127.0.0.1:8080/health ``` +For the Linux Infinite Canvas process, start the server on the Windows WSL +adapter instead: + +```bash +bazel run //qwen3_vl:serve_canvas +``` + +The canvas detects the WSL gateway and uses that address by default. + The pinned CUDA executable runs on the Windows side of WSL interop. Windows browsers and clients can use `http://127.0.0.1:8080`; use `curl.exe`, rather than Linux `curl`, when checking it from a WSL shell. diff -r b3b547563ec7 -r 49e9e591c9bb qwen3_vl/qwen3_vl.sh --- a/qwen3_vl/qwen3_vl.sh Mon Aug 17 22:22:36 2026 -0700 +++ b/qwen3_vl/qwen3_vl.sh Tue Aug 18 19:14:53 2026 -0700 @@ -146,13 +146,21 @@ --top-p 0.8 \ "${NORMALIZED_ARGS[@]}" ;; - serve) + serve|serve_canvas) require_wsl require_model + host="${QWEN3_VL_HOST:-127.0.0.1}" + if [[ "$command" == "serve_canvas" && -z "${QWEN3_VL_HOST:-}" ]]; then + host="$(ip route show default | awk '{print $3; exit}')" + if [[ -z "$host" ]]; then + echo "Unable to resolve the Windows host address for WSL." >&2 + exit 1 + fi + fi exec "$runtime/llama-server.exe" \ --model "$(windows_path "$model_path")" \ --mmproj "$(windows_path "$mmproj_path")" \ - --host "${QWEN3_VL_HOST:-127.0.0.1}" \ + --host "$host" \ --port "${QWEN3_VL_PORT:-8080}" \ --ctx-size "${QWEN3_VL_CONTEXT_SIZE:-2048}" \ --parallel 1 \