Mercurial
diff dictation/web/dictation.js @ 280:49e9e591c9bb
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Tue, 18 Aug 2026 19:14:53 -0700 |
| parents | 78699f810817 |
| children |
line wrap: on
line diff
--- a/dictation/web/dictation.js Mon Aug 17 22:22:36 2026 -0700 +++ b/dictation/web/dictation.js Tue Aug 18 19:14:53 2026 -0700 @@ -9,10 +9,26 @@ let stream = null; let sessionId = null; let stopResolver = null; +let captureActive = false; +let connectionPromise = null; +const canvasTransport = + new URLSearchParams(window.location.search).get("canvas") === "1"; + +function publishDictation(kind, text) { + const payload = { + type: "zenbu.dictation.event", + kind, + text, + }; + window.parent?.postMessage(payload, "*"); + document.title = + `Zenbu Dictation|${Date.now()}|${kind}|${encodeURIComponent(text)}`; +} function setStatus(message, state = "") { status.textContent = message; status.dataset.state = state; + publishDictation("status", message); } function waitForIceGathering(connection) { @@ -27,6 +43,24 @@ }); } +function waitForTranscriptChannel(transcriptChannel) { + if (transcriptChannel.readyState === "open") return Promise.resolve(); + return new Promise((resolve, reject) => { + const timeout = window.setTimeout( + () => reject(new Error("Transcript channel readiness timed out")), + 10000, + ); + transcriptChannel.addEventListener("open", () => { + window.clearTimeout(timeout); + resolve(); + }, { once: true }); + transcriptChannel.addEventListener("close", () => { + window.clearTimeout(timeout); + reject(new Error("Transcript channel closed during startup")); + }, { once: true }); + }); +} + function handleTranscript(message) { let event; try { @@ -37,20 +71,22 @@ } switch (event.type) { case "ready": - setStatus("Listening", "ready"); + setStatus(captureActive ? "Listening" : "Idle", "ready"); break; case "speech.started": setStatus("Speech detected", "working"); break; case "transcript.partial": partial.textContent = event.text; + publishDictation("partial", event.text); break; case "transcript.final": finalText.value += `${finalText.value ? " " : ""}${event.text}`; + publishDictation("final", event.text); partial.textContent = "Waiting for speech..."; break; case "speech.ended": - setStatus("Listening", "ready"); + setStatus(captureActive ? "Listening" : "Idle", "ready"); stopResolver?.(); stopResolver = null; break; @@ -60,11 +96,9 @@ } } -async function start() { - if (peer) return; - startButton.disabled = true; - setStatus("Requesting microphone access...", "working"); - try { +async function createConnection() { + if (peer && !["closed", "failed"].includes(peer.connectionState)) return; + if (!stream) { stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, @@ -73,6 +107,9 @@ }, video: false, }); + } + for (const track of stream.getTracks()) track.enabled = false; + try { peer = new RTCPeerConnection(); channel = peer.createDataChannel("transcripts", { ordered: true }); channel.addEventListener("message", handleTranscript); @@ -82,7 +119,7 @@ peer.addEventListener("connectionstatechange", () => { if (peer?.connectionState === "failed") { setStatus("WebRTC connection failed", "error"); - void stop(); + void stop(false, true); } }); for (const track of stream.getTracks()) peer.addTrack(track, stream); @@ -105,23 +142,50 @@ sdp: answer.sdp, type: answer.type, }); + await waitForTranscriptChannel(channel); stopButton.disabled = false; - setStatus("Connecting...", "working"); } catch (error) { - setStatus(error.message || "Unable to start dictation", "error"); - await stop(); + await stop(false, true); + throw error; } } -async function stop(waitForFinal = true) { +async function ensureConnection() { + if (peer && ["connected", "completed"].includes(peer.connectionState)) return; + if (!connectionPromise) { + connectionPromise = createConnection().finally(() => { + connectionPromise = null; + }); + } + await connectionPromise; +} + +async function start() { + captureActive = true; + startButton.disabled = true; + setStatus("Starting microphone...", "working"); + try { + await ensureConnection(); + for (const track of stream.getTracks()) track.enabled = true; + setStatus( + channel?.readyState === "open" ? "Listening" : "Connecting...", + channel?.readyState === "open" ? "ready" : "working", + ); + } catch (error) { + captureActive = false; + setStatus(error.message || "Unable to start dictation", "error"); + } +} + +async function stop(waitForFinal = true, releaseStream = !canvasTransport) { const closingSession = sessionId; - sessionId = null; const tracks = stream?.getTracks() || []; + captureActive = false; if (channel?.readyState === "open" && waitForFinal) { const ended = new Promise(resolve => { stopResolver = resolve; }); - channel.send("stop"); + channel.send(releaseStream ? "stop" : "pause"); for (const track of tracks) track.enabled = false; await Promise.race([ ended, @@ -129,8 +193,18 @@ ]); stopResolver = null; } - for (const track of tracks) track.stop(); - stream = null; + for (const track of tracks) { + track.enabled = false; + if (releaseStream) track.stop(); + } + if (!releaseStream) { + startButton.disabled = false; + stopButton.disabled = true; + setStatus("Idle"); + return; + } + sessionId = null; + if (releaseStream) stream = null; channel?.close(); channel = null; peer?.close(); @@ -148,4 +222,30 @@ startButton.addEventListener("click", () => void start()); stopButton.addEventListener("click", () => void stop()); -window.addEventListener("pagehide", () => void stop(false)); +window.ZenbuDictationStart = () => start(); +window.ZenbuDictationStop = () => stop(); +window.ZenbuDictationCommit = () => { + if (channel?.readyState === "open") channel.send("commit"); +}; +window.ZenbuDictationToggle = () => captureActive ? stop() : start(); +window.addEventListener("message", event => { + if (event.data?.type === "zenbu.dictation.set-active") { + void (event.data.active ? start() : stop()); + } else if (event.data?.type === "zenbu.dictation.commit") { + window.ZenbuDictationCommit(); + } +}); +window.addEventListener("pagehide", () => void stop(false, true)); + +async function prepareCanvasMicrophone() { + if (!canvasTransport || peer) return; + try { + await ensureConnection(); + for (const track of stream.getTracks()) track.enabled = false; + setStatus("Idle"); + } catch (error) { + setStatus(error.message || "Unable to prepare microphone", "error"); + } +} + +void prepareCanvasMicrophone();