Mercurial
view dictation/web/dictation.js @ 280:49e9e591c9bb
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Tue, 18 Aug 2026 19:14:53 -0700 |
| parents | 78699f810817 |
| children |
line wrap: on
line source
const startButton = document.querySelector("[data-start]"); const stopButton = document.querySelector("[data-stop]"); const status = document.querySelector("[data-status]"); const partial = document.querySelector("[data-partial]"); const finalText = document.querySelector("[data-final]"); let peer = null; let channel = null; let stream = null; let sessionId = null; let stopResolver = null; let captureActive = false; let connectionPromise = null; const canvasTransport = new URLSearchParams(window.location.search).get("canvas") === "1"; function publishDictation(kind, text) { const payload = { type: "zenbu.dictation.event", kind, text, }; window.parent?.postMessage(payload, "*"); document.title = `Zenbu Dictation|${Date.now()}|${kind}|${encodeURIComponent(text)}`; } function setStatus(message, state = "") { status.textContent = message; status.dataset.state = state; publishDictation("status", message); } function waitForIceGathering(connection) { if (connection.iceGatheringState === "complete") return Promise.resolve(); return new Promise(resolve => { const listener = () => { if (connection.iceGatheringState !== "complete") return; connection.removeEventListener("icegatheringstatechange", listener); resolve(); }; connection.addEventListener("icegatheringstatechange", listener); }); } function waitForTranscriptChannel(transcriptChannel) { if (transcriptChannel.readyState === "open") return Promise.resolve(); return new Promise((resolve, reject) => { const timeout = window.setTimeout( () => reject(new Error("Transcript channel readiness timed out")), 10000, ); transcriptChannel.addEventListener("open", () => { window.clearTimeout(timeout); resolve(); }, { once: true }); transcriptChannel.addEventListener("close", () => { window.clearTimeout(timeout); reject(new Error("Transcript channel closed during startup")); }, { once: true }); }); } function handleTranscript(message) { let event; try { event = JSON.parse(message.data); } catch { setStatus("Received an invalid server event", "error"); return; } switch (event.type) { case "ready": setStatus(captureActive ? "Listening" : "Idle", "ready"); break; case "speech.started": setStatus("Speech detected", "working"); break; case "transcript.partial": partial.textContent = event.text; publishDictation("partial", event.text); break; case "transcript.final": finalText.value += `${finalText.value ? " " : ""}${event.text}`; publishDictation("final", event.text); partial.textContent = "Waiting for speech..."; break; case "speech.ended": setStatus(captureActive ? "Listening" : "Idle", "ready"); stopResolver?.(); stopResolver = null; break; case "error": setStatus(event.message || "Transcription failed", "error"); break; } } async function createConnection() { if (peer && !["closed", "failed"].includes(peer.connectionState)) return; if (!stream) { stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, }, video: false, }); } for (const track of stream.getTracks()) track.enabled = false; try { peer = new RTCPeerConnection(); channel = peer.createDataChannel("transcripts", { ordered: true }); channel.addEventListener("message", handleTranscript); channel.addEventListener("close", () => { if (peer) setStatus("Transcript channel closed", "error"); }); peer.addEventListener("connectionstatechange", () => { if (peer?.connectionState === "failed") { setStatus("WebRTC connection failed", "error"); void stop(false, true); } }); for (const track of stream.getTracks()) peer.addTrack(track, stream); const offer = await peer.createOffer(); await peer.setLocalDescription(offer); await waitForIceGathering(peer); const response = await fetch("/api/webrtc/offer", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(peer.localDescription), }); if (!response.ok) { const detail = await response.text(); throw new Error(`Signaling failed (${response.status}): ${detail}`); } const answer = await response.json(); sessionId = answer.sessionId; await peer.setRemoteDescription({ sdp: answer.sdp, type: answer.type, }); await waitForTranscriptChannel(channel); stopButton.disabled = false; } catch (error) { await stop(false, true); throw error; } } async function ensureConnection() { if (peer && ["connected", "completed"].includes(peer.connectionState)) return; if (!connectionPromise) { connectionPromise = createConnection().finally(() => { connectionPromise = null; }); } await connectionPromise; } async function start() { captureActive = true; startButton.disabled = true; setStatus("Starting microphone...", "working"); try { await ensureConnection(); for (const track of stream.getTracks()) track.enabled = true; setStatus( channel?.readyState === "open" ? "Listening" : "Connecting...", channel?.readyState === "open" ? "ready" : "working", ); } catch (error) { captureActive = false; setStatus(error.message || "Unable to start dictation", "error"); } } async function stop(waitForFinal = true, releaseStream = !canvasTransport) { const closingSession = sessionId; const tracks = stream?.getTracks() || []; captureActive = false; if (channel?.readyState === "open" && waitForFinal) { const ended = new Promise(resolve => { stopResolver = resolve; }); channel.send(releaseStream ? "stop" : "pause"); for (const track of tracks) track.enabled = false; await Promise.race([ ended, new Promise(resolve => window.setTimeout(resolve, 10000)), ]); stopResolver = null; } for (const track of tracks) { track.enabled = false; if (releaseStream) track.stop(); } if (!releaseStream) { startButton.disabled = false; stopButton.disabled = true; setStatus("Idle"); return; } sessionId = null; if (releaseStream) stream = null; channel?.close(); channel = null; peer?.close(); peer = null; if (closingSession) { await fetch(`/api/webrtc/session/${encodeURIComponent(closingSession)}/close`, { method: "POST", keepalive: true, }).catch(() => {}); } startButton.disabled = false; stopButton.disabled = true; setStatus("Idle"); } startButton.addEventListener("click", () => void start()); stopButton.addEventListener("click", () => void stop()); window.ZenbuDictationStart = () => start(); window.ZenbuDictationStop = () => stop(); window.ZenbuDictationCommit = () => { if (channel?.readyState === "open") channel.send("commit"); }; window.ZenbuDictationToggle = () => captureActive ? stop() : start(); window.addEventListener("message", event => { if (event.data?.type === "zenbu.dictation.set-active") { void (event.data.active ? start() : stop()); } else if (event.data?.type === "zenbu.dictation.commit") { window.ZenbuDictationCommit(); } }); window.addEventListener("pagehide", () => void stop(false, true)); async function prepareCanvasMicrophone() { if (!canvasTransport || peer) return; try { await ensureConnection(); for (const track of stream.getTracks()) track.enabled = false; setStatus("Idle"); } catch (error) { setStatus(error.message || "Unable to prepare microphone", "error"); } } void prepareCanvasMicrophone();