Mercurial

comparison dictation/web/dictation.js @ 280:49e9e591c9bb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
author MrJuneJune <me@mrjunejune.com>
date Tue, 18 Aug 2026 19:14:53 -0700
parents 78699f810817
children
comparison
equal deleted inserted replaced
279:b3b547563ec7 280:49e9e591c9bb
7 let peer = null; 7 let peer = null;
8 let channel = null; 8 let channel = null;
9 let stream = null; 9 let stream = null;
10 let sessionId = null; 10 let sessionId = null;
11 let stopResolver = null; 11 let stopResolver = null;
12 let captureActive = false;
13 let connectionPromise = null;
14 const canvasTransport =
15 new URLSearchParams(window.location.search).get("canvas") === "1";
16
17 function publishDictation(kind, text) {
18 const payload = {
19 type: "zenbu.dictation.event",
20 kind,
21 text,
22 };
23 window.parent?.postMessage(payload, "*");
24 document.title =
25 `Zenbu Dictation|${Date.now()}|${kind}|${encodeURIComponent(text)}`;
26 }
12 27
13 function setStatus(message, state = "") { 28 function setStatus(message, state = "") {
14 status.textContent = message; 29 status.textContent = message;
15 status.dataset.state = state; 30 status.dataset.state = state;
31 publishDictation("status", message);
16 } 32 }
17 33
18 function waitForIceGathering(connection) { 34 function waitForIceGathering(connection) {
19 if (connection.iceGatheringState === "complete") return Promise.resolve(); 35 if (connection.iceGatheringState === "complete") return Promise.resolve();
20 return new Promise(resolve => { 36 return new Promise(resolve => {
25 }; 41 };
26 connection.addEventListener("icegatheringstatechange", listener); 42 connection.addEventListener("icegatheringstatechange", listener);
27 }); 43 });
28 } 44 }
29 45
46 function waitForTranscriptChannel(transcriptChannel) {
47 if (transcriptChannel.readyState === "open") return Promise.resolve();
48 return new Promise((resolve, reject) => {
49 const timeout = window.setTimeout(
50 () => reject(new Error("Transcript channel readiness timed out")),
51 10000,
52 );
53 transcriptChannel.addEventListener("open", () => {
54 window.clearTimeout(timeout);
55 resolve();
56 }, { once: true });
57 transcriptChannel.addEventListener("close", () => {
58 window.clearTimeout(timeout);
59 reject(new Error("Transcript channel closed during startup"));
60 }, { once: true });
61 });
62 }
63
30 function handleTranscript(message) { 64 function handleTranscript(message) {
31 let event; 65 let event;
32 try { 66 try {
33 event = JSON.parse(message.data); 67 event = JSON.parse(message.data);
34 } catch { 68 } catch {
35 setStatus("Received an invalid server event", "error"); 69 setStatus("Received an invalid server event", "error");
36 return; 70 return;
37 } 71 }
38 switch (event.type) { 72 switch (event.type) {
39 case "ready": 73 case "ready":
40 setStatus("Listening", "ready"); 74 setStatus(captureActive ? "Listening" : "Idle", "ready");
41 break; 75 break;
42 case "speech.started": 76 case "speech.started":
43 setStatus("Speech detected", "working"); 77 setStatus("Speech detected", "working");
44 break; 78 break;
45 case "transcript.partial": 79 case "transcript.partial":
46 partial.textContent = event.text; 80 partial.textContent = event.text;
81 publishDictation("partial", event.text);
47 break; 82 break;
48 case "transcript.final": 83 case "transcript.final":
49 finalText.value += `${finalText.value ? " " : ""}${event.text}`; 84 finalText.value += `${finalText.value ? " " : ""}${event.text}`;
85 publishDictation("final", event.text);
50 partial.textContent = "Waiting for speech..."; 86 partial.textContent = "Waiting for speech...";
51 break; 87 break;
52 case "speech.ended": 88 case "speech.ended":
53 setStatus("Listening", "ready"); 89 setStatus(captureActive ? "Listening" : "Idle", "ready");
54 stopResolver?.(); 90 stopResolver?.();
55 stopResolver = null; 91 stopResolver = null;
56 break; 92 break;
57 case "error": 93 case "error":
58 setStatus(event.message || "Transcription failed", "error"); 94 setStatus(event.message || "Transcription failed", "error");
59 break; 95 break;
60 } 96 }
61 } 97 }
62 98
63 async function start() { 99 async function createConnection() {
64 if (peer) return; 100 if (peer && !["closed", "failed"].includes(peer.connectionState)) return;
65 startButton.disabled = true; 101 if (!stream) {
66 setStatus("Requesting microphone access...", "working");
67 try {
68 stream = await navigator.mediaDevices.getUserMedia({ 102 stream = await navigator.mediaDevices.getUserMedia({
69 audio: { 103 audio: {
70 channelCount: 1, 104 channelCount: 1,
71 echoCancellation: true, 105 echoCancellation: true,
72 noiseSuppression: true, 106 noiseSuppression: true,
73 }, 107 },
74 video: false, 108 video: false,
75 }); 109 });
110 }
111 for (const track of stream.getTracks()) track.enabled = false;
112 try {
76 peer = new RTCPeerConnection(); 113 peer = new RTCPeerConnection();
77 channel = peer.createDataChannel("transcripts", { ordered: true }); 114 channel = peer.createDataChannel("transcripts", { ordered: true });
78 channel.addEventListener("message", handleTranscript); 115 channel.addEventListener("message", handleTranscript);
79 channel.addEventListener("close", () => { 116 channel.addEventListener("close", () => {
80 if (peer) setStatus("Transcript channel closed", "error"); 117 if (peer) setStatus("Transcript channel closed", "error");
81 }); 118 });
82 peer.addEventListener("connectionstatechange", () => { 119 peer.addEventListener("connectionstatechange", () => {
83 if (peer?.connectionState === "failed") { 120 if (peer?.connectionState === "failed") {
84 setStatus("WebRTC connection failed", "error"); 121 setStatus("WebRTC connection failed", "error");
85 void stop(); 122 void stop(false, true);
86 } 123 }
87 }); 124 });
88 for (const track of stream.getTracks()) peer.addTrack(track, stream); 125 for (const track of stream.getTracks()) peer.addTrack(track, stream);
89 126
90 const offer = await peer.createOffer(); 127 const offer = await peer.createOffer();
103 sessionId = answer.sessionId; 140 sessionId = answer.sessionId;
104 await peer.setRemoteDescription({ 141 await peer.setRemoteDescription({
105 sdp: answer.sdp, 142 sdp: answer.sdp,
106 type: answer.type, 143 type: answer.type,
107 }); 144 });
145 await waitForTranscriptChannel(channel);
108 stopButton.disabled = false; 146 stopButton.disabled = false;
109 setStatus("Connecting...", "working");
110 } catch (error) { 147 } catch (error) {
148 await stop(false, true);
149 throw error;
150 }
151 }
152
153 async function ensureConnection() {
154 if (peer && ["connected", "completed"].includes(peer.connectionState)) return;
155 if (!connectionPromise) {
156 connectionPromise = createConnection().finally(() => {
157 connectionPromise = null;
158 });
159 }
160 await connectionPromise;
161 }
162
163 async function start() {
164 captureActive = true;
165 startButton.disabled = true;
166 setStatus("Starting microphone...", "working");
167 try {
168 await ensureConnection();
169 for (const track of stream.getTracks()) track.enabled = true;
170 setStatus(
171 channel?.readyState === "open" ? "Listening" : "Connecting...",
172 channel?.readyState === "open" ? "ready" : "working",
173 );
174 } catch (error) {
175 captureActive = false;
111 setStatus(error.message || "Unable to start dictation", "error"); 176 setStatus(error.message || "Unable to start dictation", "error");
112 await stop(); 177 }
113 } 178 }
114 } 179
115 180 async function stop(waitForFinal = true, releaseStream = !canvasTransport) {
116 async function stop(waitForFinal = true) {
117 const closingSession = sessionId; 181 const closingSession = sessionId;
118 sessionId = null;
119 const tracks = stream?.getTracks() || []; 182 const tracks = stream?.getTracks() || [];
183 captureActive = false;
120 if (channel?.readyState === "open" && waitForFinal) { 184 if (channel?.readyState === "open" && waitForFinal) {
121 const ended = new Promise(resolve => { 185 const ended = new Promise(resolve => {
122 stopResolver = resolve; 186 stopResolver = resolve;
123 }); 187 });
124 channel.send("stop"); 188 channel.send(releaseStream ? "stop" : "pause");
125 for (const track of tracks) track.enabled = false; 189 for (const track of tracks) track.enabled = false;
126 await Promise.race([ 190 await Promise.race([
127 ended, 191 ended,
128 new Promise(resolve => window.setTimeout(resolve, 10000)), 192 new Promise(resolve => window.setTimeout(resolve, 10000)),
129 ]); 193 ]);
130 stopResolver = null; 194 stopResolver = null;
131 } 195 }
132 for (const track of tracks) track.stop(); 196 for (const track of tracks) {
133 stream = null; 197 track.enabled = false;
198 if (releaseStream) track.stop();
199 }
200 if (!releaseStream) {
201 startButton.disabled = false;
202 stopButton.disabled = true;
203 setStatus("Idle");
204 return;
205 }
206 sessionId = null;
207 if (releaseStream) stream = null;
134 channel?.close(); 208 channel?.close();
135 channel = null; 209 channel = null;
136 peer?.close(); 210 peer?.close();
137 peer = null; 211 peer = null;
138 if (closingSession) { 212 if (closingSession) {
146 setStatus("Idle"); 220 setStatus("Idle");
147 } 221 }
148 222
149 startButton.addEventListener("click", () => void start()); 223 startButton.addEventListener("click", () => void start());
150 stopButton.addEventListener("click", () => void stop()); 224 stopButton.addEventListener("click", () => void stop());
151 window.addEventListener("pagehide", () => void stop(false)); 225 window.ZenbuDictationStart = () => start();
226 window.ZenbuDictationStop = () => stop();
227 window.ZenbuDictationCommit = () => {
228 if (channel?.readyState === "open") channel.send("commit");
229 };
230 window.ZenbuDictationToggle = () => captureActive ? stop() : start();
231 window.addEventListener("message", event => {
232 if (event.data?.type === "zenbu.dictation.set-active") {
233 void (event.data.active ? start() : stop());
234 } else if (event.data?.type === "zenbu.dictation.commit") {
235 window.ZenbuDictationCommit();
236 }
237 });
238 window.addEventListener("pagehide", () => void stop(false, true));
239
240 async function prepareCanvasMicrophone() {
241 if (!canvasTransport || peer) return;
242 try {
243 await ensureConnection();
244 for (const track of stream.getTracks()) track.enabled = false;
245 setStatus("Idle");
246 } catch (error) {
247 setStatus(error.message || "Unable to prepare microphone", "error");
248 }
249 }
250
251 void prepareCanvasMicrophone();