Mercurial
diff dictation/web/dictation.js @ 275:78699f810817
Add Qwen3-VL and WebRTC dictation services
Add Bazel targets for the CUDA-backed Qwen3-VL server and a local WebRTC faster-whisper dictation service.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: e3d8cb06-6c95-4ae0-9757-651d3796ab00
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Mon, 17 Aug 2026 10:58:47 -0700 |
| parents | |
| children | 49e9e591c9bb |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dictation/web/dictation.js Mon Aug 17 10:58:47 2026 -0700 @@ -0,0 +1,151 @@ +const startButton = document.querySelector("[data-start]"); +const stopButton = document.querySelector("[data-stop]"); +const status = document.querySelector("[data-status]"); +const partial = document.querySelector("[data-partial]"); +const finalText = document.querySelector("[data-final]"); + +let peer = null; +let channel = null; +let stream = null; +let sessionId = null; +let stopResolver = null; + +function setStatus(message, state = "") { + status.textContent = message; + status.dataset.state = state; +} + +function waitForIceGathering(connection) { + if (connection.iceGatheringState === "complete") return Promise.resolve(); + return new Promise(resolve => { + const listener = () => { + if (connection.iceGatheringState !== "complete") return; + connection.removeEventListener("icegatheringstatechange", listener); + resolve(); + }; + connection.addEventListener("icegatheringstatechange", listener); + }); +} + +function handleTranscript(message) { + let event; + try { + event = JSON.parse(message.data); + } catch { + setStatus("Received an invalid server event", "error"); + return; + } + switch (event.type) { + case "ready": + setStatus("Listening", "ready"); + break; + case "speech.started": + setStatus("Speech detected", "working"); + break; + case "transcript.partial": + partial.textContent = event.text; + break; + case "transcript.final": + finalText.value += `${finalText.value ? " " : ""}${event.text}`; + partial.textContent = "Waiting for speech..."; + break; + case "speech.ended": + setStatus("Listening", "ready"); + stopResolver?.(); + stopResolver = null; + break; + case "error": + setStatus(event.message || "Transcription failed", "error"); + break; + } +} + +async function start() { + if (peer) return; + startButton.disabled = true; + setStatus("Requesting microphone access...", "working"); + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + }, + video: false, + }); + peer = new RTCPeerConnection(); + channel = peer.createDataChannel("transcripts", { ordered: true }); + channel.addEventListener("message", handleTranscript); + channel.addEventListener("close", () => { + if (peer) setStatus("Transcript channel closed", "error"); + }); + peer.addEventListener("connectionstatechange", () => { + if (peer?.connectionState === "failed") { + setStatus("WebRTC connection failed", "error"); + void stop(); + } + }); + for (const track of stream.getTracks()) peer.addTrack(track, stream); + + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer); + await waitForIceGathering(peer); + const response = await fetch("/api/webrtc/offer", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(peer.localDescription), + }); + if (!response.ok) { + const detail = await response.text(); + throw new Error(`Signaling failed (${response.status}): ${detail}`); + } + const answer = await response.json(); + sessionId = answer.sessionId; + await peer.setRemoteDescription({ + sdp: answer.sdp, + type: answer.type, + }); + stopButton.disabled = false; + setStatus("Connecting...", "working"); + } catch (error) { + setStatus(error.message || "Unable to start dictation", "error"); + await stop(); + } +} + +async function stop(waitForFinal = true) { + const closingSession = sessionId; + sessionId = null; + const tracks = stream?.getTracks() || []; + if (channel?.readyState === "open" && waitForFinal) { + const ended = new Promise(resolve => { + stopResolver = resolve; + }); + channel.send("stop"); + for (const track of tracks) track.enabled = false; + await Promise.race([ + ended, + new Promise(resolve => window.setTimeout(resolve, 10000)), + ]); + stopResolver = null; + } + for (const track of tracks) track.stop(); + stream = null; + channel?.close(); + channel = null; + peer?.close(); + peer = null; + if (closingSession) { + await fetch(`/api/webrtc/session/${encodeURIComponent(closingSession)}/close`, { + method: "POST", + keepalive: true, + }).catch(() => {}); + } + startButton.disabled = false; + stopButton.disabled = true; + setStatus("Idle"); +} + +startButton.addEventListener("click", () => void start()); +stopButton.addEventListener("click", () => void stop()); +window.addEventListener("pagehide", () => void stop(false));