diff dictation/web/dictation.js @ 275:78699f810817

Add Qwen3-VL and WebRTC dictation services Add Bazel targets for the CUDA-backed Qwen3-VL server and a local WebRTC faster-whisper dictation service. Co-authored-by: Copilot <[email protected]> Copilot-Session: e3d8cb06-6c95-4ae0-9757-651d3796ab00
author MrJuneJune <me@mrjunejune.com>
date Mon, 17 Aug 2026 10:58:47 -0700
parents
children 49e9e591c9bb
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/dictation/web/dictation.js	Mon Aug 17 10:58:47 2026 -0700
@@ -0,0 +1,151 @@
+const startButton = document.querySelector("[data-start]");
+const stopButton = document.querySelector("[data-stop]");
+const status = document.querySelector("[data-status]");
+const partial = document.querySelector("[data-partial]");
+const finalText = document.querySelector("[data-final]");
+
+let peer = null;
+let channel = null;
+let stream = null;
+let sessionId = null;
+let stopResolver = null;
+
+function setStatus(message, state = "") {
+  status.textContent = message;
+  status.dataset.state = state;
+}
+
+function waitForIceGathering(connection) {
+  if (connection.iceGatheringState === "complete") return Promise.resolve();
+  return new Promise(resolve => {
+    const listener = () => {
+      if (connection.iceGatheringState !== "complete") return;
+      connection.removeEventListener("icegatheringstatechange", listener);
+      resolve();
+    };
+    connection.addEventListener("icegatheringstatechange", listener);
+  });
+}
+
+function handleTranscript(message) {
+  let event;
+  try {
+    event = JSON.parse(message.data);
+  } catch {
+    setStatus("Received an invalid server event", "error");
+    return;
+  }
+  switch (event.type) {
+    case "ready":
+      setStatus("Listening", "ready");
+      break;
+    case "speech.started":
+      setStatus("Speech detected", "working");
+      break;
+    case "transcript.partial":
+      partial.textContent = event.text;
+      break;
+    case "transcript.final":
+      finalText.value += `${finalText.value ? " " : ""}${event.text}`;
+      partial.textContent = "Waiting for speech...";
+      break;
+    case "speech.ended":
+      setStatus("Listening", "ready");
+      stopResolver?.();
+      stopResolver = null;
+      break;
+    case "error":
+      setStatus(event.message || "Transcription failed", "error");
+      break;
+  }
+}
+
+async function start() {
+  if (peer) return;
+  startButton.disabled = true;
+  setStatus("Requesting microphone access...", "working");
+  try {
+    stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        channelCount: 1,
+        echoCancellation: true,
+        noiseSuppression: true,
+      },
+      video: false,
+    });
+    peer = new RTCPeerConnection();
+    channel = peer.createDataChannel("transcripts", { ordered: true });
+    channel.addEventListener("message", handleTranscript);
+    channel.addEventListener("close", () => {
+      if (peer) setStatus("Transcript channel closed", "error");
+    });
+    peer.addEventListener("connectionstatechange", () => {
+      if (peer?.connectionState === "failed") {
+        setStatus("WebRTC connection failed", "error");
+        void stop();
+      }
+    });
+    for (const track of stream.getTracks()) peer.addTrack(track, stream);
+
+    const offer = await peer.createOffer();
+    await peer.setLocalDescription(offer);
+    await waitForIceGathering(peer);
+    const response = await fetch("/api/webrtc/offer", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(peer.localDescription),
+    });
+    if (!response.ok) {
+      const detail = await response.text();
+      throw new Error(`Signaling failed (${response.status}): ${detail}`);
+    }
+    const answer = await response.json();
+    sessionId = answer.sessionId;
+    await peer.setRemoteDescription({
+      sdp: answer.sdp,
+      type: answer.type,
+    });
+    stopButton.disabled = false;
+    setStatus("Connecting...", "working");
+  } catch (error) {
+    setStatus(error.message || "Unable to start dictation", "error");
+    await stop();
+  }
+}
+
+async function stop(waitForFinal = true) {
+  const closingSession = sessionId;
+  sessionId = null;
+  const tracks = stream?.getTracks() || [];
+  if (channel?.readyState === "open" && waitForFinal) {
+    const ended = new Promise(resolve => {
+      stopResolver = resolve;
+    });
+    channel.send("stop");
+    for (const track of tracks) track.enabled = false;
+    await Promise.race([
+      ended,
+      new Promise(resolve => window.setTimeout(resolve, 10000)),
+    ]);
+    stopResolver = null;
+  }
+  for (const track of tracks) track.stop();
+  stream = null;
+  channel?.close();
+  channel = null;
+  peer?.close();
+  peer = null;
+  if (closingSession) {
+    await fetch(`/api/webrtc/session/${encodeURIComponent(closingSession)}/close`, {
+      method: "POST",
+      keepalive: true,
+    }).catch(() => {});
+  }
+  startButton.disabled = false;
+  stopButton.disabled = true;
+  setStatus("Idle");
+}
+
+startButton.addEventListener("click", () => void start());
+stopButton.addEventListener("click", () => void stop());
+window.addEventListener("pagehide", () => void stop(false));