view dictation/web/dictation.js @ 275:78699f810817

Add Qwen3-VL and WebRTC dictation services Add Bazel targets for the CUDA-backed Qwen3-VL server and a local WebRTC faster-whisper dictation service. Co-authored-by: Copilot <[email protected]> Copilot-Session: e3d8cb06-6c95-4ae0-9757-651d3796ab00
author MrJuneJune <me@mrjunejune.com>
date Mon, 17 Aug 2026 10:58:47 -0700
parents
children 49e9e591c9bb
line wrap: on
line source

const startButton = document.querySelector("[data-start]");
const stopButton = document.querySelector("[data-stop]");
const status = document.querySelector("[data-status]");
const partial = document.querySelector("[data-partial]");
const finalText = document.querySelector("[data-final]");

let peer = null;
let channel = null;
let stream = null;
let sessionId = null;
let stopResolver = null;

function setStatus(message, state = "") {
  status.textContent = message;
  status.dataset.state = state;
}

function waitForIceGathering(connection) {
  if (connection.iceGatheringState === "complete") return Promise.resolve();
  return new Promise(resolve => {
    const listener = () => {
      if (connection.iceGatheringState !== "complete") return;
      connection.removeEventListener("icegatheringstatechange", listener);
      resolve();
    };
    connection.addEventListener("icegatheringstatechange", listener);
  });
}

function handleTranscript(message) {
  let event;
  try {
    event = JSON.parse(message.data);
  } catch {
    setStatus("Received an invalid server event", "error");
    return;
  }
  switch (event.type) {
    case "ready":
      setStatus("Listening", "ready");
      break;
    case "speech.started":
      setStatus("Speech detected", "working");
      break;
    case "transcript.partial":
      partial.textContent = event.text;
      break;
    case "transcript.final":
      finalText.value += `${finalText.value ? " " : ""}${event.text}`;
      partial.textContent = "Waiting for speech...";
      break;
    case "speech.ended":
      setStatus("Listening", "ready");
      stopResolver?.();
      stopResolver = null;
      break;
    case "error":
      setStatus(event.message || "Transcription failed", "error");
      break;
  }
}

async function start() {
  if (peer) return;
  startButton.disabled = true;
  setStatus("Requesting microphone access...", "working");
  try {
    stream = await navigator.mediaDevices.getUserMedia({
      audio: {
        channelCount: 1,
        echoCancellation: true,
        noiseSuppression: true,
      },
      video: false,
    });
    peer = new RTCPeerConnection();
    channel = peer.createDataChannel("transcripts", { ordered: true });
    channel.addEventListener("message", handleTranscript);
    channel.addEventListener("close", () => {
      if (peer) setStatus("Transcript channel closed", "error");
    });
    peer.addEventListener("connectionstatechange", () => {
      if (peer?.connectionState === "failed") {
        setStatus("WebRTC connection failed", "error");
        void stop();
      }
    });
    for (const track of stream.getTracks()) peer.addTrack(track, stream);

    const offer = await peer.createOffer();
    await peer.setLocalDescription(offer);
    await waitForIceGathering(peer);
    const response = await fetch("/api/webrtc/offer", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify(peer.localDescription),
    });
    if (!response.ok) {
      const detail = await response.text();
      throw new Error(`Signaling failed (${response.status}): ${detail}`);
    }
    const answer = await response.json();
    sessionId = answer.sessionId;
    await peer.setRemoteDescription({
      sdp: answer.sdp,
      type: answer.type,
    });
    stopButton.disabled = false;
    setStatus("Connecting...", "working");
  } catch (error) {
    setStatus(error.message || "Unable to start dictation", "error");
    await stop();
  }
}

async function stop(waitForFinal = true) {
  const closingSession = sessionId;
  sessionId = null;
  const tracks = stream?.getTracks() || [];
  if (channel?.readyState === "open" && waitForFinal) {
    const ended = new Promise(resolve => {
      stopResolver = resolve;
    });
    channel.send("stop");
    for (const track of tracks) track.enabled = false;
    await Promise.race([
      ended,
      new Promise(resolve => window.setTimeout(resolve, 10000)),
    ]);
    stopResolver = null;
  }
  for (const track of tracks) track.stop();
  stream = null;
  channel?.close();
  channel = null;
  peer?.close();
  peer = null;
  if (closingSession) {
    await fetch(`/api/webrtc/session/${encodeURIComponent(closingSession)}/close`, {
      method: "POST",
      keepalive: true,
    }).catch(() => {});
  }
  startButton.disabled = false;
  stopButton.disabled = true;
  setStatus("Idle");
}

startButton.addEventListener("click", () => void start());
stopButton.addEventListener("click", () => void stop());
window.addEventListener("pagehide", () => void stop(false));