view dictation/web/dictation.js @ 280:49e9e591c9bb

Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
author MrJuneJune <me@mrjunejune.com>
date Tue, 18 Aug 2026 19:14:53 -0700
parents 78699f810817
children
line wrap: on
line source

const startButton = document.querySelector("[data-start]");
const stopButton = document.querySelector("[data-stop]");
const status = document.querySelector("[data-status]");
const partial = document.querySelector("[data-partial]");
const finalText = document.querySelector("[data-final]");

let peer = null;
let channel = null;
let stream = null;
let sessionId = null;
let stopResolver = null;
let captureActive = false;
let connectionPromise = null;
const canvasTransport =
  new URLSearchParams(window.location.search).get("canvas") === "1";

function publishDictation(kind, text) {
  const payload = {
    type: "zenbu.dictation.event",
    kind,
    text,
  };
  window.parent?.postMessage(payload, "*");
  document.title =
    `Zenbu Dictation|${Date.now()}|${kind}|${encodeURIComponent(text)}`;
}

function setStatus(message, state = "") {
  status.textContent = message;
  status.dataset.state = state;
  publishDictation("status", message);
}

function waitForIceGathering(connection) {
  if (connection.iceGatheringState === "complete") return Promise.resolve();
  return new Promise(resolve => {
    const listener = () => {
      if (connection.iceGatheringState !== "complete") return;
      connection.removeEventListener("icegatheringstatechange", listener);
      resolve();
    };
    connection.addEventListener("icegatheringstatechange", listener);
  });
}

function waitForTranscriptChannel(transcriptChannel) {
  if (transcriptChannel.readyState === "open") return Promise.resolve();
  return new Promise((resolve, reject) => {
    const timeout = window.setTimeout(
      () => reject(new Error("Transcript channel readiness timed out")),
      10000,
    );
    transcriptChannel.addEventListener("open", () => {
      window.clearTimeout(timeout);
      resolve();
    }, { once: true });
    transcriptChannel.addEventListener("close", () => {
      window.clearTimeout(timeout);
      reject(new Error("Transcript channel closed during startup"));
    }, { once: true });
  });
}

function handleTranscript(message) {
  let event;
  try {
    event = JSON.parse(message.data);
  } catch {
    setStatus("Received an invalid server event", "error");
    return;
  }
  switch (event.type) {
    case "ready":
      setStatus(captureActive ? "Listening" : "Idle", "ready");
      break;
    case "speech.started":
      setStatus("Speech detected", "working");
      break;
    case "transcript.partial":
      partial.textContent = event.text;
      publishDictation("partial", event.text);
      break;
    case "transcript.final":
      finalText.value += `${finalText.value ? " " : ""}${event.text}`;
      publishDictation("final", event.text);
      partial.textContent = "Waiting for speech...";
      break;
    case "speech.ended":
      setStatus(captureActive ? "Listening" : "Idle", "ready");
      stopResolver?.();
      stopResolver = null;
      break;
    case "error":
      setStatus(event.message || "Transcription failed", "error");
      break;
  }
}

async function createConnection() {
  if (peer && !["closed", "failed"].includes(peer.connectionState)) return;
  if (!stream) {
    stream = await navigator.mediaDevices.getUserMedia({
      audio: {
        channelCount: 1,
        echoCancellation: true,
        noiseSuppression: true,
      },
      video: false,
    });
  }
  for (const track of stream.getTracks()) track.enabled = false;
  try {
    peer = new RTCPeerConnection();
    channel = peer.createDataChannel("transcripts", { ordered: true });
    channel.addEventListener("message", handleTranscript);
    channel.addEventListener("close", () => {
      if (peer) setStatus("Transcript channel closed", "error");
    });
    peer.addEventListener("connectionstatechange", () => {
      if (peer?.connectionState === "failed") {
        setStatus("WebRTC connection failed", "error");
        void stop(false, true);
      }
    });
    for (const track of stream.getTracks()) peer.addTrack(track, stream);

    const offer = await peer.createOffer();
    await peer.setLocalDescription(offer);
    await waitForIceGathering(peer);
    const response = await fetch("/api/webrtc/offer", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify(peer.localDescription),
    });
    if (!response.ok) {
      const detail = await response.text();
      throw new Error(`Signaling failed (${response.status}): ${detail}`);
    }
    const answer = await response.json();
    sessionId = answer.sessionId;
    await peer.setRemoteDescription({
      sdp: answer.sdp,
      type: answer.type,
    });
    await waitForTranscriptChannel(channel);
    stopButton.disabled = false;
  } catch (error) {
    await stop(false, true);
    throw error;
  }
}

async function ensureConnection() {
  if (peer && ["connected", "completed"].includes(peer.connectionState)) return;
  if (!connectionPromise) {
    connectionPromise = createConnection().finally(() => {
      connectionPromise = null;
    });
  }
  await connectionPromise;
}

async function start() {
  captureActive = true;
  startButton.disabled = true;
  setStatus("Starting microphone...", "working");
  try {
    await ensureConnection();
    for (const track of stream.getTracks()) track.enabled = true;
    setStatus(
      channel?.readyState === "open" ? "Listening" : "Connecting...",
      channel?.readyState === "open" ? "ready" : "working",
    );
  } catch (error) {
    captureActive = false;
    setStatus(error.message || "Unable to start dictation", "error");
  }
}

async function stop(waitForFinal = true, releaseStream = !canvasTransport) {
  const closingSession = sessionId;
  const tracks = stream?.getTracks() || [];
  captureActive = false;
  if (channel?.readyState === "open" && waitForFinal) {
    const ended = new Promise(resolve => {
      stopResolver = resolve;
    });
    channel.send(releaseStream ? "stop" : "pause");
    for (const track of tracks) track.enabled = false;
    await Promise.race([
      ended,
      new Promise(resolve => window.setTimeout(resolve, 10000)),
    ]);
    stopResolver = null;
  }
  for (const track of tracks) {
    track.enabled = false;
    if (releaseStream) track.stop();
  }
  if (!releaseStream) {
    startButton.disabled = false;
    stopButton.disabled = true;
    setStatus("Idle");
    return;
  }
  sessionId = null;
  if (releaseStream) stream = null;
  channel?.close();
  channel = null;
  peer?.close();
  peer = null;
  if (closingSession) {
    await fetch(`/api/webrtc/session/${encodeURIComponent(closingSession)}/close`, {
      method: "POST",
      keepalive: true,
    }).catch(() => {});
  }
  startButton.disabled = false;
  stopButton.disabled = true;
  setStatus("Idle");
}

startButton.addEventListener("click", () => void start());
stopButton.addEventListener("click", () => void stop());
window.ZenbuDictationStart = () => start();
window.ZenbuDictationStop = () => stop();
window.ZenbuDictationCommit = () => {
  if (channel?.readyState === "open") channel.send("commit");
};
window.ZenbuDictationToggle = () => captureActive ? stop() : start();
window.addEventListener("message", event => {
  if (event.data?.type === "zenbu.dictation.set-active") {
    void (event.data.active ? start() : stop());
  } else if (event.data?.type === "zenbu.dictation.commit") {
    window.ZenbuDictationCommit();
  }
});
window.addEventListener("pagehide", () => void stop(false, true));

async function prepareCanvasMicrophone() {
  if (!canvasTransport || peer) return;
  try {
    await ensureConnection();
    for (const track of stream.getTracks()) track.enabled = false;
    setStatus("Idle");
  } catch (error) {
    setStatus(error.message || "Unable to prepare microphone", "error");
  }
}

void prepareCanvasMicrophone();