Mercurial

annotate dictation/transcriber.py @ 282:6bd4a3990696 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add cross-platform canvas orchestration and typed composer Enable native ARM dependencies, macOS Copilot orchestration, portable service supervision, CPU dictation, and a shared N-key speak-or-type composer. Co-authored-by: Copilot <[email protected]> Copilot-Session: f68442b1-fa8f-46a0-9689-81710613bbd4
author MrJuneJune <me@mrjunejune.com>
date Thu, 20 Aug 2026 20:45:25 -0700
parents 49e9e591c9bb
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
1 from __future__ import annotations
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
2
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
3 import asyncio
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
4 from concurrent.futures import ThreadPoolExecutor
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
5 from dataclasses import dataclass
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
6 from pathlib import Path
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
7 from typing import Protocol
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
8
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
9 import numpy as np
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
10
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
11
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
12 @dataclass(frozen=True)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
13 class Transcript:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
14 text: str
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
15 language: str
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
16 probability: float
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
17
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
18
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
19 class Transcriber(Protocol):
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
20 async def warmup(self) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
21 ...
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
22
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
23 async def transcribe(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
24 self,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
25 samples: np.ndarray,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
26 *,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
27 final: bool,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
28 ) -> Transcript:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
29 ...
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
30
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
31
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
32 class FasterWhisperTranscriber:
282
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
33 def __init__(
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
34 self,
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
35 model_dir: Path,
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
36 device: str,
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
37 compute_type: str,
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
38 ) -> None:
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
39 self._model_dir = model_dir
282
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
40 self._device = device
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
41 self._compute_type = compute_type
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
42 self._executor = ThreadPoolExecutor(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
43 max_workers=1,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
44 thread_name_prefix="dictation",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
45 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
46 self._model = None
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
47 self._load_lock = asyncio.Lock()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
48
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
49 async def warmup(self) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
50 if self._model is not None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
51 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
52 async with self._load_lock:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
53 if self._model is not None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
54 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
55 loop = asyncio.get_running_loop()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
56 self._model = await loop.run_in_executor(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
57 self._executor,
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
58 self._load_and_warmup,
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
59 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
60
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
61 def _load_and_warmup(self):
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
62 from faster_whisper import WhisperModel
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
63
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
64 if not (self._model_dir / "model.bin").is_file():
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
65 raise FileNotFoundError(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
66 f"Model not found at {self._model_dir}. "
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
67 "Run: bazel run //dictation:download_model"
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
68 )
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
69 model = WhisperModel(
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
70 str(self._model_dir),
282
6bd4a3990696 Add cross-platform canvas orchestration and typed composer
MrJuneJune <me@mrjunejune.com>
parents: 280
diff changeset
71 device=self._device,
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
72 compute_type=self._compute_type,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
73 local_files_only=True,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
74 )
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
75 # Loading weights does not initialize all CUDA kernels. Execute and
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
76 # consume one short silent inference now so the user's first utterance
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
77 # does not pay the one-time GPU setup cost.
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
78 segments, _ = model.transcribe(
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
79 np.zeros(8000, dtype=np.float32),
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
80 beam_size=1,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
81 best_of=1,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
82 condition_on_previous_text=False,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
83 vad_filter=False,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
84 without_timestamps=True,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
85 )
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
86 list(segments)
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
87 return model
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
88
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
89 async def transcribe(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
90 self,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
91 samples: np.ndarray,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
92 *,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
93 final: bool,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
94 ) -> Transcript:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
95 await self.warmup()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
96 loop = asyncio.get_running_loop()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
97 return await loop.run_in_executor(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
98 self._executor,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
99 self._transcribe_sync,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
100 np.asarray(samples, dtype=np.float32),
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
101 final,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
102 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
103
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
104 def _transcribe_sync(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
105 self,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
106 samples: np.ndarray,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
107 final: bool,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
108 ) -> Transcript:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
109 segments, info = self._model.transcribe(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
110 samples,
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
111 beam_size=1,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
112 best_of=1,
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
113 condition_on_previous_text=False,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
114 vad_filter=False,
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
115 without_timestamps=True,
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
116 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
117 text = "".join(segment.text for segment in segments).strip()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
118 return Transcript(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
119 text=text,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
120 language=info.language or "",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
121 probability=float(info.language_probability or 0.0),
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
122 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
123
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
124 def close(self) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
125 self._executor.shutdown(wait=False, cancel_futures=True)