Mercurial
diff dictation/audio.py @ 275:78699f810817
Add Qwen3-VL and WebRTC dictation services
Add Bazel targets for the CUDA-backed Qwen3-VL server and a local WebRTC faster-whisper dictation service.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: e3d8cb06-6c95-4ae0-9757-651d3796ab00
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Mon, 17 Aug 2026 10:58:47 -0700 |
| parents | |
| children | 49e9e591c9bb |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dictation/audio.py Mon Aug 17 10:58:47 2026 -0700 @@ -0,0 +1,112 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +import numpy as np + + +class AudioEventKind(str, Enum): + SPEECH_STARTED = "speech_started" + PARTIAL_READY = "partial_ready" + FINAL_READY = "final_ready" + + +@dataclass(frozen=True) +class AudioEvent: + kind: AudioEventKind + samples: np.ndarray | None = None + + +class AudioSegmenter: + def __init__( + self, + *, + sample_rate: int = 16000, + speech_threshold: float = 0.012, + silence_ms: int = 700, + partial_interval_ms: int = 1200, + max_utterance_seconds: int = 30, + pre_roll_ms: int = 200, + ) -> None: + self.sample_rate = sample_rate + self.speech_threshold = speech_threshold + self.silence_samples = sample_rate * silence_ms // 1000 + self.partial_samples = sample_rate * partial_interval_ms // 1000 + self.max_samples = sample_rate * max_utterance_seconds + self.pre_roll_samples = sample_rate * pre_roll_ms // 1000 + self._pre_roll = np.empty(0, dtype=np.float32) + self._utterance = np.empty(0, dtype=np.float32) + self._silence_count = 0 + self._last_partial_size = 0 + self._speaking = False + + @property + def speaking(self) -> bool: + return self._speaking + + def feed(self, samples: np.ndarray) -> list[AudioEvent]: + chunk = np.asarray(samples, dtype=np.float32).reshape(-1) + if chunk.size == 0: + return [] + rms = float(np.sqrt(np.mean(np.square(chunk), dtype=np.float64))) + is_speech = rms >= self.speech_threshold + events: list[AudioEvent] = [] + + if not self._speaking: + if not is_speech: + self._append_pre_roll(chunk) + return events + self._speaking = True + self._utterance = np.concatenate((self._pre_roll, chunk)) + self._pre_roll = np.empty(0, dtype=np.float32) + self._silence_count = 0 + self._last_partial_size = 0 + events.append(AudioEvent(AudioEventKind.SPEECH_STARTED)) + else: + self._utterance = np.concatenate((self._utterance, chunk)) + + self._silence_count = 0 if is_speech else self._silence_count + chunk.size + utterance_size = self._utterance.size + + if ( + utterance_size - self._last_partial_size >= self.partial_samples + and self._silence_count < self.silence_samples + ): + self._last_partial_size = utterance_size + events.append( + AudioEvent( + AudioEventKind.PARTIAL_READY, + self._utterance.copy(), + ) + ) + + if ( + self._silence_count >= self.silence_samples + or utterance_size >= self.max_samples + ): + final = self._utterance.copy() + self._reset() + events.append(AudioEvent(AudioEventKind.FINAL_READY, final)) + + return events + + def flush(self) -> AudioEvent | None: + if not self._speaking or self._utterance.size == 0: + self._reset() + return None + final = self._utterance.copy() + self._reset() + return AudioEvent(AudioEventKind.FINAL_READY, final) + + def _append_pre_roll(self, chunk: np.ndarray) -> None: + self._pre_roll = np.concatenate((self._pre_roll, chunk)) + if self._pre_roll.size > self.pre_roll_samples: + self._pre_roll = self._pre_roll[-self.pre_roll_samples :] + + def _reset(self) -> None: + self._pre_roll = np.empty(0, dtype=np.float32) + self._utterance = np.empty(0, dtype=np.float32) + self._silence_count = 0 + self._last_partial_size = 0 + self._speaking = False