Mercurial

annotate dictation/session.py @ 280:49e9e591c9bb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
author MrJuneJune <me@mrjunejune.com>
date Tue, 18 Aug 2026 19:14:53 -0700
parents 78699f810817
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
1 from __future__ import annotations
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
2
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
3 import asyncio
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
4 import json
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
5 from typing import Any
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
6
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
7 from aiortc.mediastreams import MediaStreamError
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
8 from av.audio.resampler import AudioResampler
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
9 import numpy as np
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
10
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
11 from dictation.audio import AudioEventKind, AudioSegmenter
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
12 from dictation.config import DictationConfig
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
13 from dictation.transcriber import Transcriber
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
14
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
15
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
16 class DictationSession:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
17 def __init__(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
18 self,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
19 session_id: str,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
20 peer: Any,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
21 transcriber: Transcriber,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
22 config: DictationConfig,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
23 ) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
24 self.session_id = session_id
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
25 self.peer = peer
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
26 self.transcriber = transcriber
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
27 self.segmenter = AudioSegmenter(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
28 speech_threshold=config.speech_threshold,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
29 silence_ms=config.silence_ms,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
30 partial_interval_ms=config.partial_interval_ms,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
31 max_utterance_seconds=config.max_utterance_seconds,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
32 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
33 self.channel = None
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
34 self.audio_task: asyncio.Task | None = None
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
35 self.partial_task: asyncio.Task | None = None
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
36 self.final_tasks: set[asyncio.Task] = set()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
37 self.closed = False
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
38 self.stopping = False
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
39 self._resampler = AudioResampler(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
40 format="s16",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
41 layout="mono",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
42 rate=16000,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
43 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
44
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
45 def attach_channel(self, channel: Any) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
46 if channel.label != "transcripts":
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
47 channel.close()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
48 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
49 self.channel = channel
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
50
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
51 @channel.on("open")
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
52 def on_open() -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
53 self.send("ready", sessionId=self.session_id)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
54
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
55 @channel.on("message")
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
56 def on_message(message: Any) -> None:
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
57 if message in {"commit", "pause", "stop"}:
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
58 if message == "stop":
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
59 self.stopping = True
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
60 if self.partial_task and not self.partial_task.done():
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
61 self.partial_task.cancel()
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
62 event = self.segmenter.flush()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
63 if event and event.samples is not None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
64 self._start_final(event.samples)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
65 else:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
66 self.send("speech.ended")
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
67
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
68 if channel.readyState == "open":
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
69 self.send("ready", sessionId=self.session_id)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
70
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
71 def attach_track(self, track: Any) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
72 if track.kind != "audio" or self.audio_task is not None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
73 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
74 self.audio_task = asyncio.create_task(self._consume_audio(track))
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
75
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
76 async def _consume_audio(self, track: Any) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
77 try:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
78 while not self.closed:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
79 frame = await track.recv()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
80 for converted in self._resampler.resample(frame):
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
81 if self.stopping:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
82 continue
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
83 samples = converted.to_ndarray().reshape(-1)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
84 normalized = samples.astype(np.float32) / 32768.0
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
85 for event in self.segmenter.feed(normalized):
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
86 if event.kind == AudioEventKind.SPEECH_STARTED:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
87 self.send("speech.started")
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
88 elif (
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
89 event.kind == AudioEventKind.PARTIAL_READY
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
90 and event.samples is not None
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
91 ):
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
92 self._start_partial(event.samples)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
93 elif (
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
94 event.kind == AudioEventKind.FINAL_READY
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
95 and event.samples is not None
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
96 ):
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
97 self._start_final(event.samples)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
98 except MediaStreamError:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
99 if not self.stopping:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
100 event = self.segmenter.flush()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
101 if event and event.samples is not None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
102 self._start_final(event.samples)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
103 else:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
104 self.send("speech.ended")
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
105 except asyncio.CancelledError:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
106 raise
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
107 except Exception as error:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
108 if not self.closed:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
109 self.send(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
110 "error",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
111 code="audio_track_failed",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
112 message=str(error),
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
113 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
114
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
115 def _start_partial(self, samples: np.ndarray) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
116 if self.partial_task and not self.partial_task.done():
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
117 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
118 self.partial_task = asyncio.create_task(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
119 self._transcribe(samples, final=False)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
120 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
121
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
122 def _start_final(self, samples: np.ndarray) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
123 task = asyncio.create_task(self._transcribe(samples, final=True))
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
124 self.final_tasks.add(task)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
125 task.add_done_callback(self.final_tasks.discard)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
126
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
127 async def _transcribe(self, samples: np.ndarray, *, final: bool) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
128 try:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
129 transcript = await self.transcriber.transcribe(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
130 samples,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
131 final=final,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
132 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
133 if transcript.text:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
134 self.send(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
135 "transcript.final" if final else "transcript.partial",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
136 text=transcript.text,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
137 language=transcript.language,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
138 probability=transcript.probability,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
139 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
140 if final:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
141 self.send("speech.ended")
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
142 except asyncio.CancelledError:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
143 raise
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
144 except Exception as error:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
145 self.send(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
146 "error",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
147 code="transcription_failed",
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
148 message=str(error),
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
149 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
150
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
151 def send(self, event_type: str, **payload: Any) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
152 if not self.channel or self.channel.readyState != "open":
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
153 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
154 self.channel.send(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
155 json.dumps(
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
156 {"type": event_type, **payload},
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
157 separators=(",", ":"),
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
158 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
159 )
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
160
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
161 async def _cancel_pending_ice_transactions(self) -> None:
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
162 # aioice 0.10.2 can leave STUN retry timers armed after its datagram
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
163 # transport closes. Cancel their futures first so Transaction.run()
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
164 # clears each timer before RTCPeerConnection.close() drops sockets.
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
165 ice_transports = getattr(
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
166 self.peer,
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
167 "_RTCPeerConnection__iceTransports",
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
168 (),
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
169 )
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
170 for ice_transport in ice_transports:
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
171 connection = getattr(ice_transport, "_connection", None)
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
172 for protocol in getattr(connection, "_protocols", ()):
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
173 for transaction in tuple(
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
174 getattr(protocol, "transactions", {}).values()
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
175 ):
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
176 future = getattr(transaction, "_Transaction__future", None)
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
177 if future is not None and not future.done():
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
178 future.cancel()
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
179 await asyncio.sleep(0)
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
180
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
181 async def close(self) -> None:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
182 if self.closed:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
183 return
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
184 self.closed = True
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
185 tasks = [
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
186 task
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
187 for task in [
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
188 self.audio_task,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
189 self.partial_task,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
190 *self.final_tasks,
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
191 ]
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
192 if task is not None and not task.done()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
193 ]
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
194 for task in tasks:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
195 task.cancel()
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
196 if tasks:
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
197 await asyncio.gather(*tasks, return_exceptions=True)
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
198 if self.channel and self.channel.readyState != "closed":
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
199 self.channel.close()
280
49e9e591c9bb Add persistent dictation, prewarmed WebRTC speech input, Copilot SDK routing, animated conversation lifecycle controls, parking, and architecture coverage.
MrJuneJune <me@mrjunejune.com>
parents: 275
diff changeset
200 await self._cancel_pending_ice_transactions()
275
78699f810817 Add Qwen3-VL and WebRTC dictation services
MrJuneJune <me@mrjunejune.com>
parents:
diff changeset
201 await self.peer.close()