Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

controller.ts

683 lines21.9 KB

1import * as Schema from "effect/Schema";
2import { replaceTextRange } from "@t3tools/shared/composerTrigger";
3​
4import {
5 VoiceTranscriptionError,
6 type PreparedVoiceTranscription,
7 type VoiceTranscriber,
8} from "./transcription.ts";
9​
10const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
11​
12export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60;
13​
14export const MICROPHONE_PERMISSION_ERROR =
15 "Microphone access is required for voice input. Allow access in your browser or system settings and try again.";
16​
17function isMicrophonePermissionError(error: unknown): boolean {
18 return (
19 typeof error === "object" &&
20 error !== null &&
21 "name" in error &&
22 error.name === "NotAllowedError"
23 );
25​
26export type VoiceInputPhase =
27 | "idle"
28 | "preparing"
29 | "recording"
30 | "transcribing"
31 | "post-processing"
32 | "error";
33​
34export type VoiceInputState = {
35 readonly phase: VoiceInputPhase;
36 readonly error: string | null;
37 readonly errorAction: "retry" | "settings" | null;
38};
39​
40export function voiceInputBlocksSubmission(state: VoiceInputState): boolean {
41 return (
42 state.phase === "preparing" ||
43 state.phase === "recording" ||
44 state.phase === "transcribing" ||
45 state.phase === "post-processing"
46 );
48​
49export type VoiceDraftSnapshot = {
50 readonly ownerKey: string;
51 readonly text: string;
52 readonly selection: { readonly start: number; readonly end: number };
53 readonly revision: number;
54};
55​
56export type VoiceRecorderStatus = {
57 readonly isFinished: boolean;
58 readonly hasError: boolean;
59 readonly error: string | null;
60 readonly url: string | null;
61};
62​
63export interface VoiceRecorder {
64 readonly uri: string | null;
65 prepareToRecordAsync(signal?: AbortSignal): Promise<void>;
66 record(options: { readonly forDuration: number }): void;
67 stop(): Promise<void>;
69​
70export type VoiceInputControllerDependencies = {
71 readonly recorder: VoiceRecorder;
72 readonly getTranscriber: () => VoiceTranscriber | null;
73 readonly requestPermission: () => Promise<{
74 readonly granted: boolean;
75 readonly canAskAgain: boolean;
76 }>;
77 readonly configureRecording: () => Promise<void>;
78 readonly releaseRecording: () => Promise<void>;
79 readonly deleteRecording: (uri: string) => void;
80 /** Keep a recording whose transcription failed so the next start retries it without recording again. */
81 readonly retryFailedRecording?: boolean;
82 /**
83 * Optional cleanup of the transcript before insertion. Returning null skips it for this
84 * recording; a failure falls back to the raw transcript.
85 */
86 readonly postProcess?:
87 | ((
88 transcript: string,
89 options: { readonly signal: AbortSignal; readonly draft: VoiceDraftSnapshot },
90 ) => Promise<string> | null)
91 | undefined;
92 readonly onPostProcessingError?: (error: unknown) => void;
93 readonly readDraft: () => VoiceDraftSnapshot | null;
94 readonly commitDraft: (
95 text: string,
96 selection: { readonly start: number; readonly end: number },
97 ) => void;
98 readonly onStateChange: (state: VoiceInputState) => void;
99};
100​
101type TranscriptCommitResult =
102 | {
103 readonly kind: "commit";
104 readonly text: string;
105 readonly selection: { readonly start: number; readonly end: number };
106 }
107 | { readonly kind: "stale" }
108 | { readonly kind: "empty" };
109​
110const WORD_SPACELESS_LANGUAGES = new Set(["ja", "zh", "yue", "th", "lo", "km", "my"]);
111​
112export function resolveTranscriptCommit(
113 captured: VoiceDraftSnapshot,
114 current: VoiceDraftSnapshot | null,
115 transcript: string,
116 locale: string,
117): TranscriptCommitResult {
118 if (
119 !current ||
120 current.ownerKey !== captured.ownerKey ||
121 current.text !== captured.text ||
122 current.revision !== captured.revision
123 ) {
124 return { kind: "stale" };
125 }
126​
127 const replacement = transcript.trim();
128 if (replacement.length === 0) {
129 return { kind: "empty" };
130 }
131​
132 const isEmptySelection = captured.selection.start === captured.selection.end;
133 const language = locale.split(/[-_]/)[0]!.toLowerCase();
134 let insertion = replacement;
135 if (isEmptySelection && !WORD_SPACELESS_LANGUAGES.has(language)) {
136 const auto = language === "auto";
137 const caret = captured.selection.start;
138 const left = captured.text.slice(Math.max(0, caret - 2), caret);
139 const right = captured.text.slice(caret, caret + 2);
140 const leftEdge = auto ? /[A-Za-z0-9.!?,:;)\]}'"]$/ : /(?:\p{L}|\p{M}|\p{N}|[.!?,:;)\]}'"])$/u;
141 const rightEdge = auto ? /^[A-Za-z0-9([{'"]/ : /^[\p{L}\p{M}\p{N}([{'"]/u;
142 const leftNeedsBoundary =
143 (!auto || /^[\p{Script=Latin}\p{N}]/u.test(replacement)) &&
144 leftEdge.test(left) &&
145 (!right || /^\s/.test(right));
146 const rightNeedsBoundary =
147 (!auto || /(?:\p{Script=Latin}|\p{N})$/u.test(replacement)) &&
148 rightEdge.test(right) &&
149 (!left || /\s$/.test(left));
150 insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`;
151 }
152​
153 const result = replaceTextRange(
154 captured.text,
155 captured.selection.start,
156 captured.selection.end,
157 insertion,
158 );
159 return {
160 kind: "commit",
161 text: result.text,
162 selection: { start: result.cursor, end: result.cursor },
163 };
165​
166type VoiceInputSession = {
167 abandoned: boolean;
168 readonly released: Promise<void>;
169 readonly release: () => void;
170};
171​
172let activeSession: VoiceInputSession | null = null;
173let activeTranscriptionOperation: Promise<unknown> | null = null;
174​
175function acquireSession(): VoiceInputSession | null {
176 if (activeSession) return null;
177 let release!: () => void;
178 const released = new Promise<void>((resolve) => {
179 release = resolve;
180 });
181 const token = { abandoned: false, released, release };
182 activeSession = token;
183 return token;
185​
186function releaseSession(token: VoiceInputSession | null): void {
187 if (token && activeSession === token) {
188 activeSession = null;
189 token.release();
190 }
192​
193async function runTranscriptionOperation<T>(operation: () => Promise<T>): Promise<T> {
194 if (activeTranscriptionOperation) {
195 throw new Error("voice-operation-busy");
196 }
197​
198 const promise = operation();
199 activeTranscriptionOperation = promise;
200 try {
201 return await promise;
202 } finally {
203 if (activeTranscriptionOperation === promise) activeTranscriptionOperation = null;
204 }
206​
207function errorCode(error: unknown): string | null {
208 if (typeof error !== "object" || error === null || !("code" in error)) return null;
209 return typeof error.code === "string" ? error.code : null;
211​
212function preparationErrorMessage(error: unknown): string {
213 if (error instanceof Error && error.message === "voice-operation-busy") {
214 return "Voice transcription is still finishing. Try again shortly.";
215 }
216 if (errorCode(error) === "unsupported-locale") {
217 return "Voice transcription is not available for this language.";
218 }
219 return isVoiceTranscriptionError(error) && error.message.trim()
220 ? error.message
221 : "Could not prepare voice transcription.";
223​
224function transcriptionErrorMessage(error: unknown): string {
225 if (error instanceof Error && error.message === "voice-operation-busy") {
226 return "Voice transcription is still finishing. Try again shortly.";
227 }
228 return isVoiceTranscriptionError(error) ? error.message : "Could not transcribe this recording.";
230​
231const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
232​
233export class VoiceInputController {
234 private readonly dependencies: VoiceInputControllerDependencies;
235 private state: VoiceInputState = IDLE_STATE;
236 private operationToken = 0;
237 private sessionToken: VoiceInputSession | null = null;
238 private transcription: PreparedVoiceTranscription | null = null;
239 private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;
240 private transcriptionAbortController: AbortController | null = null;
241 private postProcessingAbortController: AbortController | null = null;
242 private skipPostProcessingRequested = false;
243 private capturedDraft: VoiceDraftSnapshot | null = null;
244 private recordingUri: string | null = null;
245 private readonly ownedRecordingUris = new Set<string>();
246 private recordingConfigured = false;
247 private finishing = false;
248 private retryRecording: {
249 uri: string;
250 ownerKey: string;
251 transcription: PreparedVoiceTranscription;
252 } | null = null;
253​
254 constructor(dependencies: VoiceInputControllerDependencies) {
255 this.dependencies = dependencies;
256 }
257​
258 get currentState(): VoiceInputState {
259 return this.state;
260 }
261​
262 async start(): Promise<void> {
263 if (this.state.phase !== "idle" && this.state.phase !== "error") return;
264 const initiatingDraft = this.dependencies.readDraft();
265 if (!initiatingDraft) {
266 this.setError("This draft is no longer available.", "retry");
267 return;
268 }
269 let sessionToken: VoiceInputSession | null = null;
270 const operationToken = ++this.operationToken;
271 this.setState({ phase: "preparing", error: null, errorAction: null });
272​
273 try {
274 for (
275 let previousSession = activeSession;
276 previousSession?.abandoned;
277 previousSession = activeSession
278 ) {
279 await previousSession.released;
280 if (!this.isCurrent(operationToken)) return;
281 }
282 sessionToken = acquireSession();
283 if (!sessionToken) {
284 this.setError("Another voice recording is already active.", "retry");
285 return;
286 }
287 this.sessionToken = sessionToken;
288 const abortController = new AbortController();
289 this.transcriptionAbortController = abortController;
290​
291 if (this.retryRecording && this.retryRecording.ownerKey !== initiatingDraft.ownerKey) {
292 this.retryRecording = null;
293 this.deleteOwnedRecordings();
294 }
295 if (this.retryRecording) {
296 this.recordingUri = this.retryRecording.uri;
297 this.capturedDraft = initiatingDraft;
298 this.transcription = this.retryRecording.transcription;
299 this.retryRecording = null;
300 await this.finishRecording(true, this.recordingUri);
301 return;
302 }
303 const transcriber = this.dependencies.getTranscriber();
304 if (!transcriber) {
305 this.setError("Voice transcription is not available.", null);
306 return;
307 }
308​
309 const permission = await this.dependencies.requestPermission();
310 if (!this.isCurrent(operationToken)) return;
311 if (!permission.granted) {
312 this.setError(
313 "Microphone access is required for voice input.",
314 permission.canAskAgain ? "retry" : "settings",
315 );
316 return;
317 }
318​
319 try {
320 this.transcription = await runTranscriptionOperation(() =>
321 transcriber.prepare({ signal: abortController.signal }),
322 );
323 } catch (error) {
324 if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry");
325 return;
326 }
327 if (!this.isCurrent(operationToken)) return;
328​
329 await this.dependencies.configureRecording();
330 this.recordingConfigured = true;
331 if (!this.isCurrent(operationToken)) return;
332 await this.dependencies.recorder.prepareToRecordAsync(abortController.signal);
333 if (!this.isCurrent(operationToken)) return;
334 this.recordingUri = this.dependencies.recorder.uri;
335 this.rememberRecordingUri(this.recordingUri);
336​
337 const capturedDraft = this.dependencies.readDraft();
338 if (!capturedDraft || capturedDraft.ownerKey !== initiatingDraft.ownerKey) {
339 this.setError("This draft is no longer available.", "retry");
340 return;
341 }
342 this.capturedDraft = capturedDraft;
343 if (this.transcription.start && this.recordingUri) {
344 const start = this.transcription.start;
345 const uri = this.recordingUri;
346 this.liveTranscription = runTranscriptionOperation(() =>
347 start(uri, { signal: abortController.signal }),
348 ).then(
349 (text) => ({ text }),
350 (error: unknown) => {
351 if (this.isCurrent(operationToken) && this.state.phase === "recording") {
352 void this.discardRecording(transcriptionErrorMessage(error));
353 }
354 return { error };
355 },
356 );
357 }
358 this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS });
359 this.setState({ phase: "recording", error: null, errorAction: null });
360 } catch (error) {
361 if (this.isCurrent(operationToken))
362 this.setError(
363 isMicrophonePermissionError(error)
364 ? MICROPHONE_PERMISSION_ERROR
365 : "Could not start voice recording.",
366 "retry",
367 );
368 } finally {
369 if (sessionToken && this.sessionToken === sessionToken) {
370 if (this.isCurrent(operationToken) && this.state.phase === "error") {
371 await this.releaseResources();
372 } else if (!this.isCurrent(operationToken) && !this.finishing) {
373 await this.releaseResources();
374 }
375 }
376 }
377 }
378​
379 stop(): Promise<void> {
380 if (this.state.phase !== "recording") return Promise.resolve();
381 return this.finishRecording(false, null);
382 }
383​
384 cancel(): void {
385 switch (this.state.phase) {
386 case "idle":
387 return;
388 case "error":
389 this.retryRecording = null;
390 this.deleteOwnedRecordings();
391 this.setState(IDLE_STATE);
392 return;
393 case "preparing":
394 this.invalidateOperation();
395 this.setState(IDLE_STATE);
396 return;
397 case "recording":
398 this.discardRecording(null);
399 return;
400 case "transcribing":
401 case "post-processing":
402 this.invalidateOperation();
403 this.setState(IDLE_STATE);
404 return;
405 }
406 }
407​
408 skipPostProcessing(): void {
409 if (this.state.phase !== "post-processing") return;
410 this.skipPostProcessingRequested = true;
411 this.postProcessingAbortController?.abort();
412 }
413​
414 interruptRecording(
415 message = "Voice recording was interrupted.",
416 completedUri: string | null = null,
417 ): Promise<void> | void {
418 if (this.state.phase === "preparing") {
419 this.invalidateOperation();
420 this.setError(message, "retry");
421 return;
422 }
423 if (this.state.phase !== "recording") return;
424 this.rememberRecordingUri(completedUri);
425 this.recordingUri = completedUri ?? this.recordingUri;
426 return this.discardRecording(message);
427 }
428​
429 appMovedToBackground(): Promise<void> | void {
430 return this.interruptRecording(
431 this.state.phase === "preparing"
432 ? "Voice input stopped when the app moved to the background."
433 : undefined,
434 );
435 }
436​
437 handleRecorderStatus(status: VoiceRecorderStatus): Promise<void> | void {
438 if (this.state.phase !== "recording") return;
439 if (status.url && this.recordingUri && status.url !== this.recordingUri) return;
440 if (status.hasError) {
441 return this.interruptRecording(
442 status.error ?? "Voice recording was interrupted.",
443 status.url,
444 );
445 }
446 if (status.isFinished) {
447 if (!status.url) {
448 return this.interruptRecording();
449 }
450 return this.finishRecording(true, status.url);
451 }
452 }
453​
454 ownerChanged(): void {
455 if (this.state.phase === "idle") return;
456 this.cancel();
457 }
458​
459 dispose(): void {
460 switch (this.state.phase) {
461 case "idle":
462 return;
463 case "error":
464 this.cancel();
465 return;
466 case "recording":
467 this.discardRecording(null);
468 return;
469 default:
470 this.invalidateOperation();
471 this.setState(IDLE_STATE);
472 }
473 }
474​
475 private async finishRecording(
476 alreadyStopped: boolean,
477 completedUri: string | null,
478 ): Promise<void> {
479 if (this.finishing || (this.state.phase !== "recording" && this.state.phase !== "preparing"))
480 return;
481 this.finishing = true;
482 const operationToken = this.operationToken;
483 this.setState({ phase: "transcribing", error: null, errorAction: null });
484​
485 try {
486 if (!alreadyStopped) await this.dependencies.recorder.stop();
487 await this.releaseAudioSession();
488 this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri;
489 this.rememberRecordingUri(this.recordingUri);
490 if (!this.isCurrent(operationToken)) return;
491 const transcription = this.transcription;
492 const recordingUri = this.recordingUri;
493 if (
494 !transcription ||
495 (!transcription.streaming && !recordingUri) ||
496 !this.transcriptionAbortController ||
497 !this.capturedDraft
498 ) {
499 this.setError("Could not finish voice recording.", "retry");
500 return;
501 }
502​
503 const signal = this.transcriptionAbortController.signal;
504 const capturedDraft = this.capturedDraft;
505 let transcript: string;
506 try {
507 const live = this.liveTranscription ? await this.liveTranscription : null;
508 if (live && "error" in live) throw live.error;
509 transcript =
510 live?.text ??
511 (await runTranscriptionOperation(() =>
512 transcription.streaming
513 ? transcription.streaming.finish({ signal })
514 : transcription.transcribe(recordingUri!, { signal }),
515 ));
516 } catch (error) {
517 if (this.isCurrent(operationToken)) {
518 if (
519 this.dependencies.retryFailedRecording &&
520 !this.liveTranscription &&
521 !transcription.streaming &&
522 recordingUri
523 ) {
524 this.retryRecording = {
525 uri: recordingUri,
526 ownerKey: capturedDraft.ownerKey,
527 transcription,
528 };
529 }
530 this.setError(transcriptionErrorMessage(error), "retry");
531 }
532 return;
533 }
534 if (!this.isCurrent(operationToken)) return;
535​
536 const postProcessingAbortController = new AbortController();
537 const cleanup =
538 transcript.trim().length > 0
539 ? this.dependencies.postProcess?.(transcript, {
540 signal: postProcessingAbortController.signal,
541 draft: capturedDraft,
542 })
543 : null;
544 if (cleanup) {
545 this.setState({ phase: "post-processing", error: null, errorAction: null });
546 this.postProcessingAbortController = postProcessingAbortController;
547 this.skipPostProcessingRequested = false;
548 const rawTranscript = transcript;
549 try {
550 // Skip resolves immediately with the raw transcript; the abort signal tells the
551 // provider request to stop but its settlement is not awaited.
552 transcript = await Promise.race([
553 cleanup,
554 new Promise<string>((resolve) =>
555 postProcessingAbortController.signal.addEventListener(
556 "abort",
557 () => resolve(rawTranscript),
558 { once: true },
559 ),
560 ),
561 ]);
562 } catch (error) {
563 if (!this.skipPostProcessingRequested) {
564 this.dependencies.onPostProcessingError?.(error);
565 }
566 }
567 if (!this.isCurrent(operationToken)) return;
568 }
569​
570 const result = resolveTranscriptCommit(
571 capturedDraft,
572 this.dependencies.readDraft(),
573 transcript,
574 transcription.locale,
575 );
576 if (result.kind === "stale") {
577 this.setError(
578 "The draft changed while voice input was running. The transcript was not added.",
579 "retry",
580 );
581 return;
582 }
583 if (result.kind === "empty") {
584 this.setError("No speech was detected.", "retry");
585 return;
586 }
587​
588 this.dependencies.commitDraft(result.text, result.selection);
589 this.setState(IDLE_STATE);
590 } catch {
591 if (this.isCurrent(operationToken)) {
592 this.setError("Could not finish voice recording.", "retry");
593 }
594 } finally {
595 this.finishing = false;
596 await this.releaseResources();
597 }
598 }
599​
600 private async discardRecording(error: string | null): Promise<void> {
601 this.invalidateOperation();
602 this.setState(
603 error
604 ? { phase: "error", error, errorAction: "retry" }
605 : { phase: "idle", error: null, errorAction: null },
606 );
607 await this.releaseResources();
608 }
609​
610 private async releaseResources(): Promise<void> {
611 this.transcriptionAbortController?.abort();
612 this.postProcessingAbortController?.abort();
613 this.rememberRecordingUri(this.recordingUri);
614 try {
615 this.rememberRecordingUri(this.dependencies.recorder.uri);
616 } catch {}
617 if (this.recordingConfigured) {
618 try {
619 await this.dependencies.recorder.stop();
620 } catch {}
621 }
622 await this.liveTranscription;
623 this.liveTranscription = null;
624 this.recordingUri = null;
625 this.deleteOwnedRecordings();
626 await this.releaseAudioSession();
627 releaseSession(this.sessionToken);
628 this.sessionToken = null;
629 this.capturedDraft = null;
630 this.transcription = null;
631 this.transcriptionAbortController = null;
632 this.postProcessingAbortController = null;
633 this.skipPostProcessingRequested = false;
634 }
635​
636 private deleteOwnedRecordings(): void {
637 for (const uri of this.ownedRecordingUris) {
638 if (uri === this.retryRecording?.uri) continue;
639 try {
640 this.dependencies.deleteRecording(uri);
641 } catch {}
642 }
643 this.ownedRecordingUris.clear();
644 if (this.retryRecording) this.ownedRecordingUris.add(this.retryRecording.uri);
645 }
646​
647 private rememberRecordingUri(uri: string | null): void {
648 if (uri) this.ownedRecordingUris.add(uri);
649 }
650​
651 private async releaseAudioSession(): Promise<void> {
652 if (!this.recordingConfigured) return;
653 try {
654 await this.dependencies.releaseRecording();
655 this.recordingConfigured = false;
656 } catch {}
657 }
658​
659 private invalidateOperation(): void {
660 this.operationToken += 1;
661 if (this.sessionToken) this.sessionToken.abandoned = true;
662 this.transcriptionAbortController?.abort();
663 this.postProcessingAbortController?.abort();
664 }
665​
666 private isCurrent(operationToken: number): boolean {
667 return operationToken === this.operationToken;
668 }
669​
670 private setError(error: string, errorAction: VoiceInputState["errorAction"]): void {
671 this.setState({ phase: "error", error, errorAction });
672 }
673​
674 private setState(state: VoiceInputState): void {
675 this.state = state;
676 this.dependencies.onStateChange(state);
677 }
679​
680export function resetVoiceInputGlobalsForTests(): void {
681 activeSession = null;
682 activeTranscriptionOperation = null;