1import * as Schema from "effect/Schema";
2import { replaceTextRange } from "@t3tools/shared/composerTrigger";
3
4import {
5 VoiceTranscriptionError,
6 type PreparedVoiceTranscription,
7 type VoiceTranscriber,
8} from "./transcription.ts";
9
10const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
11
12export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60;
13
14export const MICROPHONE_PERMISSION_ERROR =
15 "Microphone access is required for voice input. Allow access in your browser or system settings and try again.";
16
17function isMicrophonePermissionError(error: unknown): boolean {
18 return (
19 typeof error === "object" &&
20 error !== null &&
21 "name" in error &&
22 error.name === "NotAllowedError"
23 );
24}
25
26export type VoiceInputPhase =
27 | "idle"
28 | "preparing"
29 | "recording"
30 | "transcribing"
31 | "post-processing"
32 | "error";
33
34export type VoiceInputState = {
35 readonly phase: VoiceInputPhase;
36 readonly error: string | null;
37 readonly errorAction: "retry" | "settings" | null;
38};
39
40export function voiceInputBlocksSubmission(state: VoiceInputState): boolean {
41 return (
42 state.phase === "preparing" ||
43 state.phase === "recording" ||
44 state.phase === "transcribing" ||
45 state.phase === "post-processing"
46 );
47}
48
49export type VoiceDraftSnapshot = {
50 readonly ownerKey: string;
51 readonly text: string;
52 readonly selection: { readonly start: number; readonly end: number };
53};
54
55export type VoiceRecorderStatus = {
56 readonly isFinished: boolean;
57 readonly hasError: boolean;
58 readonly error: string | null;
59 readonly url: string | null;
60};
61
62export interface VoiceRecorder {
63 readonly uri: string | null;
64 prepareToRecordAsync(signal?: AbortSignal): Promise<void>;
65 record(options: { readonly forDuration: number }): void;
66 stop(): Promise<void>;
67}
68
69export type VoiceInputControllerDependencies = {
70 readonly recorder: VoiceRecorder;
71 readonly getTranscriber: () => VoiceTranscriber | null;
72 readonly requestPermission: () => Promise<{
73 readonly granted: boolean;
74 readonly canAskAgain: boolean;
75 }>;
76 readonly configureRecording: () => Promise<void>;
77 readonly releaseRecording: () => Promise<void>;
78 readonly deleteRecording: (uri: string) => void;
79 /** Keep a recording whose transcription failed so the next start retries it without recording again. */
80 readonly retryFailedRecording?: boolean;
81 /**
82 * Optional cleanup of the transcript before insertion. Returning null skips it for this
83 * recording; a failure falls back to the raw transcript.
84 */
85 readonly postProcess?:
86 | ((
87 transcript: string,
88 options: { readonly signal: AbortSignal; readonly draft: VoiceDraftSnapshot },
89 ) => Promise<string> | null)
90 | undefined;
91 readonly onPostProcessingError?: (error: unknown) => void;
92 readonly readDraft: () => VoiceDraftSnapshot | null;
93 readonly commitDraft: (
94 text: string,
95 selection: { readonly start: number; readonly end: number },
96 ) => void;
97 readonly onStateChange: (state: VoiceInputState) => void;
98};
99
100type TranscriptCommitResult =
101 | {
102 readonly kind: "commit";
103 readonly text: string;
104 readonly selection: { readonly start: number; readonly end: number };
105 }
106 | { readonly kind: "stale" }
107 | { readonly kind: "empty" };
108
109const WORD_SPACELESS_LANGUAGES = new Set(["ja", "zh", "yue", "th", "lo", "km", "my"]);
110
111const isHighSurrogate = (code: number) => code >= 0xd800 && code <= 0xdbff;
112const isLowSurrogate = (code: number) => code >= 0xdc00 && code <= 0xdfff;
113
114// Where `position` in `before` lands in `text`, placed after anything inserted at it.
115function mapPosition(before: string, text: string, position: number, suffixFirst: boolean) {
116 const shared = Math.min(before.length, text.length);
117 const matchesFromEnd = (offset: number) =>
118 before[before.length - 1 - offset] === text[text.length - 1 - offset];
119 let prefix = 0;
120 let suffix = 0;
121 // Matching is per UTF-16 unit; never let a boundary split a surrogate pair.
122 const scanSuffix = (limit: number) => {
123 while (suffix < limit && matchesFromEnd(suffix)) suffix += 1;
124 if (suffix > 0 && isLowSurrogate(before.charCodeAt(before.length - suffix))) suffix -= 1;
125 };
126 if (suffixFirst) scanSuffix(shared);
127 while (prefix < shared - suffix && before[prefix] === text[prefix]) prefix += 1;
128 if (prefix > 0 && isHighSurrogate(before.charCodeAt(prefix - 1))) prefix -= 1;
129 if (!suffixFirst) scanSuffix(shared - prefix);
130 if (position < prefix) return position;
131 if (position >= before.length - suffix) return position + text.length - before.length;
132 return text.length - suffix;
133}
134
135export function resolveTranscriptCommit(
136 captured: VoiceDraftSnapshot,
137 current: VoiceDraftSnapshot | null,
138 transcript: string,
139 locale: string,
140): TranscriptCommitResult {
141 if (!current || current.ownerKey !== captured.ownerKey) {
142 return { kind: "stale" };
143 }
144
145 const replacement = transcript.trim();
146 if (replacement.length === 0) {
147 return { kind: "empty" };
148 }
149
150 const { text } = current;
151 // An edit made while recording (a citation, a paste) is never replaced. The edit's position is
152 // ambiguous over repeated text, so insert after the selection under both extreme alignments.
153 const unchanged = text === captured.text;
154 const end = unchanged
155 ? captured.selection.end
156 : Math.max(
157 mapPosition(captured.text, text, captured.selection.end, false),
158 mapPosition(captured.text, text, captured.selection.end, true),
159 );
160 const start = unchanged ? captured.selection.start : end;
161 const isEmptySelection = start === end;
162 const language = locale.split(/[-_]/)[0]!.toLowerCase();
163 let insertion = replacement;
164 if (isEmptySelection && !WORD_SPACELESS_LANGUAGES.has(language)) {
165 const auto = language === "auto";
166 const left = text.slice(Math.max(0, start - 2), start);
167 const right = text.slice(start, start + 2);
168 const leftEdge = auto ? /[A-Za-z0-9.!?,:;)\]}'"]$/ : /(?:\p{L}|\p{M}|\p{N}|[.!?,:;)\]}'"])$/u;
169 const rightEdge = auto ? /^[A-Za-z0-9([{'"]/ : /^[\p{L}\p{M}\p{N}([{'"]/u;
170 const leftNeedsBoundary =
171 (!auto || /^[\p{Script=Latin}\p{N}]/u.test(replacement)) &&
172 leftEdge.test(left) &&
173 (!right || /^\s/.test(right));
174 const rightNeedsBoundary =
175 (!auto || /(?:\p{Script=Latin}|\p{N})$/u.test(replacement)) &&
176 rightEdge.test(right) &&
177 (!left || /\s$/.test(left));
178 insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`;
179 }
180
181 const result = replaceTextRange(text, start, end, insertion);
182 return {
183 kind: "commit",
184 text: result.text,
185 selection: { start: result.cursor, end: result.cursor },
186 };
187}
188
189type VoiceInputSession = {
190 abandoned: boolean;
191 readonly released: Promise<void>;
192 readonly release: () => void;
193};
194
195let activeSession: VoiceInputSession | null = null;
196let activeTranscriptionOperation: Promise<unknown> | null = null;
197
198function acquireSession(): VoiceInputSession | null {
199 if (activeSession) return null;
200 let release!: () => void;
201 const released = new Promise<void>((resolve) => {
202 release = resolve;
203 });
204 const token = { abandoned: false, released, release };
205 activeSession = token;
206 return token;
207}
208
209function releaseSession(token: VoiceInputSession | null): void {
210 if (token && activeSession === token) {
211 activeSession = null;
212 token.release();
213 }
214}
215
216async function runTranscriptionOperation<T>(operation: () => Promise<T>): Promise<T> {
217 if (activeTranscriptionOperation) {
218 throw new Error("voice-operation-busy");
219 }
220
221 const promise = operation();
222 activeTranscriptionOperation = promise;
223 try {
224 return await promise;
225 } finally {
226 if (activeTranscriptionOperation === promise) activeTranscriptionOperation = null;
227 }
228}
229
230function errorCode(error: unknown): string | null {
231 if (typeof error !== "object" || error === null || !("code" in error)) return null;
232 return typeof error.code === "string" ? error.code : null;
233}
234
235function preparationErrorMessage(error: unknown): string {
236 if (error instanceof Error && error.message === "voice-operation-busy") {
237 return "Voice transcription is still finishing. Try again shortly.";
238 }
239 if (errorCode(error) === "unsupported-locale") {
240 return "Voice transcription is not available for this language.";
241 }
242 return isVoiceTranscriptionError(error) && error.message.trim()
243 ? error.message
244 : "Could not prepare voice transcription.";
245}
246
247function transcriptionErrorMessage(error: unknown): string {
248 if (error instanceof Error && error.message === "voice-operation-busy") {
249 return "Voice transcription is still finishing. Try again shortly.";
250 }
251 return isVoiceTranscriptionError(error) ? error.message : "Could not transcribe this recording.";
252}
253
254const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
255
256export class VoiceInputController {
257 private readonly dependencies: VoiceInputControllerDependencies;
258 private state: VoiceInputState = IDLE_STATE;
259 private operationToken = 0;
260 private sessionToken: VoiceInputSession | null = null;
261 private transcription: PreparedVoiceTranscription | null = null;
262 private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;
263 private transcriptionAbortController: AbortController | null = null;
264 private postProcessingAbortController: AbortController | null = null;
265 private skipPostProcessingRequested = false;
266 private capturedDraft: VoiceDraftSnapshot | null = null;
267 private recordingUri: string | null = null;
268 private readonly ownedRecordingUris = new Set<string>();
269 private recordingConfigured = false;
270 private finishing = false;
271 private retryRecording: {
272 uri: string;
273 ownerKey: string;
274 transcription: PreparedVoiceTranscription;
275 } | null = null;
276
277 constructor(dependencies: VoiceInputControllerDependencies) {
278 this.dependencies = dependencies;
279 }
280
281 get currentState(): VoiceInputState {
282 return this.state;
283 }
284
285 async start(): Promise<void> {
286 if (this.state.phase !== "idle" && this.state.phase !== "error") return;
287 const initiatingDraft = this.dependencies.readDraft();
288 if (!initiatingDraft) {
289 this.setError("This draft is no longer available.", "retry");
290 return;
291 }
292 let sessionToken: VoiceInputSession | null = null;
293 const operationToken = ++this.operationToken;
294 this.setState({ phase: "preparing", error: null, errorAction: null });
295
296 try {
297 for (
298 let previousSession = activeSession;
299 previousSession?.abandoned;
300 previousSession = activeSession
301 ) {
302 await previousSession.released;
303 if (!this.isCurrent(operationToken)) return;
304 }
305 sessionToken = acquireSession();
306 if (!sessionToken) {
307 this.setError("Another voice recording is already active.", "retry");
308 return;
309 }
310 this.sessionToken = sessionToken;
311 const abortController = new AbortController();
312 this.transcriptionAbortController = abortController;
313
314 if (this.retryRecording && this.retryRecording.ownerKey !== initiatingDraft.ownerKey) {
315 this.retryRecording = null;
316 this.deleteOwnedRecordings();
317 }
318 if (this.retryRecording) {
319 this.recordingUri = this.retryRecording.uri;
320 this.capturedDraft = initiatingDraft;
321 this.transcription = this.retryRecording.transcription;
322 this.retryRecording = null;
323 await this.finishRecording(true, this.recordingUri);
324 return;
325 }
326 const transcriber = this.dependencies.getTranscriber();
327 if (!transcriber) {
328 this.setError("Voice transcription is not available.", null);
329 return;
330 }
331
332 const permission = await this.dependencies.requestPermission();
333 if (!this.isCurrent(operationToken)) return;
334 if (!permission.granted) {
335 this.setError(
336 "Microphone access is required for voice input.",
337 permission.canAskAgain ? "retry" : "settings",
338 );
339 return;
340 }
341
342 try {
343 this.transcription = await runTranscriptionOperation(() =>
344 transcriber.prepare({ signal: abortController.signal }),
345 );
346 } catch (error) {
347 if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry");
348 return;
349 }
350 if (!this.isCurrent(operationToken)) return;
351
352 await this.dependencies.configureRecording();
353 this.recordingConfigured = true;
354 if (!this.isCurrent(operationToken)) return;
355 await this.dependencies.recorder.prepareToRecordAsync(abortController.signal);
356 if (!this.isCurrent(operationToken)) return;
357 this.recordingUri = this.dependencies.recorder.uri;
358 this.rememberRecordingUri(this.recordingUri);
359
360 const capturedDraft = this.dependencies.readDraft();
361 if (!capturedDraft || capturedDraft.ownerKey !== initiatingDraft.ownerKey) {
362 this.setError("This draft is no longer available.", "retry");
363 return;
364 }
365 this.capturedDraft = capturedDraft;
366 if (this.transcription.start && this.recordingUri) {
367 const start = this.transcription.start;
368 const uri = this.recordingUri;
369 this.liveTranscription = runTranscriptionOperation(() =>
370 start(uri, { signal: abortController.signal }),
371 ).then(
372 (text) => ({ text }),
373 (error: unknown) => {
374 if (this.isCurrent(operationToken) && this.state.phase === "recording") {
375 void this.discardRecording(transcriptionErrorMessage(error));
376 }
377 return { error };
378 },
379 );
380 }
381 this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS });
382 this.setState({ phase: "recording", error: null, errorAction: null });
383 } catch (error) {
384 if (this.isCurrent(operationToken))
385 this.setError(
386 isMicrophonePermissionError(error)
387 ? MICROPHONE_PERMISSION_ERROR
388 : "Could not start voice recording.",
389 "retry",
390 );
391 } finally {
392 if (sessionToken && this.sessionToken === sessionToken) {
393 if (this.isCurrent(operationToken) && this.state.phase === "error") {
394 await this.releaseResources();
395 } else if (!this.isCurrent(operationToken) && !this.finishing) {
396 await this.releaseResources();
397 }
398 }
399 }
400 }
401
402 stop(): Promise<void> {
403 if (this.state.phase !== "recording") return Promise.resolve();
404 return this.finishRecording(false, null);
405 }
406
407 cancel(): void {
408 switch (this.state.phase) {
409 case "idle":
410 return;
411 case "error":
412 this.retryRecording = null;
413 this.deleteOwnedRecordings();
414 this.setState(IDLE_STATE);
415 return;
416 case "preparing":
417 this.invalidateOperation();
418 this.setState(IDLE_STATE);
419 return;
420 case "recording":
421 this.discardRecording(null);
422 return;
423 case "transcribing":
424 case "post-processing":
425 this.invalidateOperation();
426 this.setState(IDLE_STATE);
427 return;
428 }
429 }
430
431 skipPostProcessing(): void {
432 if (this.state.phase !== "post-processing") return;
433 this.skipPostProcessingRequested = true;
434 this.postProcessingAbortController?.abort();
435 }
436
437 interruptRecording(
438 message = "Voice recording was interrupted.",
439 completedUri: string | null = null,
440 ): Promise<void> | void {
441 if (this.state.phase === "preparing") {
442 this.invalidateOperation();
443 this.setError(message, "retry");
444 return;
445 }
446 if (this.state.phase !== "recording") return;
447 this.rememberRecordingUri(completedUri);
448 this.recordingUri = completedUri ?? this.recordingUri;
449 return this.discardRecording(message);
450 }
451
452 appMovedToBackground(): Promise<void> | void {
453 return this.interruptRecording(
454 this.state.phase === "preparing"
455 ? "Voice input stopped when the app moved to the background."
456 : undefined,
457 );
458 }
459
460 handleRecorderStatus(status: VoiceRecorderStatus): Promise<void> | void {
461 if (this.state.phase !== "recording") return;
462 if (status.url && this.recordingUri && status.url !== this.recordingUri) return;
463 if (status.hasError) {
464 return this.interruptRecording(
465 status.error ?? "Voice recording was interrupted.",
466 status.url,
467 );
468 }
469 if (status.isFinished) {
470 if (!status.url) {
471 return this.interruptRecording();
472 }
473 return this.finishRecording(true, status.url);
474 }
475 }
476
477 ownerChanged(): void {
478 if (this.state.phase === "idle") return;
479 this.cancel();
480 }
481
482 dispose(): void {
483 switch (this.state.phase) {
484 case "idle":
485 return;
486 case "error":
487 this.cancel();
488 return;
489 case "recording":
490 this.discardRecording(null);
491 return;
492 default:
493 this.invalidateOperation();
494 this.setState(IDLE_STATE);
495 }
496 }
497
498 private async finishRecording(
499 alreadyStopped: boolean,
500 completedUri: string | null,
501 ): Promise<void> {
502 if (this.finishing || (this.state.phase !== "recording" && this.state.phase !== "preparing"))
503 return;
504 this.finishing = true;
505 const operationToken = this.operationToken;
506 this.setState({ phase: "transcribing", error: null, errorAction: null });
507
508 try {
509 if (!alreadyStopped) await this.dependencies.recorder.stop();
510 await this.releaseAudioSession();
511 this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri;
512 this.rememberRecordingUri(this.recordingUri);
513 if (!this.isCurrent(operationToken)) return;
514 const transcription = this.transcription;
515 const recordingUri = this.recordingUri;
516 if (
517 !transcription ||
518 (!transcription.streaming && !recordingUri) ||
519 !this.transcriptionAbortController ||
520 !this.capturedDraft
521 ) {
522 this.setError("Could not finish voice recording.", "retry");
523 return;
524 }
525
526 const signal = this.transcriptionAbortController.signal;
527 const capturedDraft = this.capturedDraft;
528 let transcript: string;
529 try {
530 const live = this.liveTranscription ? await this.liveTranscription : null;
531 if (live && "error" in live) throw live.error;
532 transcript =
533 live?.text ??
534 (await runTranscriptionOperation(() =>
535 transcription.streaming
536 ? transcription.streaming.finish({ signal })
537 : transcription.transcribe(recordingUri!, { signal }),
538 ));
539 } catch (error) {
540 if (this.isCurrent(operationToken)) {
541 if (
542 this.dependencies.retryFailedRecording &&
543 !this.liveTranscription &&
544 !transcription.streaming &&
545 recordingUri
546 ) {
547 this.retryRecording = {
548 uri: recordingUri,
549 ownerKey: capturedDraft.ownerKey,
550 transcription,
551 };
552 }
553 this.setError(transcriptionErrorMessage(error), "retry");
554 }
555 return;
556 }
557 if (!this.isCurrent(operationToken)) return;
558
559 const postProcessingAbortController = new AbortController();
560 const cleanup =
561 transcript.trim().length > 0
562 ? this.dependencies.postProcess?.(transcript, {
563 signal: postProcessingAbortController.signal,
564 draft: capturedDraft,
565 })
566 : null;
567 if (cleanup) {
568 this.setState({ phase: "post-processing", error: null, errorAction: null });
569 this.postProcessingAbortController = postProcessingAbortController;
570 this.skipPostProcessingRequested = false;
571 const rawTranscript = transcript;
572 try {
573 // Skip resolves immediately with the raw transcript; the abort signal tells the
574 // provider request to stop but its settlement is not awaited.
575 transcript = await Promise.race([
576 cleanup,
577 new Promise<string>((resolve) =>
578 postProcessingAbortController.signal.addEventListener(
579 "abort",
580 () => resolve(rawTranscript),
581 { once: true },
582 ),
583 ),
584 ]);
585 } catch (error) {
586 if (!this.skipPostProcessingRequested) {
587 this.dependencies.onPostProcessingError?.(error);
588 }
589 }
590 if (!this.isCurrent(operationToken)) return;
591 }
592
593 const result = resolveTranscriptCommit(
594 capturedDraft,
595 this.dependencies.readDraft(),
596 transcript,
597 transcription.locale,
598 );
599 if (result.kind === "stale") {
600 this.setError(
601 "The draft changed while voice input was running. The transcript was not added.",
602 "retry",
603 );
604 return;
605 }
606 if (result.kind === "empty") {
607 this.setError("No speech was detected.", "retry");
608 return;
609 }
610
611 this.dependencies.commitDraft(result.text, result.selection);
612 this.setState(IDLE_STATE);
613 } catch {
614 if (this.isCurrent(operationToken)) {
615 this.setError("Could not finish voice recording.", "retry");
616 }
617 } finally {
618 this.finishing = false;
619 await this.releaseResources();
620 }
621 }
622
623 private async discardRecording(error: string | null): Promise<void> {
624 this.invalidateOperation();
625 this.setState(
626 error
627 ? { phase: "error", error, errorAction: "retry" }
628 : { phase: "idle", error: null, errorAction: null },
629 );
630 await this.releaseResources();
631 }
632
633 private async releaseResources(): Promise<void> {
634 this.transcriptionAbortController?.abort();
635 this.postProcessingAbortController?.abort();
636 this.rememberRecordingUri(this.recordingUri);
637 try {
638 this.rememberRecordingUri(this.dependencies.recorder.uri);
639 } catch {}
640 if (this.recordingConfigured) {
641 try {
642 await this.dependencies.recorder.stop();
643 } catch {}
644 }
645 await this.liveTranscription;
646 this.liveTranscription = null;
647 this.recordingUri = null;
648 this.deleteOwnedRecordings();
649 await this.releaseAudioSession();
650 releaseSession(this.sessionToken);
651 this.sessionToken = null;
652 this.capturedDraft = null;
653 this.transcription = null;
654 this.transcriptionAbortController = null;
655 this.postProcessingAbortController = null;
656 this.skipPostProcessingRequested = false;
657 }
658
659 private deleteOwnedRecordings(): void {
660 for (const uri of this.ownedRecordingUris) {
661 if (uri === this.retryRecording?.uri) continue;
662 try {
663 this.dependencies.deleteRecording(uri);
664 } catch {}
665 }
666 this.ownedRecordingUris.clear();
667 if (this.retryRecording) this.ownedRecordingUris.add(this.retryRecording.uri);
668 }
669
670 private rememberRecordingUri(uri: string | null): void {
671 if (uri) this.ownedRecordingUris.add(uri);
672 }
673
674 private async releaseAudioSession(): Promise<void> {
675 if (!this.recordingConfigured) return;
676 try {
677 await this.dependencies.releaseRecording();
678 this.recordingConfigured = false;
679 } catch {}
680 }
681
682 private invalidateOperation(): void {
683 this.operationToken += 1;
684 if (this.sessionToken) this.sessionToken.abandoned = true;
685 this.transcriptionAbortController?.abort();
686 this.postProcessingAbortController?.abort();
687 }
688
689 private isCurrent(operationToken: number): boolean {
690 return operationToken === this.operationToken;
691 }
692
693 private setError(error: string, errorAction: VoiceInputState["errorAction"]): void {
694 this.setState({ phase: "error", error, errorAction });
695 }
696
697 private setState(state: VoiceInputState): void {
698 this.state = state;
699 this.dependencies.onStateChange(state);
700 }
701}
702
703export function resetVoiceInputGlobalsForTests(): void {
704 activeSession = null;
705 activeTranscriptionOperation = null;
706}