Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

browserVoiceInput.ts

127 lines4.8 KB

1import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
2import {
3 requireCloudSpeechKey,
4 transcribeCloudPcm,
5 getEnvironmentSpeechStatus,
6 getEnvironmentSpeechStreamUrl,
7 openSpeechStream,
8 throwIfVoiceTranscriptionAborted,
9 transcribeEnvironmentPcm,
10 VoiceTranscriptionError,
11 type VoiceTranscriptionOptions,
12} from "@t3tools/client-runtime/voice-input";
13import type { PreparedConnection } from "@t3tools/client-runtime/connection";
14import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
15import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts";
16import * as Schema from "effect/Schema";
17​
18import { createBrowserPcmCapture } from "./browserPcmCapture";
19import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
20​
21const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
22​
23export function createBrowserVoiceInputPlatform(input: {
24 readonly prepared: PreparedConnection;
25 readonly cloudProvider?: CloudSpeechProvider | undefined;
26 /** The exact OpenRouter model the grant names; required with that provider. */
27 readonly cloudModel?: CloudSpeechModelId | undefined;
28 readonly hasAudioConsent: () => boolean;
29 readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
30 readonly getMicrophoneId: () => string | Promise<string>;
31 readonly onLevel: (level: number) => void;
32 readonly onDurationLimit: () => void;
33 readonly onText: (text: SpeechStreamText) => void;
34 readonly onError: (message: string) => void;
35}) {
36 return createBrowserPcmCapture(input, async (options) => {
37 const recordingOptions = input.getTranscriptionOptions();
38 const transcribeRecording = async (
39 uri: string,
40 { signal: transcriptionSignal }: VoiceTranscriptionOptions,
41 ) => {
42 try {
43 const response = await fetch(uri, { signal: transcriptionSignal });
44 const pcm = new Uint8Array(await response.arrayBuffer());
45 const result = await runEnvironmentSpeech(
46 input.cloudProvider
47 ? transcribeCloudPcm(
48 input.prepared,
49 input.cloudProvider,
50 pcm,
51 recordingOptions,
52 input.hasAudioConsent,
53 input.cloudModel,
54 )
55 : transcribeEnvironmentPcm(
56 input.prepared,
57 pcm,
58 recordingOptions,
59 input.hasAudioConsent,
60 ),
61 { signal: transcriptionSignal },
62 );
63 throwIfVoiceTranscriptionAborted(transcriptionSignal);
64 return result.text;
65 } catch (cause) {
66 throwIfVoiceTranscriptionAborted(transcriptionSignal);
67 if (isVoiceTranscriptionError(cause)) throw cause;
68 throw new VoiceTranscriptionError({
69 code: "transcription-failed",
70 message: "Voice transcription failed. Check the destination in Settings → Voice.",
71 cause,
72 });
73 }
74 };
75 if (input.cloudProvider) {
76 if (input.cloudProvider === "openrouter" && !input.cloudModel)
77 throw new VoiceTranscriptionError({
78 code: "unavailable",
79 message: "Choose an OpenRouter model in Settings → Voice.",
80 });
81 await runEnvironmentSpeech(
82 requireCloudSpeechKey(input.prepared, input.cloudProvider),
83 options,
84 );
85 throwIfVoiceTranscriptionAborted(options.signal);
86 return {
87 locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage),
88 transcribe: transcribeRecording,
89 };
90 }
91 const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
92 throwIfVoiceTranscriptionAborted(options.signal);
93 if (!status.supported)
94 throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
95 const language = status.languages
96 ? effectiveSpeechLanguage(
97 {
98 languages: status.languages,
99 supportsLanguageDetection: status.supportsLanguageDetection ?? false,
100 },
101 recordingOptions.speechLanguage,
102 )
103 : "auto";
104 const locale =
105 !status.supportsStreaming &&
106 status.supportsTranslation &&
107 recordingOptions.speechTranslateToEnglish
108 ? "en"
109 : language;
110 if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
111 return { locale, transcribe: transcribeRecording };
112 }
113 const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
114 return {
115 locale,
116 transcribe: transcribeRecording,
117 streaming: openSpeechStream({
118 url,
119 canSend: input.hasAudioConsent,
120 options: recordingOptions,
121 signal: options.signal,
122 onText: input.onText,
123 onError: (error) => input.onError(error.message),
124 }),
125 };
126 });