Open History to see the last commit
1import {
2 AuthOrchestrationOperateScope,
3 type CloudSpeechModelId,
4 type CloudSpeechProvider,
5 type EnvironmentId,
6 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
7 type VoiceTranscriptionDestination,
8} from "@t3tools/contracts";
9import {
10 getEnvironmentSpeechModels,
11 getCloudSpeechKeys,
12 getCloudSpeechModels,
13 requireCloudSpeechKey,
14 transcribeCloudPcm,
15 getEnvironmentSpeechStatus,
16 transcribeEnvironmentPcm,
17 VoiceTranscriptionError,
18} from "@t3tools/client-runtime/voice-input";
19import { FetchHttpClient, type HttpClient } from "effect/http";
20import type { PreparedConnection } from "@t3tools/client-runtime/connection";
21import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
22import * as Effect from "effect/Effect";
23import * as Option from "effect/Option";
24import { Atom, AsyncResult, AtomRegistry } from "effect/reactivity";
25import { requireOptionalNativeModule } from "expo";
26import { fetch as expoFetch } from "expo/fetch";
27
28import { appAtomRegistry } from "../../state/atom-registry";
29import { environmentPresentations } from "../../state/presentation";
30import { environmentSession, sessionHasScope } from "../../state/session";
31import { connectionAtomRuntime } from "../../connection/runtime";
32import { uuidv4 } from "../../lib/uuid";
33import { createEnvironmentTranscriber } from "./environmentTranscriber";
34import { readVoiceGrant, writeVoiceGrant, type VoiceGrant } from "./voiceGrantStorage";
35
36const storedGrant = Atom.make(Effect.promise(readVoiceGrant)).pipe(Atom.keepAlive);
37const choice = Atom.make<VoiceGrant | null | undefined>(undefined).pipe(Atom.keepAlive);
38export const voiceGrantAtom = Atom.make((get) => {
39 const chosen = get(choice);
40 if (chosen !== undefined) return chosen;
41 return AsyncResult.getOrElse(get(storedGrant), () => null);
42});
43
44let choiceRevision = 0;
45
46type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
47/** OpenRouter has no default model, so a cloud choice without one is no choice at all. */
48function cloudVoiceDestination(
49 environmentId: EnvironmentId,
50 provider: CloudSpeechProvider,
51 model?: CloudSpeechModelId,
52): CloudDestination | null {
53 if (provider === "openrouter")
54 return model ? { kind: "cloud", environmentId, provider, model } : null;
55 return { kind: "cloud", environmentId, provider };
56}
57
58export function chooseVoiceDestination(
59 environmentId: EnvironmentId | null,
60 provider?: CloudSpeechProvider,
61 model?: CloudSpeechModelId,
62) {
63 choiceRevision++;
64 // A failed change must not leave the old destination armed in this session.
65 appAtomRegistry.set(choice, null);
66 const destination: VoiceTranscriptionDestination | null =
67 environmentId === null
68 ? null
69 : provider
70 ? cloudVoiceDestination(environmentId, provider, model)
71 : { kind: "environment", environmentId };
72 const grant: VoiceGrant | null = destination && { id: uuidv4(), destination };
73 writeVoiceGrant(grant);
74 appAtomRegistry.set(choice, grant);
75}
76
77export function pendingCloudVoiceChoice(
78 id: EnvironmentId,
79 provider: CloudSpeechProvider,
80 model?: CloudSpeechModelId,
81) {
82 const revision = ++choiceRevision;
83 return () => {
84 if (revision === choiceRevision) chooseVoiceDestination(id, provider, model);
85 };
86}
87
88const native = requireOptionalNativeModule<{
89 decodeRecording?: (uri: string) => Promise<Uint8Array>;
90}>("T3VoiceTranscription");
91export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function";
92
93const voiceConnections = (capability: "voiceTranscription" | "cloudVoiceTranscription") =>
94 Atom.family((id: EnvironmentId) =>
95 Atom.make((get) => {
96 const environment = get(environmentPresentations.presentationAtom(id));
97 if (
98 !hasEnvironmentVoiceCapture ||
99 environment?.connection.phase !== "connected" ||
100 !environment.serverConfig?.environment.capabilities[capability] ||
101 !sessionHasScope(
102 get(environmentSession.sessionStateAtom(id)),
103 AuthOrchestrationOperateScope,
104 )
105 )
106 return null;
107 return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
108 }),
109 );
110export const voiceConnectionAtom = voiceConnections("voiceTranscription");
111export const cloudVoiceConnectionAtom = voiceConnections("cloudVoiceTranscription");
112
113// React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error".
114export function runSpeech<A, E>(
115 effect: Effect.Effect<A, E, HttpClient.HttpClient>,
116 signal?: AbortSignal,
117 canSend: () => boolean = () => true,
118) {
119 return Effect.runPromise(
120 AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
121 Effect.flatMap((context) =>
122 effect.pipe(
123 Effect.provideService(FetchHttpClient.Fetch, async (url, options) => {
124 if (signal?.aborted || !canSend())
125 throw new VoiceTranscriptionError({
126 code: "cancelled",
127 message: "Your transcription choice changed.",
128 });
129 return expoFetch(url, { ...options, redirect: "error" });
130 }),
131 Effect.provide(context),
132 ),
133 ),
134 ),
135 { signal },
136 );
137}
138function environmentTranscriber(
139 prepared: PreparedConnection,
140 canSend: () => boolean,
141 cloud?: CloudDestination,
142) {
143 const provider = cloud?.provider;
144 const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
145 return createEnvironmentTranscriber({
146 canSend,
147 prepare: async (signal) => {
148 if (provider) {
149 await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend);
150 return "auto";
151 }
152 const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend);
153 if (!status.supported)
154 throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
155 if (status.state === "missing-model")
156 throw new VoiceTranscriptionError({
157 code: "unavailable",
158 message:
159 "Install a transcription model for this environment in Settings → Voice, then try again.",
160 });
161 return status.languages
162 ? effectiveSpeechLanguage(
163 {
164 languages: status.languages,
165 supportsLanguageDetection: status.supportsLanguageDetection ?? false,
166 },
167 "auto",
168 )
169 : "auto";
170 },
171 decode: (uri) => {
172 if (!native?.decodeRecording)
173 throw new VoiceTranscriptionError({
174 code: "unavailable",
175 message: "Update T3 Code to transcribe on an environment.",
176 });
177 return native.decodeRecording(uri);
178 },
179 upload: async (pcm, signal) =>
180 (
181 await runSpeech(
182 provider
183 ? transcribeCloudPcm(
184 prepared,
185 provider,
186 pcm,
187 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
188 canSend,
189 model,
190 )
191 : transcribeEnvironmentPcm(
192 prepared,
193 pcm,
194 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
195 canSend,
196 ),
197 signal,
198 canSend,
199 )
200 ).text,
201 });
202}
203
204const speechAtom = <A, E>(
205 id: EnvironmentId,
206 read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
207 connectionAtom = voiceConnectionAtom,
208) =>
209 Atom.make((get) => {
210 const prepared = get(connectionAtom(id));
211 return prepared
212 ? Effect.tryPromise({
213 try: (signal) => runSpeech(read(prepared), signal),
214 catch: (cause) =>
215 new VoiceTranscriptionError({
216 code: "unavailable",
217 message: cause instanceof Error ? cause.message : String(cause),
218 }),
219 })
220 : Effect.succeed(null);
221 });
222
223export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) =>
224 speechAtom(id, getEnvironmentSpeechStatus),
225);
226export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) =>
227 speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom),
228);
229export const cloudVoiceModelsAtom = Atom.family((id: EnvironmentId | null) =>
230 id === null
231 ? Atom.make(Effect.succeed(null))
232 : speechAtom(id, getCloudSpeechModels, cloudVoiceConnectionAtom),
233);
234export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) =>
235 speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)),
236);
237// Fetched once for the model sheet; download progress polls the lighter list above.
238export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) =>
239 speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared, true)),
240);
241
242// null selects on-device; a grant with no usable transcriber must never fall back.
243export const voiceRouteAtom = Atom.make((get) => {
244 const grant = get(voiceGrantAtom);
245 if (!grant) return null;
246 const cloud = grant.destination.kind === "cloud" ? grant.destination : undefined;
247 const connectionAtom = (cloud ? cloudVoiceConnectionAtom : voiceConnectionAtom)(
248 grant.destination.environmentId,
249 );
250 const connection = get(connectionAtom);
251 const registry = get.registry;
252 return {
253 transcriber:
254 connection &&
255 environmentTranscriber(
256 connection,
257 () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
258 cloud,
259 ),
260 };
261});