Open History to see the last commit
1import {
2 cancelEnvironmentSpeechModelDownload,
3 downloadEnvironmentSpeechModel,
4 getEnvironmentSpeechModels,
5 getEnvironmentSpeechStatus,
6 postProcessEnvironmentTranscript,
7 selectEnvironmentSpeechModel,
8 VoiceInputController,
9 voiceInputBlocksSubmission,
10 type VoiceDraftSnapshot,
11 type VoiceInputControllerDependencies,
12 type VoiceInputState,
13} from "@t3tools/client-runtime/voice-input";
14import type { PreparedConnection } from "@t3tools/client-runtime/connection";
15import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts";
16import type {
17 EnvironmentId,
18 EnvironmentSpeechModel,
19 EnvironmentSpeechStatus,
20 ProjectId,
21 SpeechPostProcessingOptions,
22 SpeechStreamText,
23 SpeechTranscriptionOptions,
24} from "@t3tools/contracts";
25import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
26import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
27import { useNavigate } from "@tanstack/react-router";
28import * as Option from "effect/Option";
29import { useEffect, useRef, useState } from "react";
30
31import { useStableCallback } from "~/hooks/useStableCallback";
32import { toastManager } from "../components/ui/toast";
33import {
34 useClientSettings,
35 useClientSettingsHydrated,
36 useEnvironmentSettings,
37} from "../hooks/useSettings";
38import { useEnvironment } from "../state/environments";
39import { useEnvironmentScope, usePreparedConnection } from "../state/session";
40import { chooseRecordingMicrophone } from "./browserMicrophones";
41import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
42import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
43import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
44import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";
45
46const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
47const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
48const INITIAL_SETUP = {
49 open: false,
50 step: 0,
51 model: null as EnvironmentSpeechModel | null,
52 language: null as string | null,
53 downloading: false,
54 catalogFailed: false,
55 error: null as string | null,
56};
57
58// Every mounted editor shares one status probe per destination; starting a recording refreshes it.
59const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechStatus>>();
60function probeSpeechStatus(prepared: PreparedConnection, refresh = false) {
61 const cached = refresh ? undefined : statusProbes.get(prepared);
62 if (cached) return cached;
63 const probe = runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared));
64 statusProbes.set(prepared, probe);
65 probe.catch(() => {
66 if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);
67 });
68 return probe;
69}
70
71type DraftInput = {
72 readonly text: string;
73 readonly selection: { readonly start: number; readonly end: number };
74};
75
76function cleanupContext({ text, selection }: DraftInput): DraftInput {
77 const limit = 1_024;
78 const start = Math.max(0, Math.min(selection.start, text.length));
79 const end = Math.max(start, Math.min(selection.end, text.length));
80 const before = text.slice(Math.max(0, start - limit), start);
81 const selected = text.slice(start, Math.min(end, start + limit));
82 return {
83 text: before + selected + text.slice(end, end + limit),
84 selection: { start: before.length, end: before.length + selected.length },
85 };
86}
87
88type HookInput = {
89 /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */
90 readonly environmentId: EnvironmentId;
91 /** The draft's project: its vocabulary on that environment and its name become recognition hints. */
92 readonly projectId?: ProjectId | null | undefined;
93 readonly projectName?: string | null | undefined;
94 readonly ownerKey: string;
95 readonly disabled?: boolean | undefined;
96 /**
97 * Until a destination is chosen, voice input is offered and its first gesture opens Voice
98 * settings. False hides it instead, for editors whose draft would not survive the navigation.
99 */
100 readonly offerSetup?: boolean | undefined;
101 /** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */
102 readonly cleanup?: boolean | undefined;
103 readonly readDraft: () => DraftInput;
104 readonly commitDraft: (
105 text: string,
106 selection: { readonly start: number; readonly end: number },
107 ) => void;
108};
109
110/**
111 * Microphone audio leaves this device only for the environment the person chose in
112 * Voice settings. Nothing is inferred from the primary or the draft's environment, and
113 * the recording is abandoned the moment that choice changes or its connection drops.
114 */
115export function useVoiceTranscriptionDestination() {
116 const hydrated = useClientSettingsHydrated();
117 const choice = useAudioChoice();
118 const grant = choice === "off" ? null : choice;
119 const environmentId = hydrated && grant ? grant.destination.environmentId : null;
120 const environment = useEnvironment(environmentId);
121 const connection = Option.getOrNull(usePreparedConnection(environmentId));
122 const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope);
123 const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined;
124 const provider = cloud?.provider;
125 const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
126 const supported =
127 environment?.serverConfig?.environment.capabilities[
128 provider ? "cloudVoiceTranscription" : "voiceTranscription"
129 ] === true;
130 return {
131 ...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}),
132 environmentId,
133 ...(provider ? { cloudProvider: provider } : {}),
134 ...(model ? { cloudModel: model } : {}),
135 grantId: grant?.id ?? null,
136 label:
137 provider && environment
138 ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}`
139 : (environment?.label ?? null),
140 prepared: supported && canDictate ? connection : null,
141 };
142}
143
144export function useEnvironmentSpeechInput(input: HookInput) {
145 const navigate = useNavigate();
146 const destination = useVoiceTranscriptionDestination();
147 const prepared = destination.prepared;
148 const grantId = destination.grantId;
149 const cloudProvider = destination.cloudProvider;
150 const cloudModel = destination.cloudModel;
151 const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment
152 .capabilities;
153 const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId));
154 const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope);
155 const cleanupPrepared =
156 canCleanUp &&
157 (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription)
158 ? originatingConnection
159 : null;
160 const clientSettings = useClientSettings();
161 const projectId = input.projectId ?? null;
162 const projectWords = useEnvironmentSettings(
163 input.environmentId,
164 (settings) => resolveProjectSettings(settings, projectId).settings.speechProjectCustomWords,
165 );
166 const [status, setStatus] = useState<{
167 readonly prepared: PreparedConnection;
168 readonly value: EnvironmentSpeechStatus | null;
169 } | null>(null);
170 const [controllerState, setControllerState] = useState({
171 prepared,
172 grantId,
173 value: INITIAL_STATE,
174 });
175 const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
176 const [level, setLevel] = useState(0);
177 const [preview, setPreview] = useState<SpeechStreamText | null>(null);
178 const [setup, setSetup] = useState(INITIAL_SETUP);
179 const [previousOwner, setPreviousOwner] = useState({
180 key: input.ownerKey,
181 disabled: input.disabled,
182 });
183 if (controllerState.prepared !== prepared || controllerState.grantId !== grantId) {
184 setControllerState({ prepared, grantId, value: INITIAL_STATE });
185 setSetup(INITIAL_SETUP);
186 setQueuedStart(null);
187 setPreview(null);
188 setLevel(0);
189 }
190 if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
191 setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
192 setQueuedStart(null);
193 setSetup(INITIAL_SETUP);
194 }
195 const state: VoiceInputState =
196 queuedStart === prepared && prepared
197 ? WAITING_STATE
198 : controllerState.prepared === prepared
199 ? controllerState.value
200 : INITIAL_STATE;
201
202 const controllerRef = useRef<VoiceInputController | null>(null);
203 const startRequestRef = useRef(0);
204 const pendingSubmissionRef = useRef<{
205 ownerKey: string;
206 submit: (text: string) => void;
207 text?: string;
208 } | null>(null);
209 const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords });
210 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props and settings to the asynchronous recording controller
211 useEffect(() => {
212 latestRef.current = { input, clientSettings, cleanupPrepared, projectWords };
213 });
214 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent
215 useEffect(() => {
216 if (!prepared || !grantId || typeof navigator === "undefined") return;
217 const hasAudioConsent = bindAudioGrant(grantId);
218 let disposed = false;
219 let controller: VoiceInputController;
220 let recordingPreferences:
221 | {
222 readonly cleanup: SpeechPostProcessingOptions | null;
223 readonly cleanupGrant: string | null;
224 readonly environmentId: EnvironmentId;
225 }
226 | undefined;
227
228 const readDraft = (): VoiceDraftSnapshot => {
229 const current = latestRef.current.input;
230 const draft = current.readDraft();
231 return { ownerKey: current.ownerKey, text: draft.text, selection: draft.selection };
232 };
233 const platform = createBrowserVoiceInputPlatform({
234 prepared,
235 cloudProvider,
236 cloudModel,
237 hasAudioConsent,
238 getTranscriptionOptions: (): SpeechTranscriptionOptions => {
239 const {
240 clientSettings: settings,
241 projectWords: project,
242 input: current,
243 } = latestRef.current;
244 const cleanupGrant = current.cleanup === false ? null : readCleanupGrant();
245 const cleanupEnabled = cleanupGrant !== null;
246 const cue =
247 cleanupEnabled && latestRef.current.cleanupPrepared ? settings.speechCorrectionWord : "";
248 const speechCustomWords = resolveSpeechVocabulary({
249 personal: settings.speechCustomWords,
250 project,
251 hints: [cue, current.projectName ?? ""],
252 });
253 recordingPreferences = {
254 cleanupGrant,
255 environmentId: current.environmentId,
256 cleanup: cleanupEnabled
257 ? {
258 speechCustomWords,
259 speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
260 ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
261 }
262 : null,
263 };
264 return {
265 speechLanguage: settings.speechLanguage,
266 speechTranslateToEnglish: settings.speechTranslateToEnglish,
267 speechCustomWords,
268 speechRemoveFillerWords: settings.speechRemoveFillerWords,
269 ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
270 speechSilenceFiltering: settings.speechSilenceFiltering,
271 };
272 },
273 getMicrophoneId: chooseRecordingMicrophone,
274 onLevel: setLevel,
275 onDurationLimit: () => void controller.stop(),
276 onText: (text) => {
277 if (!disposed) setPreview(text);
278 },
279 onError: (message) => {
280 if (!disposed) void controller.interruptRecording(message);
281 },
282 });
283 const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
284 transcript,
285 options,
286 ) => {
287 const preferences = recordingPreferences;
288 const { input: current, cleanupPrepared: target } = latestRef.current;
289 if (
290 !preferences?.cleanup ||
291 !hasAudioConsent() ||
292 preferences.cleanupGrant !== readCleanupGrant() ||
293 current.cleanup === false ||
294 current.environmentId !== preferences.environmentId ||
295 current.ownerKey !== options.draft.ownerKey
296 )
297 return null;
298 if (!target)
299 return Promise.reject(
300 new Error("The draft's environment is unavailable for transcript cleanup."),
301 );
302 return runEnvironmentSpeech(
303 postProcessEnvironmentTranscript(
304 target,
305 transcript,
306 cleanupContext(options.draft),
307 preferences.cleanup,
308 () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
309 ),
310 options,
311 ).then((result) => result.text);
312 };
313 controller = new VoiceInputController({
314 retryFailedRecording: true,
315 recorder: platform.recorder,
316 getTranscriber: () => platform.transcriber,
317 requestPermission: async () => ({ granted: true, canAskAgain: true }),
318 configureRecording: async () => undefined,
319 releaseRecording: async () => platform.cancelRecording(),
320 deleteRecording: platform.deleteRecording,
321 postProcess,
322 onPostProcessingError: () =>
323 toastManager.add({
324 type: "warning",
325 title: "Transcript cleanup failed",
326 description: "The original transcript was added.",
327 }),
328 readDraft,
329 commitDraft: (text, selection) => {
330 if (latestRef.current.input.disabled) return;
331 latestRef.current.input.commitDraft(text, selection);
332 if (pendingSubmissionRef.current) pendingSubmissionRef.current.text = text;
333 },
334 onStateChange: (value) => {
335 if (disposed) return;
336 if (value.phase === "error") pendingSubmissionRef.current = null;
337 setControllerState({ prepared, grantId, value });
338 if (value.phase !== "recording" && value.phase !== "transcribing") setPreview(null);
339 },
340 });
341 controllerRef.current = controller;
342 return () => {
343 disposed = true;
344 pendingSubmissionRef.current = null;
345 startRequestRef.current += 1;
346 setQueuedStart(null);
347 controller.dispose();
348 if (controllerRef.current === controller) controllerRef.current = null;
349 };
350 }, [prepared, grantId, cloudProvider, cloudModel]);
351
352 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
353 useEffect(() => {
354 if (!prepared || cloudProvider) return;
355 let disposed = false;
356 void probeSpeechStatus(prepared)
357 .then((value) => {
358 if (!disposed) setStatus({ prepared, value });
359 })
360 .catch(() => {
361 if (!disposed) setStatus({ prepared, value: null });
362 });
363 return () => {
364 disposed = true;
365 };
366 }, [prepared, cloudProvider]);
367 const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
368 const setupModelId = setup.model?.id;
369
370 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
371 useEffect(() => {
372 if (!setup.open || !setup.downloading || !prepared || !setupModelId) return;
373 let disposed = false;
374 let refreshing = false;
375 const timer = window.setInterval(() => {
376 if (refreshing) return;
377 refreshing = true;
378 void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
379 .then((result) => {
380 if (disposed) return;
381 const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
382 if (model) setSetup((current) => ({ ...current, model }));
383 })
384 .catch(() => undefined)
385 .finally(() => {
386 refreshing = false;
387 });
388 }, 1_000);
389 return () => {
390 disposed = true;
391 window.clearInterval(timer);
392 };
393 }, [prepared, setup.downloading, setup.open, setupModelId]);
394
395 const previousOwnerRef = useRef(input.ownerKey);
396 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
397 useEffect(() => {
398 if (previousOwnerRef.current === input.ownerKey && !input.disabled) return;
399 previousOwnerRef.current = input.ownerKey;
400 pendingSubmissionRef.current = null;
401 startRequestRef.current += 1;
402 controllerRef.current?.ownerChanged();
403 }, [input.ownerKey, input.disabled]);
404
405 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft
406 useEffect(() => {
407 const pending = pendingSubmissionRef.current;
408 if (state.phase !== "idle" || !pending) return;
409 pendingSubmissionRef.current = null;
410 if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined)
411 pending.submit(pending.text);
412 }, [state, input.ownerKey, input.disabled]);
413
414 const submitAfterDictation = useStableCallback((submit: (text: string) => void) => {
415 if (latestRef.current.input.disabled || state.phase === "preparing") return;
416 const controller = controllerRef.current;
417 if (!controller || !voiceInputBlocksSubmission(state)) {
418 submit(latestRef.current.input.readDraft().text);
419 return;
420 }
421 if (pendingSubmissionRef.current) return;
422 pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit };
423 if (controller.currentState.phase === "recording") void controller.stop();
424 });
425
426 const loadSetupCatalog = useStableCallback(
427 async (connection: PreparedConnection, selectedModelId: string, request: number) => {
428 setSetup({ ...INITIAL_SETUP, open: true });
429 const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
430 () => null,
431 );
432 if (request !== startRequestRef.current) return;
433 const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
434 selectedModelId,
435 intent: latestRef.current.clientSettings.speechLanguage,
436 preferred: preferredSpeechLanguages(),
437 });
438 setSetup((current) => ({
439 ...current,
440 model: choice?.model ?? null,
441 language: choice?.language ?? null,
442 catalogFailed: catalog === null,
443 error: choice
444 ? null
445 : catalog
446 ? "No model matches your language. Choose a model or change the language in Voice settings."
447 : "Could not load transcription models. Check the connection and try again.",
448 }));
449 },
450 );
451
452 const start = useStableCallback(async () => {
453 if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return;
454 if (!grantId) {
455 if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)
456 await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
457 return;
458 }
459 const request = ++startRequestRef.current;
460 const controller = controllerRef.current;
461 const owner = latestRef.current.input.ownerKey;
462 if (!controller || !prepared) return;
463 if (cloudProvider) {
464 setLevel(0);
465 await controller.start();
466 return;
467 }
468 const stillCurrent = () =>
469 request === startRequestRef.current &&
470 controllerRef.current === controller &&
471 latestRef.current.input.ownerKey === owner &&
472 !latestRef.current.input.disabled;
473 setSetup(INITIAL_SETUP);
474 setQueuedStart(prepared);
475 let latestStatus: EnvironmentSpeechStatus;
476 try {
477 latestStatus = await probeSpeechStatus(prepared, true);
478 } catch (error) {
479 if (!stillCurrent()) return;
480 setQueuedStart(null);
481 toastManager.add({
482 type: "error",
483 title: "Could not start voice input",
484 description: error instanceof Error ? error.message : String(error),
485 });
486 return;
487 }
488 if (!stillCurrent()) return;
489 setStatus({ prepared, value: latestStatus });
490 setQueuedStart(null);
491 if (latestStatus.supported && latestStatus.state === "missing-model") {
492 return loadSetupCatalog(prepared, latestStatus.modelId, request);
493 }
494 if (!latestStatus.supported) {
495 toastManager.add({
496 type: "error",
497 title: "Voice input is not available",
498 description: latestStatus.reason,
499 });
500 return;
501 }
502 setLevel(0);
503 await controller.start();
504 });
505
506 const downloadSetupModel = useStableCallback(async () => {
507 if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return;
508 const modelId = setupModelId;
509 const request = ++startRequestRef.current;
510 setSetup((current) => ({ ...current, downloading: true, error: null }));
511 try {
512 const { models } = await runEnvironmentSpeech(
513 downloadEnvironmentSpeechModel(prepared, modelId),
514 );
515 if (request !== startRequestRef.current) return;
516 // A download cancelled from elsewhere still resolves, so select the model only once it landed.
517 const installed = models.some(({ id, state }) => id === modelId && state === "installed");
518 if (installed && modelId !== currentStatus.modelId) {
519 try {
520 await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId));
521 } catch {
522 if (request !== startRequestRef.current) return;
523 setSetup((current) => ({
524 ...current,
525 downloading: false,
526 error: "The model downloaded, but this environment could not select it. Try again.",
527 }));
528 return;
529 }
530 }
531 if (request !== startRequestRef.current) return;
532 const next = await probeSpeechStatus(prepared, true);
533 if (request !== startRequestRef.current) return;
534 setStatus({ prepared, value: next });
535 const ready =
536 installed && next.supported && next.modelId === modelId && next.state !== "missing-model";
537 setSetup((current) => ({
538 ...current,
539 downloading: false,
540 step: ready ? 1 : current.step,
541 error: ready ? null : "The model is not ready. Try downloading it again.",
542 }));
543 } catch {
544 if (request !== startRequestRef.current) return;
545 setSetup((current) => ({
546 ...current,
547 downloading: false,
548 error: "Could not download the model. Check this environment's connection and try again.",
549 }));
550 }
551 });
552
553 const cancelSetupDownload = useStableCallback(async () => {
554 if (!prepared || !setupModelId) return;
555 startRequestRef.current += 1;
556 setSetup((current) => ({ ...current, open: false, downloading: false }));
557 try {
558 await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId));
559 } catch (error) {
560 setSetup((current) => ({
561 ...current,
562 error: error instanceof Error ? error.message : String(error),
563 }));
564 }
565 });
566
567 const destinationReady =
568 grantId === null
569 ? Boolean(destination.offerFirstRunSetup) && input.offerSetup !== false
570 : prepared !== null &&
571 (cloudProvider !== undefined ||
572 (status?.prepared === prepared && status.value?.supported !== false));
573 return {
574 available:
575 destinationReady &&
576 typeof navigator !== "undefined" &&
577 Boolean(navigator.mediaDevices?.getUserMedia) &&
578 typeof AudioWorkletNode !== "undefined",
579 status: currentStatus,
580 transcriptionEnvironmentId: destination.environmentId,
581 transcriptionEnvironmentLabel: destination.label,
582 setup: {
583 ...setup,
584 setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
585 retryCatalog:
586 setup.open && setup.catalogFailed && prepared && currentStatus?.supported
587 ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
588 : null,
589 download: downloadSetupModel,
590 cancelDownload: cancelSetupDownload,
591 startRecording: async () => {
592 setSetup((current) => ({ ...current, open: false }));
593 await start();
594 },
595 },
596 state,
597 preview: state.phase === "recording" || state.phase === "transcribing" ? preview : null,
598 level,
599 preparing: state.phase === "preparing",
600 blocksSubmission: voiceInputBlocksSubmission(state),
601 freezesEditor: voiceInputBlocksSubmission(state),
602 start,
603 submitAfterDictation,
604 stop: useStableCallback(() => controllerRef.current?.stop() ?? Promise.resolve()),
605 cancel: useStableCallback(() => {
606 pendingSubmissionRef.current = null;
607 startRequestRef.current += 1;
608 setQueuedStart(null);
609 setSetup(INITIAL_SETUP);
610 controllerRef.current?.cancel();
611 }),
612 skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
613 };
614}