Open History to see the last commit
1import type {
2 PreparedVoiceTranscription,
3 VoiceRecorder,
4 VoiceStreamingSession,
5 VoiceTranscriptionOptions,
6} from "@t3tools/client-runtime/voice-input";
7import workletUrl from "./pcmWorklet.ts?worker&url";
8
9type PreparedBrowserTranscription = PreparedVoiceTranscription & {
10 readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
11};
12
13/** Opens a microphone. A saved device the browser can no longer find falls back to System default. */
14export async function openMicrophone(microphoneId: string): Promise<MediaStream> {
15 if (!microphoneId) return navigator.mediaDevices.getUserMedia({ audio: true });
16 try {
17 return await navigator.mediaDevices.getUserMedia({
18 audio: { deviceId: { exact: microphoneId } },
19 });
20 } catch (error) {
21 const name = typeof error === "object" && error !== null && "name" in error ? error.name : "";
22 if (name !== "OverconstrainedError" && name !== "NotFoundError") throw error;
23 return navigator.mediaDevices.getUserMedia({ audio: true });
24 }
25}
26
27export function createBrowserPcmCapture(
28 input: {
29 readonly getMicrophoneId: () => string | Promise<string>;
30 readonly onLevel: (level: number) => void;
31 readonly onDurationLimit: () => void;
32 readonly onError: (message: string) => void;
33 },
34 prepare: (options: VoiceTranscriptionOptions) => Promise<PreparedBrowserTranscription>,
35) {
36 let stream: MediaStream | undefined;
37 let recordingUri: string | null = null;
38 let chunks: Float32Array<ArrayBuffer>[] = [];
39 let audioContext: AudioContext | undefined;
40 let worklet: AudioWorkletNode | undefined;
41 let durationTimer: ReturnType<typeof setTimeout> | undefined;
42 let signal: AbortSignal | undefined;
43 let feed: ((pcm: Float32Array) => void) | undefined;
44 let stopped:
45 | { readonly resolve: () => void; readonly reject: (error: Error) => void }
46 | undefined;
47
48 const cleanupCapture = () => {
49 if (durationTimer) clearTimeout(durationTimer);
50 durationTimer = undefined;
51 worklet?.disconnect();
52 worklet?.port.close();
53 worklet = undefined;
54 stream?.getTracks().forEach((track) => track.stop());
55 stream = undefined;
56 void audioContext?.close();
57 audioContext = undefined;
58 stopped?.resolve();
59 stopped = undefined;
60 chunks = [];
61 feed = undefined;
62 input.onLevel(0);
63 };
64
65 const recorder: VoiceRecorder = {
66 get uri() {
67 return recordingUri;
68 },
69 prepareToRecordAsync: async () => {
70 const captureSignal = signal;
71 if (!captureSignal) throw new Error("Voice transcription is not prepared.");
72 captureSignal.throwIfAborted();
73 if (recordingUri) URL.revokeObjectURL(recordingUri);
74 recordingUri = null;
75 chunks = [];
76 const microphoneId = await input.getMicrophoneId();
77 captureSignal.throwIfAborted();
78 const captured = await new Promise<MediaStream>((resolve, reject) => {
79 const onAbort = () => reject(captureSignal.reason);
80 const permission = openMicrophone(microphoneId);
81 captureSignal.addEventListener("abort", onAbort, { once: true });
82 void permission.then(
83 (media) => {
84 captureSignal.removeEventListener("abort", onAbort);
85 if (captureSignal.aborted) media.getTracks().forEach((track) => track.stop());
86 else resolve(media);
87 },
88 (error: unknown) => {
89 captureSignal.removeEventListener("abort", onAbort);
90 reject(error);
91 },
92 );
93 });
94 if (captureSignal.aborted) {
95 captured.getTracks().forEach((track) => track.stop());
96 captureSignal.throwIfAborted();
97 }
98 stream = captured;
99 try {
100 const context = new AudioContext();
101 audioContext = context;
102 await context.audioWorklet.addModule(workletUrl);
103 captureSignal.throwIfAborted();
104 const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 });
105 worklet = node;
106 node.port.onmessage = ({
107 data,
108 }: MessageEvent<Float32Array<ArrayBuffer> | number | "stopped" | "limit">) => {
109 if (captureSignal.aborted) return;
110 if (data === "stopped") {
111 stopped?.resolve();
112 return;
113 }
114 if (data === "limit") {
115 input.onDurationLimit();
116 return;
117 }
118 if (typeof data === "number") {
119 input.onLevel(data);
120 return;
121 }
122 if (feed) feed(data);
123 else chunks.push(data);
124 };
125 node.onprocessorerror = () => {
126 stopped?.reject(new Error("Microphone processing failed."));
127 input.onError("Microphone processing failed.");
128 };
129 for (const track of captured.getTracks())
130 track.addEventListener("ended", () => input.onError("The microphone disconnected."), {
131 once: true,
132 });
133 context.createMediaStreamSource(captured).connect(node);
134 // The processor emits silence, keeping the graph active without microphone playback.
135 node.connect(context.destination);
136 await context.resume();
137 captureSignal.throwIfAborted();
138 } catch (error) {
139 cleanupCapture();
140 throw error;
141 }
142 },
143 record: ({ forDuration }) => {
144 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
145 worklet?.port.postMessage("start");
146 durationTimer = setTimeout(input.onDurationLimit, forDuration * 1_000);
147 },
148 stop: async () => {
149 if (!worklet) return;
150 if (signal?.aborted) {
151 cleanupCapture();
152 return;
153 }
154 const pending = new Promise<void>((resolve, reject) => {
155 stopped = { resolve, reject };
156 });
157 const timeout = setTimeout(
158 () => stopped?.reject(new Error("Microphone capture did not stop.")),
159 5_000,
160 );
161 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
162 worklet.port.postMessage("stop");
163 try {
164 await pending;
165 if (!feed && chunks.length) {
166 const blob = new Blob(chunks, { type: "application/octet-stream" });
167 recordingUri = URL.createObjectURL(blob);
168 }
169 } finally {
170 clearTimeout(timeout);
171 cleanupCapture();
172 }
173 },
174 };
175
176 return {
177 recorder,
178 cancelRecording: cleanupCapture,
179 deleteRecording: (uri: string) => {
180 URL.revokeObjectURL(uri);
181 if (recordingUri === uri) recordingUri = null;
182 },
183 transcriber: {
184 prepare: async (options: VoiceTranscriptionOptions) => {
185 const prepared = await prepare(options);
186 options.signal.throwIfAborted();
187 signal = options.signal;
188 feed = prepared.streaming?.feed;
189 return prepared;
190 },
191 },
192 };
193}