1// Self-contained because the native worker embeds this factory in Node, Electron and SEA.
2export function createSpeechAudio(config: {
3 frameSamples: number;
4 filterSilence: boolean;
5 nativeStreaming: boolean;
6 longForm: boolean;
7 maxAudioMs: number;
8}) {
9 const rate = 16_000;
10 const frame = config.frameSamples;
11 const prefill = Math.ceil((450 * rate) / 1000 / frame);
12 const onset = Math.ceil((60 * rate) / 1000 / frame);
13 const tail = Math.ceil(((config.nativeStreaming ? 1650 : 450) * rate) / 1000 / frame);
14 const eligible = !config.nativeStreaming && !config.longForm;
15 // Preserve the existing output-limit headroom and bounded offline retries.
16 const maximum = Math.min(
17 40 * rate,
18 config.maxAudioMs > 0 ? Math.max(1, Math.floor((config.maxAudioMs * rate) / 2000)) : 40 * rate,
19 );
20 const minimum = Math.min(5 * rate, Math.floor(maximum / 4));
21 const target = Math.min(30 * rate, maximum * 0.75);
22 const blocks: { start: number; pcm: Float32Array }[] = [];
23 const probs: number[] = [];
24 const keep: boolean[] = [];
25 let length = 0;
26 let start = 0;
27 let streamAt = 0;
28 let inSpeech = false;
29 let onsetCount = 0;
30 let hangover = 0;
31
32 const raw = (from: number, to: number) => {
33 const output = new Float32Array(to - from);
34 for (const block of blocks) {
35 const left = Math.max(from, block.start);
36 const right = Math.min(to, block.start + block.pcm.length);
37 if (left < right)
38 output.set(block.pcm.subarray(left - block.start, right - block.start), left - from);
39 }
40 return output;
41 };
42 const read = (from: number, to: number) => {
43 const pcm = raw(from, to);
44 if (!config.filterSilence) return pcm;
45 let count = 0;
46 for (let at = from; at < to;) {
47 const end = Math.min(to, (Math.floor(at / frame) + 1) * frame);
48 if (keep[Math.floor(at / frame)]) {
49 pcm.copyWithin(count, at - from, end - from);
50 count += end - at;
51 }
52 at = end;
53 }
54 return pcm.subarray(0, count);
55 };
56 // Silero timestamp hysteresis is separate from the capture filter's 0.3 threshold.
57 const pauses = (samples: number, final: boolean) => {
58 const segments: { start: number; end: number }[] = [];
59 let speechStart: number | undefined;
60 let silenceStart: number | undefined;
61 for (let i = 0; i < probs.length; i++) {
62 const at = i * frame;
63 if (probs[i]! >= 0.5) {
64 silenceStart = undefined;
65 speechStart ??= at;
66 } else if (probs[i]! < 0.35 && speechStart !== undefined) {
67 silenceStart ??= at;
68 if (at - silenceStart >= 0.1 * rate) {
69 if (silenceStart - speechStart > 0.25 * rate)
70 segments.push({ start: speechStart, end: silenceStart });
71 speechStart = silenceStart = undefined;
72 }
73 }
74 }
75 if (speechStart !== undefined && samples - speechStart > 0.25 * rate)
76 segments.push({ start: speechStart, end: samples });
77 const candidates: { cut: number; gap: number }[] = [];
78 let end = 0;
79 for (const segment of segments) {
80 if (segment.start > end)
81 candidates.push({
82 cut: Math.floor((end + segment.start) / 2),
83 gap: (segment.start - end) / rate,
84 });
85 end = segment.end;
86 }
87 // An open pause's midpoint can still move, so only final audio uses it.
88 if (final && samples > end)
89 candidates.push({ cut: Math.floor((end + samples) / 2), gap: (samples - end) / rate });
90 return candidates;
91 };
92
93 return {
94 append(pcm: Float32Array) {
95 blocks.push({ start: length, pcm });
96 length += pcm.length;
97 },
98 score(firstFrame: number, scores: readonly number[]) {
99 if (firstFrame !== probs.length || scores.some((p) => !Number.isFinite(p) || p < 0 || p > 1))
100 throw new Error("Invalid Silero frame sequence.");
101 for (const probability of scores) {
102 const i = probs.length;
103 probs.push(probability);
104 keep.push(false);
105 const voiced = probability > 0.3;
106 if (!inSpeech && voiced) {
107 if (++onsetCount >= onset) {
108 inSpeech = true;
109 hangover = tail;
110 onsetCount = 0;
111 for (let j = Math.max(0, i - prefill); j <= i; j++) keep[j] = true;
112 }
113 } else if (inSpeech && voiced) {
114 hangover = tail;
115 keep[i] = true;
116 } else if (inSpeech && hangover > 0) {
117 hangover--;
118 keep[i] = true;
119 } else {
120 inSpeech = false;
121 onsetCount = 0;
122 }
123 }
124 },
125 audio: () => raw(0, length),
126 samples: () => length,
127 take(final = false): Float32Array[] {
128 const samples = Math.min(length, probs.length * frame);
129 // Preserve a very short utterance or syllable stopped before onset confirms.
130 if (final && onsetCount > 0)
131 for (let i = Math.max(0, probs.length - 1 - prefill); i < probs.length; i++) keep[i] = true;
132 if (config.nativeStreaming) {
133 const output = read(streamAt, samples);
134 let lastKept = streamAt;
135 for (let i = Math.floor(streamAt / frame); i < probs.length; i++)
136 if (!config.filterSilence || keep[i]) lastKept = Math.min(samples, (i + 1) * frame);
137 streamAt = Math.max(lastKept, Math.max(0, samples - prefill * frame));
138 return output.length ? [output] : [];
139 }
140 const chunks: Float32Array[] = [];
141 const lookahead = Math.max(minimum, (prefill + onset) * frame);
142 if (eligible && samples - start > maximum + (final ? 0 : lookahead)) {
143 const candidates = pauses(samples, final);
144 while (samples - start > maximum + (final ? 0 : lookahead)) {
145 let best: { cut: number; score: number } | undefined;
146 let fallback: typeof best;
147 for (const candidate of candidates) {
148 const position = candidate.cut - start;
149 if (position < minimum || position > maximum) continue;
150 const offTarget = (position - target) / rate;
151 const score =
152 Math.min(candidate.gap, 1) - Math.abs(offTarget) * (offTarget > 0 ? 0.1 : 0.05);
153 if (!fallback || score > fallback.score) fallback = { cut: candidate.cut, score };
154 if (samples - candidate.cut >= minimum && (!best || score > best.score))
155 best = { cut: candidate.cut, score };
156 }
157 const hard =
158 start +
159 (samples - start < maximum + minimum ? Math.floor((samples - start) / 2) : maximum);
160 const cut = (best ?? fallback)?.cut ?? hard;
161 const chunk = read(start, cut);
162 if (chunk.length) chunks.push(chunk);
163 start = cut;
164 }
165 }
166 if (final && samples > start) {
167 const chunk = read(start, samples);
168 if (chunk.length) chunks.push(chunk);
169 start = samples;
170 }
171 return chunks;
172 },
173 };
174}