1import { beforeEach, expect, vi } from "vite-plus/test";
2import { it } from "@effect/vitest";
3import * as NodeServices from "@effect/platform-node/NodeServices";
4import * as Effect from "effect/Effect";
5import * as Deferred from "effect/Deferred";
6import * as TestClock from "effect/testing/TestClock";
7import * as Fiber from "effect/Fiber";
8import * as Layer from "effect/Layer";
9import * as Result from "effect/Result";
10import * as Scheduler from "effect/Scheduler";
11import * as Scope from "effect/Scope";
12import * as Exit from "effect/Exit";
13import * as PlatformError from "effect/PlatformError";
14import * as HostProcess from "@t3tools/shared/HostProcess";
15import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
16import * as ServerSettings from "../serverSettings.ts";
17import { listNativeSpeechGpuDevices } from "./native.ts";
18import * as SpeechModelStore from "./SpeechModelStore.ts";
19import * as SpeechService from "./SpeechService.ts";
20import { SILERO_VAD } from "./silero.ts";
21
22const native = vi.hoisted(() => ({
23 backend: "Vulkan0",
24 supportsStreaming: true,
25 supportsLongForm: false,
26 supportsTranslation: true,
27 supportsVocabulary: false,
28 languages: ["en", "fr"] as readonly string[],
29 begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
30 feed: vi.fn(async (_pcm: Float32Array) => ({
31 revision: 1,
32 text: { committed: "", tentative: "hello" },
33 })),
34 finish: vi.fn(async () => "hello"),
35 reset: vi.fn(async () => {}),
36 dispose: vi.fn(),
37 transcribe: vi.fn(async (_pcm: Float32Array, _options?: unknown) => ({ text: "hello" })),
38}));
39const loadNative = vi.hoisted(() =>
40 vi.fn(
41 async (
42 _path: string,
43 _signal: AbortSignal,
44 _moduleUrl?: string,
45 backend: "auto" | "cpu" = "auto",
46 ) => {
47 native.backend = backend === "cpu" ? "CPU" : "Vulkan0";
48 return native;
49 },
50 ),
51);
52const readyModels = vi.hoisted(() => new Set(["test-model", "fallback-model"]));
53const modelDefinitions = vi.hoisted(() => [
54 {
55 id: "test-model",
56 name: "test",
57 description: "test",
58 size: 4,
59 languages: ["en", "fr"],
60 accuracy: 1,
61 benchmark: {
62 accuracy: { en: { grade: "A-", errorPercent: 4 } },
63 secondsPer30s: { cpu: 3, accelerated: 1.2 },
64 },
65 speed: 1,
66 recommended: true,
67 supportsStreaming: true,
68 supportsLanguageDetection: false,
69 supportsTranslation: true,
70 },
71 {
72 id: "fallback-model",
73 name: "fallback",
74 description: "fallback",
75 size: 8,
76 languages: ["en", "fr"],
77 accuracy: 2,
78 speed: 2,
79 recommended: false,
80 supportsStreaming: false,
81 supportsLanguageDetection: false,
82 supportsTranslation: true,
83 },
84]);
85vi.mock("./native.ts", () => ({
86 loadNativeSpeechModel: loadNative,
87 listNativeSpeechGpuDevices: vi.fn(async () => [{ id: '["vulkan","gpu-1"]', name: "Test GPU" }]),
88}));
89vi.mock("./model.ts", () => ({
90 DEFAULT_SPEECH_MODEL_ID: "test-model",
91 SPEECH_MODELS: modelDefinitions,
92 getSpeechModel: (modelId: string) => modelDefinitions.find((model) => model.id === modelId),
93 effectiveSpeechLanguage: (model: { languages: string[] }, intent: string) =>
94 intent !== "auto" && model.languages.includes(intent) ? intent : "en",
95}));
96const download = vi.fn<SpeechModelStore.SpeechModelStore["Service"]["download"]>(() =>
97 Effect.succeed("test.gguf"),
98);
99const store = Layer.succeed(
100 SpeechModelStore.SpeechModelStore,
101 SpeechModelStore.SpeechModelStore.of({
102 isReady: (model) => Effect.sync(() => readyModels.has(model.id)),
103 installedPath: (model) =>
104 Effect.sync(() => (readyModels.has(model.id) ? "test.gguf" : undefined)),
105 download,
106 link: () => Effect.succeed([]),
107 linkedFrom: () => Effect.undefined,
108 remove: (model) =>
109 Effect.sync(() => {
110 readyModels.delete(model.id);
111 }),
112 }),
113);
114const layer = SpeechService.layer.pipe(
115 Layer.provide(store),
116 Layer.provideMerge(ServerSettings.layerTest({ speechModelId: "test-model" })),
117 Layer.provide(NodeServices.layer),
118);
119const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer);
120
121beforeEach(() => {
122 vi.clearAllMocks();
123 loadNative.mockReset();
124 native.finish.mockReset();
125 native.backend = "Vulkan0";
126 native.supportsVocabulary = false;
127 native.supportsStreaming = true;
128 native.languages = ["en", "fr"];
129 readyModels.clear();
130 readyModels.add("test-model");
131 readyModels.add("fallback-model");
132});
133
134it("decodes a pooled Node Buffer without reading adjacent bytes", () => {
135 const pool = Buffer.allocUnsafe(64);
136 pool.fill(0xff);
137 Buffer.from(new Float32Array([0.25]).buffer).copy(pool, 20);
138 expect([...SpeechService.decodeSpeechPcm(pool.subarray(20, 24))]).toEqual([0.25]);
139});
140
141it("rejects empty, misaligned, non-finite, and oversized PCM as invalid audio", () => {
142 for (const input of [
143 new Uint8Array(),
144 new Uint8Array(3),
145 new Uint8Array(new Float32Array([Number.NaN]).buffer),
146 new Uint8Array(SpeechService.MAX_SPEECH_BYTES + 4),
147 ]) {
148 expect(() => SpeechService.decodeSpeechPcm(input)).toThrow(
149 expect.objectContaining({ _tag: "SpeechInvalidAudioError" }),
150 );
151 }
152});
153
154it.effect.each(["CPU", "Vulkan0"])(
155 "prepares a batch model on %s before audio arrives and reuses it for transcription",
156 (backend) =>
157 Effect.gen(function* () {
158 const speech = yield* SpeechService.SpeechService;
159 yield* speech.selectModel("fallback-model");
160 loadNative.mockImplementationOnce(async () => {
161 native.backend = backend;
162 return native;
163 });
164 yield* speech.prepareModel;
165 expect(yield* speech.status).toMatchObject({
166 benchmarkTier: backend === "CPU" ? "cpu" : "accelerated",
167 });
168 expect(loadNative).toHaveBeenCalledTimes(1);
169 expect(native.transcribe).not.toHaveBeenCalled();
170 native.supportsStreaming = false;
171 yield* Effect.gen(function* () {
172 const stream = yield* speech.startStream({
173 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
174 speechLanguage: "fr",
175 speechTranslateToEnglish: true,
176 speechSilenceFiltering: false,
177 });
178 expect(native.begin).toHaveBeenLastCalledWith(
179 { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
180 false,
181 "test.gguf",
182 );
183 expect(yield* stream.finish).toBe("hello");
184 }).pipe(Effect.scoped);
185 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
186 expect(loadNative).toHaveBeenCalledTimes(1);
187 }).pipe(Effect.provide(layer)),
188);
189
190it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
191 Effect.gen(function* () {
192 const speech = yield* SpeechService.SpeechService;
193 // Nemotron Streaming 3.5 advertises regional tags while the catalog lists bare codes.
194 native.languages = ["en-US", "en-GB", "fr-FR"];
195 yield* Effect.gen(function* () {
196 const stream = yield* speech.startStream({
197 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
198 speechLanguage: "fr",
199 });
200 expect(native.begin).toHaveBeenLastCalledWith(
201 { timestamps: "none", language: "fr-FR" },
202 true,
203 "test.gguf",
204 );
205 expect(yield* stream.finish).toBe("hello");
206 }).pipe(Effect.scoped);
207 native.supportsStreaming = false;
208 expect(
209 yield* speech.transcribe(pcm(), {
210 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
211 speechLanguage: "fr",
212 }),
213 ).toBe("hello");
214 expect(native.transcribe).toHaveBeenLastCalledWith(
215 expect.any(Float32Array),
216 { timestamps: "none", language: "fr-FR" },
217 true,
218 "test.gguf",
219 );
220 // A language the model does not advertise reaches it unchanged, so the model reports it.
221 yield* speech.updateAcceleration("cpu");
222 native.languages = ["en-US"];
223 native.transcribe.mockRejectedValueOnce(new Error("transcribe_run: unsupported language"));
224 const rejected = yield* Effect.result(
225 speech.transcribe(pcm(), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }),
226 );
227 expect(Result.isFailure(rejected) && rejected.failure).toMatchObject({
228 _tag: "SpeechOperationError",
229 operation: "inference",
230 });
231 expect(native.transcribe).toHaveBeenLastCalledWith(
232 expect.any(Float32Array),
233 { timestamps: "none", language: "fr" },
234 true,
235 "test.gguf",
236 );
237 expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual([
238 ["test.gguf", "auto"],
239 ["test.gguf", "cpu"],
240 ]);
241 }).pipe(Effect.provide(layer)),
242);
243
244it.effect.each(["cancellation", "timeout"] as const)(
245 "releases a stalled model preparation on %s",
246 (reason) =>
247 Effect.gen(function* () {
248 const speech = yield* SpeechService.SpeechService;
249 const started = Promise.withResolvers<AbortSignal>();
250 const stalled = Promise.withResolvers<typeof native>();
251 const disposed = Promise.withResolvers<void>();
252 const abandoned = { ...native, dispose: vi.fn(() => disposed.resolve()) };
253 loadNative.mockImplementationOnce((_path, signal) => {
254 started.resolve(signal);
255 return stalled.promise;
256 });
257 const request = yield* speech.prepareModel.pipe(Effect.forkChild);
258 const signal = yield* Effect.promise(() => started.promise);
259 try {
260 if (reason === "cancellation") {
261 yield* Fiber.interrupt(request).pipe(Effect.forkChild({ startImmediately: true }));
262 }
263 yield* TestClock.adjust(reason === "timeout" ? "2 minutes" : 0);
264 expect(signal.aborted).toBe(true);
265 const exit = yield* Fiber.await(request);
266 expect(Exit.isFailure(exit)).toBe(true);
267 if (reason === "timeout") {
268 expect(Exit.findErrorOption(exit)).toMatchObject({
269 value: { _tag: "SpeechOperationError", operation: "model preparation" },
270 });
271 }
272 yield* speech.selectModel("fallback-model");
273 const transcribe = speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
274 expect(yield* transcribe).toBe("hello");
275 stalled.resolve(abandoned);
276 yield* Effect.promise(() => disposed.promise);
277 expect(yield* transcribe).toBe("hello");
278 expect(loadNative).toHaveBeenCalledTimes(2);
279 } finally {
280 stalled.resolve(abandoned);
281 }
282 }).pipe(Effect.provide(layer)),
283);
284
285it.effect("rejects model mutations outside the offered registry", () =>
286 Effect.gen(function* () {
287 const speech = yield* SpeechService.SpeechService;
288 for (const id of [
289 "handy-computer/canary-1b-gguf",
290 "handy-computer/granite-speech-5.0-470m-turboctc-nc-gguf",
291 "unknown-model",
292 ]) {
293 for (const mutation of [speech.downloadModel, speech.selectModel, speech.removeModel]) {
294 const result = yield* Effect.result(mutation(id));
295 expect(Result.isFailure(result) && result.failure).toMatchObject({
296 _tag: "SpeechModelNotFoundError",
297 modelId: id,
298 });
299 }
300 }
301 expect(download).not.toHaveBeenCalled();
302 const current = (yield* speech.models(false)).models.find((model) => model.active);
303 expect(current?.id).toBe("test-model");
304 expect(current).not.toHaveProperty("benchmark");
305 expect(
306 (yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
307 ).toEqual({ cpu: 3, accelerated: 1.2 });
308 }).pipe(Effect.provide(layer)),
309);
310
311it.effect("keeps a detached download exclusive until the transfer finishes", () =>
312 Effect.gen(function* () {
313 const speech = yield* SpeechService.SpeechService;
314 const started = yield* Deferred.make<void>();
315 const finish = yield* Deferred.make<void>();
316 const helperStarted = yield* Deferred.make<void>();
317 let helperStartedDuringInstall = false;
318 let helperInterrupted = false;
319 download
320 .mockImplementationOnce(() =>
321 Deferred.succeed(started, undefined).pipe(
322 Effect.andThen(Deferred.await(finish)),
323 Effect.as("test.gguf"),
324 ),
325 )
326 .mockImplementationOnce(() =>
327 Deferred.isDone(finish).pipe(
328 Effect.tap((done) =>
329 Effect.sync(() => {
330 helperStartedDuringInstall = !done;
331 }),
332 ),
333 Effect.andThen(Deferred.succeed(helperStarted, undefined)),
334 Effect.andThen(Effect.never),
335 Effect.onInterrupt(() =>
336 Effect.sync(() => {
337 helperInterrupted = true;
338 }),
339 ),
340 ),
341 );
342 const request = yield* speech.downloadModel("test-model").pipe(Effect.forkChild);
343 yield* Deferred.await(started);
344 yield* Deferred.await(helperStarted);
345 yield* Fiber.interrupt(request);
346 yield* Deferred.succeed(finish, undefined);
347 yield* TestClock.adjust("16 seconds");
348 expect(helperInterrupted).toBe(false);
349 const removed = yield* Effect.result(speech.removeModel("test-model"));
350 expect(Result.isFailure(removed) && removed.failure).toMatchObject({ _tag: "SpeechBusyError" });
351 const join = yield* speech
352 .downloadModel("test-model")
353 .pipe(Effect.forkChild({ startImmediately: true }));
354 yield* TestClock.adjust("2 minutes");
355 expect(helperInterrupted).toBe(true);
356 yield* Fiber.join(join);
357 expect(download.mock.calls.map(([model]) => model.id)).toEqual(["test-model", SILERO_VAD.id]);
358 expect(helperStartedDuringInstall).toBe(true);
359 yield* speech.removeModel("test-model");
360 expect(readyModels.has("test-model")).toBe(false);
361 }).pipe(Effect.provide(layer)),
362);
363
364it.effect("bounds missing Silero recovery during recording and continues with raw audio", () =>
365 Effect.gen(function* () {
366 const helperStarted = yield* Deferred.make<void>();
367 const helperInterrupted = yield* Deferred.make<void>();
368 download.mockImplementationOnce(() =>
369 Deferred.succeed(helperStarted, undefined).pipe(
370 Effect.andThen(Effect.never),
371 Effect.onInterrupt(() => Deferred.succeed(helperInterrupted, undefined)),
372 ),
373 );
374 const speech = yield* SpeechService.SpeechService;
375 const recording = yield* speech
376 .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
377 .pipe(Effect.forkChild);
378 yield* Deferred.await(helperStarted);
379 yield* TestClock.adjust("14999 millis");
380 expect(native.begin).not.toHaveBeenCalled();
381 yield* TestClock.adjust("1 millis");
382 yield* Deferred.await(helperInterrupted);
383 const stream = yield* Fiber.join(recording);
384 expect(native.begin).toHaveBeenLastCalledWith(expect.anything(), true, undefined);
385 yield* stream.feed(pcm());
386 expect(yield* stream.finish).toBe("hello");
387 }).pipe(Effect.scoped, Effect.provide(layer)),
388);
389
390it.effect("keeps the installed speech model usable when optional Silero acquisition fails", () =>
391 Effect.gen(function* () {
392 download
393 .mockImplementationOnce(() => Effect.succeed("test.gguf"))
394 .mockImplementationOnce(() =>
395 Effect.fail(
396 new PlatformError.PlatformError(
397 new PlatformError.SystemError({
398 _tag: "NotFound",
399 module: "FileSystem",
400 method: "open",
401 }),
402 ),
403 ),
404 );
405 const speech = yield* SpeechService.SpeechService;
406 const installed = yield* speech.downloadModel("test-model");
407 expect(installed.models.find((model) => model.id === "test-model")?.state).toBe("installed");
408 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
409 yield* Effect.scoped(
410 Effect.gen(function* () {
411 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
412 expect(yield* stream.finish).toBe("hello");
413 }),
414 );
415 expect(download).toHaveBeenCalledTimes(2);
416 yield* TestClock.adjust("10 minutes");
417 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
418 expect(download).toHaveBeenCalledTimes(3);
419 }).pipe(Effect.provide(layer)),
420);
421
422it.effect(
423 "allows another model operation after an already-installed download completes immediately",
424 () =>
425 Effect.gen(function* () {
426 const speech = yield* SpeechService.SpeechService;
427 const models = yield* speech.downloadModel("test-model").pipe(
428 Effect.provideService(Scheduler.Scheduler, {
429 executionMode: "sync",
430 shouldYield: () => false,
431 makeDispatcher: () => ({ scheduleTask: (task) => task(), flush: () => {} }),
432 }),
433 );
434 expect(models.models.find((model) => model.id === "test-model")?.state).toBe("installed");
435 yield* speech.selectModel("fallback-model");
436 }).pipe(Effect.provide(layer)),
437);
438
439it.effect("honors exact aliases and phrase boundaries in batch and streamed text", () =>
440 Effect.gen(function* () {
441 const speech = yield* SpeechService.SpeechService;
442 const options = {
443 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
444 speechCustomWords: [
445 { term: "AI", aliases: ["ay eye"] },
446 { term: "Bob", aliases: ["Robert"] },
447 { term: "Roberts", aliases: [] },
448 { term: "O'Reilly", aliases: ['"the publisher"'] },
449 { term: "GitHub", aliases: [] },
450 { term: "McDonald’s", aliases: [] },
451 { term: "King’s Cross", aliases: [] },
452 { term: "C#", aliases: [] },
453 ],
454 };
455 const spoken =
456 'a test with a i and ay eye. Robert, Roberts. "the publisher" and the publisher. (git hub), git (hub). ' +
457 "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross. vitamin c and plan c.";
458 const expected =
459 "a test with AI and AI. Bob, Roberts. O'Reilly and the publisher. (GitHub), git (hub). " +
460 "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross. vitamin c and plan c.";
461 native.transcribe.mockResolvedValueOnce({ text: spoken });
462 native.finish.mockResolvedValueOnce(spoken);
463 expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
464 const stream = yield* speech.startStream(options);
465 expect(yield* stream.finish).toBe(expected);
466 }).pipe(Effect.scoped, Effect.provide(layer)),
467);
468
469it.effect("uses vocabulary only for supported transcription and always applies aliases", () =>
470 Effect.gen(function* () {
471 for (const supportsVocabulary of [false, true]) {
472 for (const translate of [false, true]) {
473 yield* Effect.gen(function* () {
474 native.supportsVocabulary = supportsVocabulary;
475 const speech = yield* SpeechService.SpeechService;
476 const options = {
477 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
478 speechLanguage: "fr" as const,
479 speechTranslateToEnglish: translate,
480 speechCustomWords: [
481 { term: "GitHub", aliases: [] },
482 { term: "Bob", aliases: ["Robert"] },
483 { term: "T3 Code", aliases: [] },
484 ],
485 };
486 const vocabulary =
487 supportsVocabulary && !translate ? ["GitHub", "Bob", "T3 Code"] : undefined;
488 native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert" });
489 expect(yield* speech.transcribe(pcm(), options)).toBe(
490 vocabulary ? "git hub and Bob" : "GitHub and Bob",
491 );
492 const delivered = native.transcribe.mock.lastCall?.[1];
493 expect(delivered).toEqual({
494 timestamps: "none",
495 language: "fr",
496 ...(translate ? { task: "translate", targetLanguage: "en" } : {}),
497 ...(vocabulary ? { vocabulary } : {}),
498 });
499 if (translate) return;
500 native.feed.mockResolvedValueOnce({
501 revision: 1,
502 text: { committed: "git hub", tentative: "Robert" },
503 });
504 native.finish.mockResolvedValueOnce("git hub and Robert");
505 const stream = yield* speech.startStream(options);
506 expect(yield* stream.feed(pcm())).toEqual({
507 revision: 1,
508 text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
509 });
510 expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
511 expect(native.begin.mock.lastCall).toEqual([
512 { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
513 true,
514 "test.gguf",
515 ]);
516 }).pipe(Effect.scoped, Effect.provide(layer));
517 }
518 }
519 }),
520);
521
522it.effect("unloads after the chosen idle period and reloads on the next recording", () =>
523 Effect.gen(function* () {
524 const speech = yield* SpeechService.SpeechService;
525 yield* speech.updateModelUnloadTimeout("min_2");
526 yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
527 yield* TestClock.adjust(2 * 60_000 - 1);
528 expect(native.dispose).not.toHaveBeenCalled();
529 yield* TestClock.adjust(1);
530 expect(native.dispose).toHaveBeenCalledOnce();
531 yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
532 expect(loadNative).toHaveBeenCalledTimes(2);
533 }).pipe(Effect.provide(layer)),
534);
535
536it.effect.each([
537 {
538 cue: undefined,
539 remove: true,
540 text: "First paragraph.\n\nSecond paragraph.",
541 expected: "First paragraph.\n\nSecond paragraph.",
542 },
543 {
544 cue: undefined,
545 remove: true,
546 text: "First um\n\nuh Second.",
547 expected: "First\n\nSecond.",
548 },
549 {
550 cue: undefined,
551 remove: true,
552 text: "First um\r\n\r\nuh Second.",
553 expected: "First\r\n\r\nSecond.",
554 },
555 { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." },
556 {
557 cue: "uh no",
558 remove: true,
559 text: "Make it red, uh no, blue.",
560 expected: "Make it red, uh no, blue.",
561 },
562 {
563 cue: "uh no",
564 remove: true,
565 text: "Uh, Make it red, UH NO, blue. Uh nope.",
566 expected: "Make it red, UH NO, blue. nope.",
567 },
568 {
569 cue: " UM ",
570 remove: true,
571 text: "Uh, Make it red, um, blue.",
572 expected: "Make it red, um, blue.",
573 },
574 {
575 cue: undefined,
576 remove: true,
577 text: "Uh, Make it red, um, blue.",
578 expected: "Make it red, blue.",
579 },
580 {
581 cue: "um",
582 remove: false,
583 text: "Uh, Make it red, um, blue.",
584 expected: "Uh, Make it red, um, blue.",
585 },
586])(
587 "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove",
588 ({ cue, remove, text, expected }) =>
589 Effect.gen(function* () {
590 const speech = yield* SpeechService.SpeechService;
591 const options = {
592 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
593 speechLanguage: "en",
594 speechRemoveFillerWords: remove,
595 speechCustomWords: [{ term: "um", aliases: [] }],
596 ...(cue === undefined ? {} : { speechCorrectionWord: cue }),
597 };
598 native.transcribe.mockResolvedValueOnce({ text });
599 native.finish.mockResolvedValueOnce(text);
600 expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
601 const stream = yield* speech.startStream(options);
602 expect(yield* stream.finish).toBe(expected);
603 }).pipe(Effect.scoped, Effect.provide(layer)),
604);
605
606it.effect("holds model ownership until the stream scope closes and preserves silence", () =>
607 Effect.gen(function* () {
608 const speech = yield* SpeechService.SpeechService;
609 yield* Effect.gen(function* () {
610 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
611 const busy = yield* Effect.result(speech.selectModel("fallback-model"));
612 expect(Result.isFailure(busy) && busy.failure).toMatchObject({ _tag: "SpeechBusyError" });
613 yield* stream.feed(new Uint8Array(new Float32Array(160).buffer));
614 expect(native.feed.mock.calls[0]?.[0]).toHaveLength(160);
615 expect(yield* stream.finish).toBe("hello");
616 }).pipe(Effect.scoped);
617 expect(native.dispose).not.toHaveBeenCalled();
618 yield* speech.selectModel("fallback-model");
619 expect(native.dispose).toHaveBeenCalledOnce();
620 }).pipe(Effect.provide(layer)),
621);
622
623it.effect("resets cancelled streams and reuses the loaded model", () =>
624 Effect.gen(function* () {
625 const speech = yield* SpeechService.SpeechService;
626 yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS).pipe(Effect.scoped);
627 expect(native.reset).toHaveBeenCalledOnce();
628 expect(native.dispose).not.toHaveBeenCalled();
629 yield* Effect.gen(function* () {
630 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
631 yield* stream.feed(pcm());
632 yield* stream.finish;
633 }).pipe(Effect.scoped);
634 expect(loadNative).toHaveBeenCalledOnce();
635 }).pipe(Effect.provide(layer)),
636);
637
638it.effect("releases a failed stream start without waiting for its socket scope to close", () =>
639 Effect.gen(function* () {
640 const speech = yield* SpeechService.SpeechService;
641 const socket = yield* Scope.fork(yield* Effect.scope);
642 yield* speech.updateAcceleration("cpu");
643 native.begin.mockRejectedValueOnce(new Error("cannot start streaming"));
644 const error = yield* speech
645 .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
646 .pipe(Scope.provide(socket), Effect.flip);
647 expect(error._tag).toBe("SpeechOperationError");
648 expect(yield* speech.status).toMatchObject({ state: "ready" });
649 expect(native.dispose).toHaveBeenCalledOnce();
650 const next = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
651 yield* Scope.close(socket, Exit.void);
652 expect(yield* speech.status).toMatchObject({ state: "transcribing" });
653 expect(yield* next.finish).toBe("hello");
654 }).pipe(Effect.scoped, Effect.provide(layer)),
655);
656
657it.effect("closes its scope without waiting for hung native inference", () =>
658 Effect.gen(function* () {
659 const started = Promise.withResolvers<void>();
660 native.transcribe.mockImplementationOnce(() => {
661 started.resolve();
662 return new Promise(() => {});
663 });
664 yield* Effect.gen(function* () {
665 const speech = yield* SpeechService.SpeechService;
666 const request = yield* speech
667 .transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
668 .pipe(Effect.forkChild);
669 yield* Effect.promise(() => started.promise);
670 yield* Fiber.interrupt(request);
671 }).pipe(Effect.provide(layer));
672 expect(native.dispose).toHaveBeenCalledOnce();
673 }),
674);
675
676it.effect("reports unsupported hosts without wrapping a synthetic cause", () =>
677 Effect.gen(function* () {
678 const result = yield* Effect.gen(function* () {
679 const speech = yield* SpeechService.SpeechService;
680 return yield* Effect.result(speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS));
681 }).pipe(
682 Effect.provide(layer),
683 Effect.provideService(HostProcess.Platform, "win32"),
684 Effect.provideService(HostProcess.Architecture, "arm64"),
685 );
686 expect(Result.isFailure(result) && result.failure).toMatchObject({
687 _tag: "SpeechUnsupportedPlatformError",
688 platform: "win32",
689 architecture: "arm64",
690 });
691 }),
692);
693
694it.effect.each([{ devices: [] }, { devices: [{ id: "gpu", name: "Test GPU" }] }])(
695 "retries failed GPU discovery without repeatedly probing on status polls: $devices",
696 ({ devices }) =>
697 Effect.gen(function* () {
698 const speech = yield* SpeechService.SpeechService;
699 const discover = vi.mocked(listNativeSpeechGpuDevices).mockReset();
700 discover.mockRejectedValueOnce(new Error("driver not ready")).mockResolvedValueOnce(devices);
701 expect(yield* speech.status).toMatchObject({ gpuDevices: [], benchmarkTier: "cpu" });
702 yield* speech.status;
703 expect(discover).toHaveBeenCalledOnce();
704 yield* TestClock.adjust("1 minute");
705 expect(yield* speech.status).toMatchObject({
706 gpuDevices: devices,
707 benchmarkTier: devices.length ? "accelerated" : "cpu",
708 });
709 expect(yield* speech.updateAcceleration("cpu")).toMatchObject({ benchmarkTier: "cpu" });
710 expect(yield* speech.updateAcceleration('gpu:["vulkan","gpu-1"]')).toMatchObject({
711 benchmarkTier: "accelerated",
712 });
713 yield* speech.updateAcceleration("auto");
714 yield* TestClock.adjust("1 minute");
715 expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
716 expect(discover).toHaveBeenCalledTimes(2);
717 }).pipe(Effect.provide(layer)),
718);
719
720it.effect(
721 "retries Auto-placed inference with the dictionary and the CPU's vocabulary capability",
722 () =>
723 Effect.gen(function* () {
724 for (const cpuSupportsVocabulary of [true, false]) {
725 yield* Effect.gen(function* () {
726 native.supportsVocabulary = true;
727 loadNative.mockImplementationOnce(async () => native);
728 loadNative.mockImplementationOnce(async () => ({
729 ...native,
730 backend: "CPU",
731 supportsVocabulary: cpuSupportsVocabulary,
732 }));
733 native.transcribe
734 .mockRejectedValueOnce(new Error("worker stopped"))
735 .mockResolvedValueOnce({ text: "git hub Robert" });
736 const options = {
737 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
738 speechCustomWords: [
739 { term: "GitHub", aliases: [] },
740 { term: "Bob", aliases: ["Robert"] },
741 ],
742 };
743 const speech = yield* SpeechService.SpeechService;
744 expect(yield* speech.status).toMatchObject({ benchmarkTier: "accelerated" });
745 expect(yield* speech.transcribe(pcm(), options)).toBe(
746 cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
747 );
748 expect(yield* speech.status).toMatchObject({ benchmarkTier: "cpu" });
749 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
750 "hello",
751 );
752 expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
753 expect(native.transcribe.mock.calls.slice(-3, -1).map((call) => call[1])).toEqual([
754 { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
755 {
756 timestamps: "none",
757 language: "en",
758 ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
759 },
760 ]);
761 }).pipe(Effect.provide(layer));
762 }
763 }),
764);
765
766it.effect("reports a failure on a GPU the person chose instead of switching to CPU", () =>
767 Effect.gen(function* () {
768 const failure = new Error("GPU failed");
769 native.transcribe.mockRejectedValueOnce(failure);
770 const speech = yield* SpeechService.SpeechService;
771 const acceleration = 'gpu:["vulkan","gpu-1"]';
772 yield* speech.updateAcceleration(acceleration);
773 const result = yield* Effect.result(
774 speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
775 );
776 expect(Result.isFailure(result) && result.failure).toMatchObject({
777 _tag: "SpeechOperationError",
778 operation: "inference",
779 cause: failure,
780 });
781 expect(loadNative).toHaveBeenCalledTimes(1);
782 expect(loadNative.mock.calls[0]?.[3]).toBe(acceleration);
783 }).pipe(Effect.provide(layer)),
784);
785
786it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once", () =>
787 Effect.gen(function* () {
788 for (const cpuSupportsVocabulary of [true, false]) {
789 yield* Effect.gen(function* () {
790 native.supportsVocabulary = true;
791 const feedCalls = native.feed.mock.calls.length;
792 const text = { committed: "", tentative: "git hub Robert" };
793 const corrected = { committed: "", tentative: "git hub Bob" };
794 const options = {
795 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
796 speechCustomWords: [
797 { term: "GitHub", aliases: [] },
798 { term: "Bob", aliases: ["Robert"] },
799 ],
800 };
801 native.feed
802 .mockResolvedValueOnce({ revision: 3, text })
803 .mockRejectedValueOnce(new Error("worker stopped"))
804 .mockResolvedValueOnce({ revision: 1, text })
805 .mockResolvedValueOnce({ revision: 2, text })
806 .mockResolvedValueOnce({ revision: 3, text });
807 loadNative.mockImplementationOnce(async () => native);
808 loadNative.mockImplementationOnce(async () => ({
809 ...native,
810 backend: "CPU",
811 supportsVocabulary: cpuSupportsVocabulary,
812 }));
813 const speech = yield* SpeechService.SpeechService;
814 const stream = yield* speech.startStream(options);
815 expect(yield* stream.feed(pcm())).toEqual({ revision: 3, text: corrected });
816 const recovered = {
817 committed: "",
818 tentative: cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
819 };
820 expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered });
821 expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
822 expect(native.begin.mock.calls.slice(-2).map(([options]) => options)).toEqual([
823 { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
824 {
825 timestamps: "none",
826 language: "en",
827 ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
828 },
829 ]);
830 expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4);
831 expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered });
832 const loads = loadNative.mock.calls.length;
833 native.finish.mockRejectedValueOnce(new Error("CPU finish failed"));
834 expect(Result.isFailure(yield* Effect.result(stream.finish))).toBe(true);
835 expect(Result.isFailure(yield* Effect.result(stream.feed(pcm())))).toBe(true);
836 expect(loadNative).toHaveBeenCalledTimes(loads);
837 }).pipe(Effect.scoped, Effect.provide(layer));
838 }
839 }),
840);
841
842it.effect.each([false, true])(
843 "recovers Auto stream finalization on a fresh CPU worker with retained audio: %s",
844 (withAudio) =>
845 Effect.gen(function* () {
846 const cpu = {
847 ...native,
848 backend: "CPU",
849 begin: vi.fn(async (_options?: unknown) => {}),
850 feed: vi.fn(async (_pcm: Float32Array) => ({
851 revision: 1,
852 text: { committed: "", tentative: "" },
853 })),
854 finish: vi.fn(async () => "Robert"),
855 dispose: vi.fn(),
856 };
857 const gpu = {
858 ...native,
859 finish: vi.fn(async () => {
860 throw new Error("GPU finish failed");
861 }),
862 };
863 loadNative.mockResolvedValueOnce(gpu).mockResolvedValueOnce(cpu);
864 const speech = yield* SpeechService.SpeechService;
865 const chunks = withAudio ? [new Float32Array([0.25, 0]), new Float32Array([-0.5])] : [];
866 yield* Effect.gen(function* () {
867 const stream = yield* speech.startStream({
868 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
869 speechLanguage: "fr",
870 speechCustomWords: [{ term: "Bob", aliases: ["Robert"] }],
871 });
872 for (const chunk of chunks) yield* stream.feed(new Uint8Array(chunk.buffer));
873 expect(yield* stream.finish).toBe("Bob");
874 expect(cpu.feed.mock.calls.map(([chunk]) => [...chunk])).toEqual(
875 chunks.map((chunk) => [...chunk]),
876 );
877 expect(cpu.begin).toHaveBeenCalledWith(
878 { timestamps: "none", language: "fr" },
879 true,
880 "test.gguf",
881 );
882 expect(gpu.dispose).toHaveBeenCalledOnce();
883 expect(cpu.dispose).not.toHaveBeenCalled();
884 const busy = yield* Effect.result(speech.selectModel("fallback-model"));
885 expect(Result.isFailure(busy) && busy.failure).toMatchObject({ _tag: "SpeechBusyError" });
886 }).pipe(Effect.scoped);
887 expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual(
888 [
889 ["test.gguf", "auto"],
890 ["test.gguf", "cpu"],
891 ],
892 );
893 yield* speech.selectModel("fallback-model");
894 expect(cpu.dispose).toHaveBeenCalledOnce();
895 }).pipe(Effect.provide(layer)),
896);
897
898it.effect.each(["auto", 'gpu:["vulkan","gpu-1"]', "cpu"] as const)(
899 "reports finalization failure without repeated or explicit-backend fallback: %s",
900 (acceleration) =>
901 Effect.gen(function* () {
902 const failure = new Error("finish failed");
903 const worker = {
904 ...native,
905 backend: acceleration === "cpu" ? "CPU" : "Vulkan0",
906 finish: vi.fn(async () => {
907 throw failure;
908 }),
909 };
910 const cpu = { ...worker, backend: "CPU", dispose: vi.fn() };
911 loadNative.mockResolvedValueOnce(worker);
912 if (acceleration === "auto") loadNative.mockResolvedValueOnce(cpu);
913 const speech = yield* SpeechService.SpeechService;
914 yield* speech.updateAcceleration(acceleration);
915 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
916 expect(Result.isFailure(yield* Effect.result(stream.finish))).toBe(true);
917 expect(worker.finish).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1);
918 expect(loadNative).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1);
919 }).pipe(Effect.scoped, Effect.provide(layer)),
920);
921
922it.effect("does not restart a stream cancelled while native finalization is pending", () =>
923 Effect.gen(function* () {
924 const pending = Promise.withResolvers<string>();
925 const started = Promise.withResolvers<void>();
926 native.finish.mockImplementationOnce(() => {
927 started.resolve();
928 return pending.promise;
929 });
930 const speech = yield* SpeechService.SpeechService;
931 const streamScope = yield* Scope.make();
932 const stream = yield* speech
933 .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
934 .pipe(Effect.provideService(Scope.Scope, streamScope));
935 const request = yield* Effect.result(stream.finish).pipe(Effect.forkChild);
936 yield* Effect.promise(() => started.promise);
937 yield* Scope.close(streamScope, Exit.void);
938 pending.reject(new Error("cancelled worker stopped"));
939 expect(Result.isFailure(yield* Fiber.join(request))).toBe(true);
940 expect(loadNative).toHaveBeenCalledOnce();
941 expect(native.reset).toHaveBeenCalledOnce();
942 }).pipe(Effect.provide(layer)),
943);