Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off - compares the generated bundle directly with its checked-in bytes.
2import * as NodeFS from "node:fs";
3
4import { catalog } from "@handy-computer/models";
5import { expect, it } from "vite-plus/test";
6
7import manifest from "../package.json" with { type: "json" };
8import { generateSpeechCatalog, renderSpeechCatalog } from "./speech-catalog.ts";
9
10const generate = (models = catalog.models) =>
11 generateSpeechCatalog(
12 { ...catalog.data, models },
13 manifest.devDependencies["@handy-computer/models"],
14 manifest.dependencies["transcribe-cpp"],
15 );
16
17it("reproduces the bundled registry byte for byte without upstream requests", async () => {
18 expect(NodeFS.readFileSync(new URL("../src/speech/catalog.json", import.meta.url), "utf8")).toBe(
19 await renderSpeechCatalog(
20 catalog.data,
21 manifest.devDependencies["@handy-computer/models"],
22 manifest.dependencies["transcribe-cpp"],
23 ),
24 );
25 for (const model of generate().models) {
26 const source = catalog.models.find((entry) => entry.repository === model.id)!;
27 expect(model).toMatchObject({
28 revision: source.revision,
29 name: source.name,
30 ...source.files[source.quant],
31 supportsStreaming: source.capabilities.streaming,
32 supportsLanguageDetection: source.capabilities.languageDetection,
33 supportsTranslation: source.capabilities.translate,
34 });
35 }
36 const parakeet = catalog.models[0]!;
37 const withoutFrenchOrSpeed = generateSpeechCatalog(
38 {
39 ...catalog.data,
40 models: [{ ...parakeet, languages: ["en", "fr"] }],
41 results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance: {} } },
42 },
43 "0.2.3",
44 "0.4.1",
45 );
46 expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
47 accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
48 secondsPer30s: { cpu: null, accelerated: null },
49 });
50 expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
51 for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
52 const result = generateSpeechCatalog(
53 {
54 ...catalog.data,
55 models: [parakeet],
56 results: { [parakeet.id]: { accuracy: {}, performance } },
57 },
58 "0.2.3",
59 "0.4.1",
60 );
61 expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62 cpu: performance.cpu ? 15 : null,
63 accelerated: performance.accelerated ? 3 : null,
64 });
65 }
66});
67
68it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
69 const model = catalog.models[0]!;
70 for (const override of [
71 { role: "langid" as const },
72 { role: "vad" as const },
73 { role: "diarize" as const },
74 { license: "cc-by-nc-4.0" },
75 { license: "CC-BY-NC-SA-4.0" },
76 { license: "cc-by-nc-nd-3.0" },
77 { license: "cc-by-nc" },
78 { license: "CC BY-NC 4.0" },
79 { license: "custom_NC" },
80 { minEngine: "0.4.2" },
81 { minEngine: "0.5.0" },
82 { family: "future_arch" },
83 ...[
84 "nested/model.gguf",
85 "nested\\model.gguf",
86 "../model.gguf",
87 "model.bin",
88 "model.safetensors",
89 ].map((filename) => ({ files: { [model.quant]: { ...model.files[model.quant]!, filename } } })),
90 ]) {
91 expect(generate([{ ...model, ...override }]).models).toEqual([]);
92 }
93 for (const license of ["other", "apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "unlicense"]) {
94 expect(generate([{ ...model, license, minEngine: "0.4.1" }]).models).toHaveLength(1);
95 }
96 const moonshine = catalog.models.find((entry) => entry.id === "moonshine-tiny-ja")!;
97 for (const [id, allowed] of [
98 ["moonshine-tiny-ja", false],
99 ["moonshine-base-vi", false],
100 ["moonshine-base-es", false],
101 ["moonshine-base", true],
102 ["moonshine-streaming-tiny-ja", true],
103 ] as const) {
104 const entry = { ...moonshine, id, repository: `handy-computer/${id}-gguf`, license: "mit" };
105 expect(generate([entry]).models, id).toHaveLength(allowed ? 1 : 0);
106 }
107});
108
109it("rejects missing defaults, unpinned or corrupt metadata, collisions, and unreviewed engine/schema changes", () => {
110 const model = catalog.models[0]!;
111 const file = model.files[model.quant]!;
112 for (const override of [
113 { files: {} },
114 { revision: "main" },
115 { repository: "../model" },
116 { languages: [] },
117 ...[{ sha256: "bad" }, { size: 0 }, { size: 1.5 }].map((entry) => ({
118 files: { [model.quant]: { ...file, ...entry } },
119 })),
120 ]) {
121 expect(() => generate([{ ...model, ...override }])).toThrow();
122 }
123 expect(() => generate([model, model])).toThrow(/Duplicate/);
124 expect(() => generate([model, { ...model, repository: "handy-computer/another" }])).toThrow(
125 /Duplicate/,
126 );
127 expect(() => generateSpeechCatalog(catalog.data, "0.2.3", "0.4.2")).toThrow(/loader support/);
128 expect(() =>
129 generateSpeechCatalog({ ...catalog.data, schema: 3 as 2 }, "0.2.3", "0.4.1"),
130 ).toThrow(/schema/);
131});