1import * as NodeUtil from "node:util";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import {
4 CodexSettings,
5 DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
6 ProviderInstanceId,
7} from "@t3tools/contracts";
8import * as Effect from "effect/Effect";
9import * as FileSystem from "effect/FileSystem";
10import * as Layer from "effect/Layer";
11import * as Path from "effect/Path";
12import * as Random from "effect/Random";
13import * as Schema from "effect/Schema";
14import * as ServerConfig from "../../apps/server/src/config.ts";
15import { makeCodexTextGeneration } from "../../apps/server/src/textGeneration/CodexTextGeneration.ts";
16import { buildTranscriptionPostProcessingPrompt } from "../../apps/server/src/textGeneration/TranscriptionPostProcessing.ts";
17import { cases } from "./cases.ts";
18
19const { values } = NodeUtil.parseArgs({
20 options: {
21 out: { type: "string" },
22 model: { type: "string" },
23 baseline: { type: "string" },
24 case: { type: "string" },
25 samples: { type: "string", default: "3" },
26 },
27});
28const { out, model } = values;
29const samples = Number(values.samples);
30const fixtures = cases.filter((fixture) => !values.case || fixture.id === values.case);
31if (!out || !model || !fixtures.length || !Number.isSafeInteger(samples) || samples < 1) {
32 throw new Error(
33 "Use --out <directory> --model <configured model> [--case <id>] [--samples <count>].",
34 );
35}
36const Results = Schema.fromJsonString(
37 Schema.Array(
38 Schema.Struct({
39 id: Schema.String,
40 sample: Schema.Number,
41 text: Schema.String,
42 }),
43 ),
44);
45const decodeSettings = Schema.decodeEffect(CodexSettings);
46const decodeResults = Schema.decodeEffect(Results);
47const encodeRows = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
48await Effect.runPromise(
49 Effect.gen(function* () {
50 const fs = yield* FileSystem.FileSystem;
51 const path = yield* Path.Path;
52 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-voice-eval-" });
53 const service = yield* makeCodexTextGeneration(yield* decodeSettings({}));
54 const baseline = values.baseline
55 ? yield* fs.readFileString(values.baseline).pipe(Effect.flatMap(decodeResults))
56 : [];
57 const results = [];
58 const review = [];
59 const key = [];
60 for (const fixture of fixtures) {
61 const start = fixture.before.length;
62 const prompt = buildTranscriptionPostProcessingPrompt(
63 DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
64 fixture.transcript,
65 {
66 text: fixture.before + fixture.selection + fixture.after,
67 selection: { start, end: start + fixture.selection.length },
68 },
69 fixture.cue,
70 );
71 for (let sample = 1; sample <= samples; sample++) {
72 const previous = baseline.find((row) => row.id === fixture.id && row.sample === sample);
73 if (values.baseline && !previous)
74 throw new Error(`Missing baseline for ${fixture.id} sample ${sample}.`);
75 const { transcription } = yield* service.generateTranscriptionPostProcessing({
76 cwd,
77 prompt,
78 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model },
79 });
80 results.push({ id: fixture.id, sample, text: transcription });
81 const first = !previous || (yield* Random.nextBoolean);
82 review.push({
83 ...fixture,
84 sample,
85 A: first ? transcription : previous?.text,
86 B: first ? previous?.text : transcription,
87 });
88 key.push({ id: fixture.id, sample, candidate: first ? "A" : "B" });
89 }
90 }
91 yield* fs.makeDirectory(out, { recursive: true });
92 for (const [name, rows] of [
93 ["results", results],
94 ["review", review],
95 ["answer-key", key],
96 ] as const) {
97 yield* fs.writeFileString(path.join(out, name + ".json"), yield* encodeRows(rows));
98 }
99 }).pipe(
100 Effect.provide(
101 ServerConfig.layerTest(process.cwd(), { prefix: "t3-voice-eval-state-" }).pipe(
102 Layer.provideMerge(NodeServices.layer),
103 ),
104 ),
105 Effect.scoped,
106 ),
107);