1#!/usr/bin/env node
2// @effect-diagnostics nodeBuiltinImport:off
3import * as NodeUtil from "node:util";
4import * as NodeCrypto from "node:crypto";
5import * as FetchHttpClient from "effect/http/FetchHttpClient";
6import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
7import * as NodeServices from "@effect/platform-node/NodeServices";
8import { CodexSettings, ProviderInstanceId } from "@t3tools/contracts";
9import * as Effect from "effect/Effect";
10import * as Layer from "effect/Layer";
11import * as Duration from "effect/Duration";
12import * as FileSystem from "effect/FileSystem";
13import * as Path from "effect/Path";
14import * as Schema from "effect/Schema";
15import * as CodexTextGeneration from "../../apps/server/src/textGeneration/CodexTextGeneration.ts";
16import { threadTitleEvaluationCases } from "./cases.ts";
17import {
18 formatThreadTitleContext,
19 type ThreadTitleMessage,
20} from "@t3tools/provider-core/server/threadTitleContext";
21import * as ThreadTitleLinks from "../../apps/server/src/textGeneration/ThreadTitleLinks.ts";
22import * as SourceControlProviderRegistry from "../../apps/server/src/sourceControl/SourceControlProviderRegistry.ts";
23import * as SourceControlBuiltInDrivers from "../../apps/server/src/sourceControl/builtInDrivers.ts";
24import * as ServerSettings from "../../apps/server/src/serverSettings.ts";
25import * as VcsProcess from "../../apps/server/src/vcs/VcsProcess.ts";
26import * as VcsDriverRegistry from "../../apps/server/src/vcs/VcsDriverRegistry.ts";
27import * as VcsProjectConfig from "../../apps/server/src/vcs/VcsProjectConfig.ts";
28import * as GitVcsDriver from "../../apps/server/src/vcs/GitVcsDriver.ts";
29import * as ProcessRunner from "../../apps/server/src/processRunner.ts";
30import * as ServerConfig from "../../apps/server/src/config.ts";
31
32const { values } = NodeUtil.parseArgs({
33 options: {
34 model: { type: "string" },
35 case: { type: "string", multiple: true },
36 out: { type: "string" },
37 baseline: { type: "string" },
38 initial: { type: "boolean", default: false },
39 },
40});
41if (!values.model || !values.out)
42 throw new Error("Use --model <configured-model> --out <directory>.");
43for (const id of values.case ?? []) {
44 if (!threadTitleEvaluationCases.some((fixture) => fixture.id === id)) {
45 throw new Error(`Unknown case id: ${id}.`);
46 }
47}
48const model = values.model;
49const outputDirectory = values.out;
50const Results = Schema.fromJsonString(
51 Schema.Array(
52 Schema.Struct({
53 id: Schema.String,
54 title: Schema.String,
55 latencyMs: Schema.Number,
56 linkedContextDigest: Schema.String,
57 }),
58 ),
59);
60const decodeResults = Schema.decodeUnknownEffect(Results);
61const decodeSettings = Schema.decodeUnknownEffect(CodexSettings);
62const encodeReport = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
63
64await Effect.runPromise(
65 Effect.gen(function* () {
66 const fs = yield* FileSystem.FileSystem;
67 const path = yield* Path.Path;
68 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-title-evaluation-" });
69 const generation = yield* CodexTextGeneration.makeCodexTextGeneration(
70 yield* decodeSettings({}),
71 );
72 const baseline = values.baseline
73 ? yield* fs.readFileString(values.baseline).pipe(Effect.flatMap(decodeResults))
74 : [];
75 const results = [];
76 const review = [];
77 const answerKey = [];
78 for (const fixture of threadTitleEvaluationCases) {
79 if (values.case?.length && !values.case.includes(fixture.id)) continue;
80 const previous = baseline.find((entry) => entry.id === fixture.id);
81 if (values.baseline && !previous) throw new Error(`Baseline is missing ${fixture.id}.`);
82 const firstMessage: ThreadTitleMessage | undefined = fixture.messages.find(
83 (message) => message.role === "user",
84 );
85 if (!firstMessage) throw new Error(`Fixture ${fixture.id} has no user message.`);
86 const context = formatThreadTitleContext(fixture.messages);
87 const message = values.initial ? firstMessage.text : context.message;
88 const attachments = values.initial ? firstMessage.attachments : context.attachments;
89 const [elapsed, { generated, linkedContextDigest }] = yield* Effect.gen(function* () {
90 const linkedContext = yield* ThreadTitleLinks.resolveThreadTitleLinks({
91 cwd,
92 message,
93 });
94 const linkedContextDigest = NodeCrypto.createHash("sha256")
95 .update(linkedContext ?? "")
96 .digest("hex");
97 if (previous && previous.linkedContextDigest !== linkedContextDigest) {
98 throw new Error(
99 `Linked context changed for ${fixture.id}. Record a new baseline before comparing titles.`,
100 );
101 }
102 const generated = yield* generation.generateThreadTitle({
103 cwd,
104 message,
105 ...("projectTitle" in fixture ? { projectTitle: fixture.projectTitle } : {}),
106 previousTitle: values.initial ? undefined : fixture.previousTitle,
107 attachments,
108 linkedContext,
109 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model },
110 });
111 return { generated, linkedContextDigest };
112 }).pipe(Effect.timed);
113 const oldTitle = previous?.title ?? fixture.previousTitle;
114 const newFirst = NodeCrypto.randomInt(2) === 0;
115 results.push({
116 id: fixture.id,
117 title: generated.title,
118 latencyMs: Duration.toMillis(elapsed),
119 needsRefinement: generated.needsRefinement ?? false,
120 linkedContextDigest,
121 });
122 review.push({
123 id: fixture.id,
124 source: fixture.source,
125 request: fixture.request,
126 rubric: fixture.rubric,
127 A: newFirst ? generated.title : oldTitle,
128 B: newFirst ? oldTitle : generated.title,
129 preferred: "",
130 subjectAccuracy: "",
131 recognitionAmongNearbyThreads: "",
132 });
133 answerKey.push({ id: fixture.id, candidate: newFirst ? "A" : "B" });
134 }
135 yield* fs.makeDirectory(outputDirectory, { recursive: true });
136 for (const [name, report] of [
137 ["results", results],
138 ["review", review],
139 ["answer-key", answerKey],
140 ] as const) {
141 yield* fs.writeFileString(
142 path.join(outputDirectory, `${name}.json`),
143 yield* encodeReport(report),
144 );
145 }
146 yield* Effect.log(
147 `Wrote ${results.length} cases to ${outputDirectory}. Score review.json before opening answer-key.json. Latency is in results.json.`,
148 );
149 }).pipe(
150 Effect.provide(
151 Layer.mergeAll(
152 ProcessRunner.layer,
153 SourceControlProviderRegistry.layer.pipe(
154 Layer.provide(SourceControlBuiltInDrivers.layer),
155 Layer.provide(ServerSettings.layerTest()),
156 Layer.provide(VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer))),
157 Layer.provide(GitVcsDriver.layer),
158 Layer.provide(VcsProcess.layer),
159 Layer.provide(FetchHttpClient.layer),
160 ),
161 ).pipe(
162 Layer.provideMerge(
163 ServerConfig.layerTest(process.cwd(), { prefix: "t3-title-evaluation-state-" }),
164 ),
165 Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodePlatformCrypto.layer)),
166 ),
167 ),
168 Effect.scoped,
169 ),
170);