1#!/usr/bin/env node
2// An eval script, not server code: it shells out to git and reads files directly.
3// @effect-diagnostics nodeBuiltinImport:off
4import * as NodeChildProcess from "node:child_process";
5import * as NodeCrypto from "node:crypto";
6import * as NodeFS from "node:fs";
7import * as NodeOS from "node:os";
8import * as NodePath from "node:path";
9import * as NodeUtil from "node:util";
10import * as Effect from "effect/Effect";
11import * as Schedule from "effect/Schedule";
12import * as Schema from "effect/Schema";
13import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
14import { proxyConfig } from "../shared/proxy.ts";
15import { cases, type Case } from "./cases.ts";
16import { grade, testRange, type Comment } from "./grade.ts";
17
18const { values } = NodeUtil.parseArgs({
19 options: {
20 out: { type: "string" },
21 model: { type: "string", multiple: true },
22 prompt: { type: "string" },
23 case: { type: "string", multiple: true },
24 samples: { type: "string", default: "2" },
25 concurrency: { type: "string", default: "6" },
26 reasoning: { type: "string", default: "low" },
27 },
28});
29if (!values.out) throw new Error("Use --out <directory> [--model <id>] [--prompt <file>].");
30const outDirectory = values.out;
31const models = values.model ?? ["gpt-6.1-terra", "gpt-6-luna"];
32const repo = NodePath.resolve(import.meta.dirname, "../..");
33const checkFile =
34 values.prompt ?? NodePath.join(repo, ".macroscope/check-run-agents/test-signal.md");
35// Macroscope gives the model the body; the front matter only configures the run.
36const instructions = NodeFS.readFileSync(checkFile, "utf8")
37 .replace(/^---\n[\s\S]*?\n---\n/, "")
38 .trim();
39const words = instructions.split(/\s+/).length;
40const promptHash = NodeCrypto.createHash("sha256").update(instructions).digest("hex").slice(0, 12);
41const selected = values.case ? cases.filter((entry) => values.case!.includes(entry.id)) : cases;
42const proxy = proxyConfig();
43
44const git = (args: ReadonlyArray<string>) => {
45 const result = NodeChildProcess.spawnSync("git", args, {
46 cwd: repo,
47 encoding: "utf8",
48 maxBuffer: 1 << 26,
49 });
50 return result.status === 0 ? result.stdout : undefined;
51};
52
53/** The case's PR head as files the model can read, plus the diff it reviews. */
54const world = (entry: Case) => {
55 const files = new Map<string, string>();
56 if (entry.type === "real") {
57 const diff = git(["diff", `${entry.rev}~1`, entry.rev, "--", ...entry.paths]);
58 // An empty diff would grade a guard case as a clean pass.
59 if (!diff) throw new Error(`${entry.id}: no diff at ${entry.rev}.`);
60 return { diff, files };
61 }
62 const before = git(["show", `${entry.rev}:${entry.path}`]);
63 if (!before?.includes(entry.after)) throw new Error(`${entry.id}: anchor not found.`);
64 const after = before.replace(entry.after, entry.after + entry.insert);
65 files.set(entry.path, after);
66 const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-test-signal-"));
67 NodeFS.writeFileSync(NodePath.join(directory, "before"), before);
68 NodeFS.writeFileSync(NodePath.join(directory, "after"), after);
69 const raw = NodeChildProcess.spawnSync("git", ["diff", "--no-index", "before", "after"], {
70 cwd: directory,
71 encoding: "utf8",
72 }).stdout;
73 NodeFS.rmSync(directory, { recursive: true });
74 const diff = raw
75 .replace("diff --git a/before b/after", `diff --git a/${entry.path} b/${entry.path}`)
76 .replace("--- a/before", `--- a/${entry.path}`)
77 .replace("+++ b/after", `+++ b/${entry.path}`);
78 return { diff, files };
79};
80
81// The check reads the rules at the PR head, where they always exist. Old case commits predate
82// them, so serve the rules from this checkout.
83const ruleFiles = new Set(["docs/internals/testing.md", "AGENTS.md"]);
84const readAt = (entry: Case, files: Map<string, string>, path: string) =>
85 ruleFiles.has(path)
86 ? NodeFS.readFileSync(NodePath.join(repo, path), "utf8")
87 : (files.get(path) ?? git(["show", `${entry.rev}:${path}`]));
88
89/** `git grep` at `rev`, with overlay and rule files searched in their served form instead. */
90const grepAt = (entry: Case, files: Map<string, string>, pattern: string, scope: string) => {
91 // Status 2 is a bad pattern. Show grep's message, as a real tool would, so the model can fix it.
92 const check = NodeChildProcess.spawnSync("grep", ["-E", "--", pattern], {
93 input: "",
94 encoding: "utf8",
95 });
96 if (check.status !== 0 && check.status !== 1) return check.stderr.trim() || "grep failed.";
97 const served = new Map(files);
98 for (const path of ruleFiles) served.set(path, readAt(entry, files, path) ?? "");
99 // Served files also exist in this checkout, so git applies its own pathspec rules to the scope.
100 const matched = new Set((git(["ls-files", "--", scope || "."]) ?? "").split("\n"));
101 const inScope = (path: string) => matched.has(path);
102 const fromRev = (git(["grep", "-n", "-E", "-e", pattern, entry.rev, "--", scope || "."]) ?? "")
103 .split("\n")
104 .filter((line) => line !== "")
105 .map((line) => line.replace(`${entry.rev}:`, ""))
106 .filter((line) => !served.has(line.slice(0, line.indexOf(":"))));
107 const fromServed = [...served]
108 .filter(([path]) => inScope(path))
109 .flatMap(([path, text]) =>
110 NodeChildProcess.spawnSync("grep", ["-n", "-E", "--", pattern], {
111 input: text,
112 encoding: "utf8",
113 })
114 .stdout.split("\n")
115 .filter((line) => line !== "")
116 .map((line) => `${path}:${line}`),
117 );
118 return [...fromServed, ...fromRev].slice(0, 80).join("\n") || "No matches.";
119};
120
121const tools = [
122 {
123 type: "function",
124 name: "read_file",
125 description: "Read a file at the pull request head, with line numbers.",
126 parameters: {
127 type: "object",
128 properties: {
129 path: { type: "string" },
130 start_line: { type: "integer" },
131 end_line: { type: "integer" },
132 },
133 required: ["path"],
134 },
135 },
136 {
137 type: "function",
138 name: "grep",
139 description:
140 "Search the repository at the pull request head with an extended regular expression.",
141 parameters: {
142 type: "object",
143 properties: { pattern: { type: "string" }, path: { type: "string" } },
144 required: ["pattern"],
145 },
146 },
147 {
148 type: "function",
149 name: "post_comment",
150 description: "Post an inline review comment on a line of the pull request head.",
151 parameters: {
152 type: "object",
153 properties: { path: { type: "string" }, line: { type: "integer" }, body: { type: "string" } },
154 required: ["path", "line", "body"],
155 },
156 },
157];
158
159const system = (body: string) =>
160 [
161 "You are a Macroscope Check Run Agent reviewing a GitHub pull request.",
162 "Follow the check instructions below. Read the repository at the pull request head with read_file and grep.",
163 "Leave inline review comments with post_comment. Then give your final response.",
164 "",
165 "<check_instructions>",
166 body,
167 "</check_instructions>",
168 ].join("\n");
169
170const decodeResponse = Schema.decodeUnknownSync(
171 Schema.fromJsonString(
172 Schema.Struct({
173 output: Schema.Array(Schema.Record(Schema.String, Schema.Unknown)),
174 usage: Schema.optional(Schema.Struct({ total_tokens: Schema.optional(Schema.Number) })),
175 }),
176 ),
177);
178const decodeArguments = Schema.decodeUnknownSync(
179 Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
180);
181const encodeRow = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
182
183const post = Effect.fn("evaluateTestSignal.post")(function* (body: unknown) {
184 const client = (yield* HttpClient.HttpClient).pipe(
185 HttpClient.retryTransient({
186 retryOn: "errors-and-responses",
187 schedule: Schedule.exponential("2 seconds"),
188 times: 4,
189 }),
190 );
191 const response = yield* HttpClientRequest.post(`${proxy.url}/responses`).pipe(
192 HttpClientRequest.setHeaders(proxy.headers),
193 HttpClientRequest.bodyJson(body),
194 Effect.flatMap(client.execute),
195 );
196 const text = yield* response.text;
197 if (response.status !== 200)
198 return yield* Effect.die(new Error(`Model API ${response.status}: ${text}`));
199 return decodeResponse(text);
200});
201
202const runSample = Effect.fn("evaluateTestSignal.sample")(function* (entry: Case, model: string) {
203 const { diff, files } = world(entry);
204 const comments: Array<Comment> = [];
205 const input: Array<unknown> = [
206 {
207 role: "user",
208 content: `Pull request diff for the files this check includes:\n\n\`\`\`diff\n${diff}\`\`\``,
209 },
210 ];
211 let final = "";
212 let tokens = 0;
213 for (let step = 0; step < 16; step++) {
214 const response = yield* post({
215 model,
216 instructions: system(instructions),
217 input,
218 tools,
219 reasoning: { effort: values.reasoning },
220 store: false,
221 include: ["reasoning.encrypted_content"],
222 });
223 tokens += response.usage?.total_tokens ?? 0;
224 input.push(...response.output);
225 const calls = response.output.filter((item) => item.type === "function_call");
226 final = response.output
227 .filter((item) => item.type === "message")
228 .flatMap((item) => (item.content as Array<{ text?: string }>).map((part) => part.text ?? ""))
229 .join("");
230 if (calls.length === 0) break;
231 for (const call of calls) {
232 const args = decodeArguments(String(call.arguments));
233 let output: string;
234 if (call.name === "post_comment") {
235 comments.push({
236 path: String(args.path),
237 line: Number(args.line),
238 body: String(args.body),
239 });
240 output = "Posted.";
241 } else if (call.name === "read_file") {
242 const text = readAt(entry, files, String(args.path));
243 const lines = text?.split("\n") ?? [];
244 const start = Math.max(1, Number(args.start_line ?? 1));
245 const end = Math.min(lines.length, Number(args.end_line ?? start + 399));
246 output =
247 text === undefined
248 ? "File not found."
249 : lines
250 .slice(start - 1, end)
251 .map((line, index) => `${start + index}: ${line}`)
252 .join("\n");
253 } else {
254 output = grepAt(entry, files, String(args.pattern), String(args.path ?? ""));
255 }
256 input.push({ type: "function_call_output", call_id: call.call_id, output });
257 }
258 }
259 const targets = entry.targets.flatMap((target) => {
260 const range = testRange(readAt(entry, files, target.path) ?? "", target.title);
261 if (!range) throw new Error(`${entry.id}: target "${target.title}" not found.`);
262 return [{ path: target.path, ...range }];
263 });
264 return { ...grade({ kind: entry.kind, targets, comments, final }), comments, final, tokens };
265});
266
267const sampleSafely = (entry: Case, model: string, sample: number) =>
268 runSample(entry, model).pipe(
269 Effect.map((result) => ({ case: entry.id, kind: entry.kind, model, sample, ...result })),
270 Effect.catchCause((cause) =>
271 Effect.succeed({ case: entry.id, kind: entry.kind, model, sample, error: String(cause) }),
272 ),
273 );
274
275type Row = Effect.Success<ReturnType<typeof sampleSafely>>;
276const isGraded = (row: Row) => "pass" in row;
277const passed = (row: Row) => "pass" in row && row.pass;
278
279const summary = await Effect.runPromise(
280 Effect.gen(function* () {
281 NodeFS.mkdirSync(outDirectory, { recursive: true });
282 const samples = Number(values.samples);
283 const pairs = models.flatMap((model) => selected.map((entry) => ({ model, entry })));
284 const rows: Array<Row> = (yield* Effect.forEach(
285 pairs.flatMap(({ model, entry }) =>
286 Array.from({ length: samples }, (_, n) => ({ model, entry, n })),
287 ),
288 ({ model, entry, n }) => sampleSafely(entry, model, n),
289 { concurrency: Number(values.concurrency) },
290 )).slice();
291 // A third sample settles a split.
292 const splits = pairs.filter(({ model, entry }) => {
293 const own = rows.filter((row) => row.model === model && row.case === entry.id);
294 return own.length === 2 && own.some(passed) && !own.every(passed);
295 });
296 rows.push(
297 ...(yield* Effect.forEach(splits, ({ model, entry }) => sampleSafely(entry, model, samples), {
298 concurrency: Number(values.concurrency),
299 })),
300 );
301 NodeFS.writeFileSync(
302 NodePath.join(outDirectory, "results.ndjson"),
303 rows.map((row) => encodeRow(row)).join("\n") + "\n",
304 );
305
306 const lines = [`prompt ${promptHash}, ${words} words, ${checkFile}`, ""];
307 lines.push("| model | bad caught | guards clean | errors | tokens |", "|---|---|---|---|---|");
308 for (const model of models) {
309 const verdicts = selected.map((entry) => {
310 const own = rows.filter((row) => row.model === model && row.case === entry.id);
311 // A run error counts as a failed sample, so one pass and one error is not a majority.
312 const graded = own.filter(isGraded);
313 const passes = own.filter(passed).length;
314 return {
315 entry,
316 pass: passes > own.length / 2,
317 errors: own.length - graded.length,
318 };
319 });
320 const count = (kind: Case["kind"]) => {
321 const own = verdicts.filter((verdict) => verdict.entry.kind === kind);
322 return `${own.filter((verdict) => verdict.pass).length}/${own.length}`;
323 };
324 const tokens = rows
325 .filter((row) => row.model === model && "tokens" in row)
326 .reduce((sum, row) => sum + ("tokens" in row ? row.tokens : 0), 0);
327 const errors = verdicts.reduce((sum, verdict) => sum + verdict.errors, 0);
328 lines.push(`| ${model} | ${count("bad")} | ${count("guard")} | ${errors} | ${tokens} |`);
329 const failed = verdicts.filter((verdict) => !verdict.pass).map((verdict) => verdict.entry.id);
330 if (failed.length > 0) lines.push(`| failed: ${failed.join(", ")} | | | | |`);
331 }
332 const report = lines.join("\n") + "\n";
333 NodeFS.writeFileSync(NodePath.join(outDirectory, "summary.md"), report);
334 return report;
335 }).pipe(Effect.provide(FetchHttpClient.layer)),
336);
337process.stdout.write(summary);