1const assert = require("node:assert/strict");
2const test = require("node:test");
3
4const {
5 renderComment,
6 resolve,
7 upsertCommentForCurrentHead,
8 validateResult,
9 validateScenarios,
10} = require("./thread-transfer-report.cjs");
11
12function result(overrides = {}) {
13 const observed = {
14 totalWireBytes: 2_200_000,
15 threadSnapshotWireBytes: 1_950_000,
16 threadSnapshotDecodedBytes: 9_100_000,
17 measuredTurnWebSocketWireBytes: 250_000,
18 measuredTurnWebSocketDecodedBytes: 1_150_000,
19 measuredTurnWebSocketMessages: 15,
20 };
21 const ceiling = {
22 totalWireBytes: 2_900_000,
23 threadSnapshotWireBytes: 2_600_000,
24 measuredTurnWebSocketWireBytes: 320_000,
25 measuredTurnWebSocketDecodedBytes: 1_550_000,
26 measuredTurnWebSocketMessages: 20,
27 };
28 return {
29 schemaVersion: 1,
30 scenario: {
31 id: "thread-transfer-v1",
32 historyTurns: 10,
33 historyCommandToolsPerTurn: 5,
34 historyMcpResultBytes: 900_000,
35 measuredCommandTools: 20,
36 measuredMcpResultBytes: 1_100_000,
37 },
38 providers: {
39 codex: { observed: { ...observed, ...overrides }, ceiling },
40 claudeAgent: { observed, ceiling },
41 },
42 };
43}
44
45function scenarios(overrides = {}) {
46 const rows = [
47 {
48 phase: "long turn finished, cold open",
49 metric: "HTTP decoded",
50 unit: "bytes",
51 observed: 390_000,
52 ceiling: 420_000,
53 },
54 {
55 phase: "idle thread resume",
56 metric: "WebSocket messages",
57 unit: "count",
58 observed: 1,
59 ceiling: 1,
60 },
61 ];
62 return {
63 schemaVersion: 1,
64 providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows },
65 };
66}
67
68test("validates the fixed artifact schema", () => {
69 assert.equal(validateResult(result()).schemaVersion, 1);
70 assert.throws(
71 () => validateResult({ ...result(), injectedMarkdown: "@everyone" }),
72 /unexpected fields/,
73 );
74 assert.throws(
75 () => validateResult(result({ totalWireBytes: "lots" })),
76 /non-negative safe integer/,
77 );
78
79 assert.equal(validateScenarios(scenarios()).schemaVersion, 1);
80 for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "<img>", ""]) {
81 assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/);
82 }
83 assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/);
84 assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/);
85 assert.throws(
86 () =>
87 validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })),
88 /repeats an earlier row/,
89 );
90});
91
92test("lists scenario rows that changed or failed above the collapsed table", () => {
93 const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" };
94 const render = (currentScenarios, baselineScenarios) =>
95 renderComment({
96 current: result(),
97 baseline: result(),
98 currentScenarios,
99 baselineScenarios,
100 currentRun: run,
101 baselineRun: { ...run, matchesBase: true },
102 });
103 const open = (body) => body.slice(0, body.indexOf("<details>"));
104
105 const unchanged = render(scenarios(), scenarios());
106 assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/);
107 assert.match(open(unchanged), /All 4 scenario measurements match `main`/);
108 assert.doesNotMatch(open(unchanged), /long turn finished, cold open/);
109
110 const grown = render(scenarios({ observed: 400_000 }), scenarios());
111 assert.match(open(grown), /1 of 4 scenario measurements changed or failed/);
112 assert.match(
113 open(grown),
114 /\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/,
115 );
116
117 const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios());
118 assert.match(over, /❌ One or more thread transfer ceilings were exceeded/);
119 assert.match(
120 over,
121 /Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/,
122 );
123
124 // A row `main` does not have is not reported as matching it.
125 const added = render(scenarios({ phase: "GH-26" }), scenarios());
126 assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./);
127 assert.match(added, /\| Codex \| `GH-26` \|/);
128
129 // A main run from before the scenario file has nothing to compare; a PR run from before it has no section.
130 assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/);
131 assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/);
132});
133
134test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => {
135 const outputs = {};
136 const listWorkflowRunArtifacts = () => {};
137 // The exact base run sits far past the newest page of main CI runs.
138 const mainRuns = Array.from({ length: 1_000 }, (_, index) => ({
139 id: 1_000 - index,
140 head_sha: index === 900 ? "base-sha" : `main-sha-${index}`,
141 }));
142 const listWorkflowRuns = async ({ head_sha, per_page }) => ({
143 data: {
144 workflow_runs: mainRuns
145 .filter((run) => head_sha === undefined || run.head_sha === head_sha)
146 .slice(0, per_page),
147 },
148 });
149 const listPullRequestsAssociatedWithCommit = () => {};
150 const github = {
151 paginate: async (method, input) => {
152 if (method === listPullRequestsAssociatedWithCommit) {
153 return [
154 {
155 number: 5350,
156 state: "open",
157 head: { sha: "head-sha", ref: "feature-branch", repo: null },
158 },
159 ];
160 }
161 if (method === listWorkflowRunArtifacts) {
162 return [
163 {
164 name: "thread-transfer-results",
165 expired: false,
166 runId: input.run_id,
167 },
168 ];
169 }
170 throw new Error("unexpected pagination call");
171 },
172 rest: {
173 actions: { listWorkflowRunArtifacts, listWorkflowRuns },
174 pulls: {
175 get: async () => ({
176 data: {
177 head: { sha: "head-sha" },
178 base: { sha: "base-sha", ref: "main" },
179 },
180 }),
181 },
182 repos: { listPullRequestsAssociatedWithCommit },
183 },
184 };
185 await resolve({
186 github,
187 context: {
188 repo: { owner: "pingdotgg", repo: "t3code" },
189 payload: {
190 workflow_run: {
191 id: 2,
192 event: "pull_request",
193 workflow_id: 3,
194 head_sha: "head-sha",
195 head_branch: "feature-branch",
196 head_repository: { full_name: "pingdotgg/t3code" },
197 conclusion: "success",
198 pull_requests: [],
199 },
200 },
201 },
202 core: {
203 info: () => {},
204 setOutput: (key, value) => {
205 outputs[key] = value;
206 },
207 },
208 });
209
210 assert.equal(outputs.publish, "true");
211 assert.equal(outputs.pull_number, "5350");
212 assert.equal(outputs.pr_artifact, "true");
213 assert.equal(outputs.baseline_run_id, "100");
214 assert.equal(outputs.baseline_matches_base, "true");
215});
216
217test("does not guess when a fallback commit belongs to multiple PRs", async () => {
218 const outputs = {};
219 const listPullRequestsAssociatedWithCommit = () => {};
220 let fetchedPull = false;
221 await resolve({
222 github: {
223 paginate: async (method) => {
224 assert.equal(method, listPullRequestsAssociatedWithCommit);
225 return [5350, 5351].map((number) => ({
226 number,
227 state: "open",
228 head: {
229 sha: "head-sha",
230 ref: "feature-branch",
231 repo: { full_name: "pingdotgg/t3code" },
232 },
233 }));
234 },
235 rest: {
236 actions: {},
237 pulls: {
238 get: async () => {
239 fetchedPull = true;
240 },
241 },
242 repos: { listPullRequestsAssociatedWithCommit },
243 },
244 },
245 context: {
246 repo: { owner: "pingdotgg", repo: "t3code" },
247 payload: {
248 workflow_run: {
249 id: 2,
250 event: "pull_request",
251 workflow_id: 3,
252 head_sha: "head-sha",
253 head_branch: "feature-branch",
254 head_repository: { full_name: "pingdotgg/t3code" },
255 conclusion: "success",
256 pull_requests: [],
257 },
258 },
259 },
260 core: {
261 info: () => {},
262 setOutput: (key, value) => {
263 outputs[key] = value;
264 },
265 },
266 });
267
268 assert.equal(outputs.publish, "false");
269 assert.equal(fetchedPull, false);
270});
271
272test("skips a missing PR commit but preserves other lookup failures", async () => {
273 for (const [status, message, shouldSkip] of [
274 [422, "No commit found for SHA: old-head", true],
275 [422, "Validation Failed", false],
276 [403, "API rate limit exceeded", false],
277 ]) {
278 const outputs = {};
279 const error = Object.assign(new Error(message), {
280 status,
281 response: { data: { message } },
282 });
283 const run = resolve({
284 github: {
285 paginate: async () => {
286 throw error;
287 },
288 rest: { repos: { listPullRequestsAssociatedWithCommit: () => {} } },
289 },
290 context: {
291 repo: { owner: "pingdotgg", repo: "t3code" },
292 payload: {
293 workflow_run: { event: "pull_request", head_sha: "old-head", pull_requests: [] },
294 },
295 },
296 core: {
297 info: () => {},
298 setOutput: (key, value) => {
299 outputs[key] = value;
300 },
301 },
302 });
303 if (shouldSkip) {
304 await run;
305 assert.equal(outputs.publish, "false");
306 } else {
307 await assert.rejects(run, (caught) => caught === error);
308 assert.deepEqual(outputs, {});
309 }
310 }
311});
312
313test("does not report a passing run that skipped the server tests", async () => {
314 const outputs = {};
315 const listWorkflowRunArtifacts = () => {};
316 await resolve({
317 github: {
318 paginate: async (method) => {
319 assert.equal(method, listWorkflowRunArtifacts);
320 return [];
321 },
322 rest: {
323 actions: { listWorkflowRunArtifacts },
324 pulls: {
325 get: async () => ({
326 data: { head: { sha: "head-sha" }, base: { sha: "base-sha", ref: "main" } },
327 }),
328 },
329 },
330 },
331 context: {
332 repo: { owner: "pingdotgg", repo: "t3code" },
333 payload: {
334 workflow_run: {
335 id: 2,
336 event: "pull_request",
337 workflow_id: 3,
338 head_sha: "head-sha",
339 conclusion: "success",
340 pull_requests: [{ number: 5350 }],
341 },
342 },
343 },
344 core: {
345 info: () => {},
346 setOutput: (key, value) => {
347 outputs[key] = value;
348 },
349 },
350 });
351
352 assert.equal(outputs.publish, "false");
353});
354
355test("does not publish a stale result after the PR head advances", async () => {
356 let listedComments = false;
357 const info = [];
358 const published = await upsertCommentForCurrentHead(
359 {
360 paginate: async () => {
361 listedComments = true;
362 return [];
363 },
364 rest: {
365 issues: {
366 listComments: () => {},
367 createComment: () => {
368 throw new Error("must not create a stale comment");
369 },
370 updateComment: () => {
371 throw new Error("must not update a stale comment");
372 },
373 },
374 pulls: {
375 get: async () => ({ data: { head: { sha: "new-head-sha" } } }),
376 },
377 },
378 },
379 { repo: { owner: "pingdotgg", repo: "t3code" } },
380 { info: (message) => info.push(message) },
381 5350,
382 "old-head-sha",
383 "stale body",
384 );
385
386 assert.equal(published, false);
387 assert.equal(listedComments, false);
388 assert.deepEqual(info, ["Skipping stale CI result old-head-sha; PR head is new-head-sha."]);
389});
390
391test("preserves a successful result when a same-SHA rerun has no artifact", async () => {
392 let updatedComment = false;
393 const sha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
394 const published = await upsertCommentForCurrentHead(
395 {
396 paginate: async () => [
397 {
398 id: 1,
399 user: { login: "github-actions[bot]" },
400 body: `<!-- t3-thread-transfer-report -->\n<!-- t3-thread-transfer-result-sha:${sha} -->`,
401 },
402 ],
403 rest: {
404 issues: {
405 listComments: () => {},
406 createComment: () => {
407 updatedComment = true;
408 },
409 updateComment: () => {
410 updatedComment = true;
411 },
412 },
413 pulls: {
414 get: async () => ({ data: { head: { sha } } }),
415 },
416 },
417 },
418 { repo: { owner: "pingdotgg", repo: "t3code" } },
419 { info: () => {} },
420 5350,
421 sha,
422 "missing artifact warning",
423 { preserveResultSha: sha },
424 );
425
426 assert.equal(published, true);
427 assert.equal(updatedComment, false);
428});