Open History to see the last commit
1const fs = require("node:fs");
2const path = require("node:path");
3
4const ARTIFACT_NAME = "thread-transfer-results";
5const RESULT_FILE = "thread-transfer-result.json";
6const SCENARIOS_FILE = "thread-transfer-scenarios.json";
7const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
8const PROVIDERS = ["codex", "claudeAgent"];
9const OBSERVED_KEYS = [
10 "totalWireBytes",
11 "threadSnapshotWireBytes",
12 "threadSnapshotDecodedBytes",
13 "measuredTurnWebSocketWireBytes",
14 "measuredTurnWebSocketDecodedBytes",
15 "measuredTurnWebSocketMessages",
16];
17const CEILING_KEYS = [
18 "totalWireBytes",
19 "threadSnapshotWireBytes",
20 "measuredTurnWebSocketWireBytes",
21 "measuredTurnWebSocketDecodedBytes",
22 "measuredTurnWebSocketMessages",
23];
24const SCENARIO_KEYS = [
25 "id",
26 "historyTurns",
27 "historyCommandToolsPerTurn",
28 "historyMcpResultBytes",
29 "measuredCommandTools",
30 "measuredMcpResultBytes",
31];
32
33const SCENARIO_ROW_KEYS = ["phase", "metric", "unit", "observed", "ceiling"];
34// These labels come from the PR and reach a comment the bot posts: no markup, links or mentions.
35// GitHub still links plain text such as `GH-26`, so they are rendered as code spans.
36const SCENARIO_LABEL = /^[A-Za-z0-9][A-Za-z0-9 ,-]{0,79}$/;
37const MAX_SCENARIO_ROWS = 100;
38
39function resultShaMarker(sha) {
40 return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
41}
42
43function assertObject(value, label) {
44 if (value === null || typeof value !== "object" || Array.isArray(value)) {
45 throw new Error(`${label} must be an object`);
46 }
47}
48
49function assertExactKeys(value, expected, label) {
50 assertObject(value, label);
51 const actual = Object.keys(value).sort();
52 const wanted = [...expected].sort();
53 if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) {
54 throw new Error(`${label} has unexpected fields`);
55 }
56}
57
58function assertMetric(value, label) {
59 if (!Number.isSafeInteger(value) || value < 0 || value > 1_000_000_000) {
60 throw new Error(`${label} must be a non-negative safe integer below 1,000,000,000`);
61 }
62}
63
64function validateResult(value) {
65 assertExactKeys(value, ["schemaVersion", "scenario", "providers"], "result");
66 if (value.schemaVersion !== 1) {
67 throw new Error("result.schemaVersion must be 1");
68 }
69
70 assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");
71 if (!["thread-transfer-v1", "thread-transfer-v2"].includes(value.scenario.id)) {
72 throw new Error("result.scenario.id is not supported");
73 }
74 for (const key of SCENARIO_KEYS.slice(1)) {
75 assertMetric(value.scenario[key], `result.scenario.${key}`);
76 }
77
78 assertExactKeys(value.providers, PROVIDERS, "result.providers");
79 for (const provider of PROVIDERS) {
80 const entry = value.providers[provider];
81 assertExactKeys(entry, ["observed", "ceiling"], `result.providers.${provider}`);
82 assertExactKeys(entry.observed, OBSERVED_KEYS, `result.providers.${provider}.observed`);
83 assertExactKeys(entry.ceiling, CEILING_KEYS, `result.providers.${provider}.ceiling`);
84 for (const key of OBSERVED_KEYS) {
85 assertMetric(entry.observed[key], `result.providers.${provider}.observed.${key}`);
86 }
87 for (const key of CEILING_KEYS) {
88 assertMetric(entry.ceiling[key], `result.providers.${provider}.ceiling.${key}`);
89 }
90 }
91
92 return value;
93}
94
95function validateScenarios(value) {
96 assertExactKeys(value, ["schemaVersion", "providers"], "scenarios");
97 if (value.schemaVersion !== 1) {
98 throw new Error("scenarios.schemaVersion must be 1");
99 }
100 assertExactKeys(value.providers, PROVIDERS, "scenarios.providers");
101 for (const provider of PROVIDERS) {
102 const rows = value.providers[provider];
103 if (!Array.isArray(rows) || rows.length > MAX_SCENARIO_ROWS) {
104 throw new Error(
105 `scenarios.providers.${provider} must list at most ${MAX_SCENARIO_ROWS} rows`,
106 );
107 }
108 const seen = new Set();
109 for (const [index, row] of rows.entries()) {
110 const label = `scenarios.providers.${provider}[${index}]`;
111 assertExactKeys(row, SCENARIO_ROW_KEYS, label);
112 for (const key of ["phase", "metric"]) {
113 if (typeof row[key] !== "string" || !SCENARIO_LABEL.test(row[key])) {
114 throw new Error(`${label}.${key} must be a short plain label`);
115 }
116 }
117 if (row.unit !== "bytes" && row.unit !== "count") {
118 throw new Error(`${label}.unit must be bytes or count`);
119 }
120 assertMetric(row.observed, `${label}.observed`);
121 assertMetric(row.ceiling, `${label}.ceiling`);
122 const key = scenarioKey(row);
123 if (seen.has(key)) throw new Error(`${label} repeats an earlier row`);
124 seen.add(key);
125 }
126 }
127 return value;
128}
129
130function scenarioKey(row) {
131 return `${row.phase}|${row.metric}`;
132}
133
134function readArtifactFile(directory, name, validate) {
135 if (!directory) return undefined;
136 const file = path.join(directory, name);
137 if (!fs.existsSync(file)) return undefined;
138 const stat = fs.lstatSync(file);
139 if (!stat.isFile() || stat.size > 64 * 1_024) {
140 throw new Error(`${name} must be a regular file smaller than 64 KiB`);
141 }
142 return validate(JSON.parse(fs.readFileSync(file, "utf8")));
143}
144
145function readResult(directory) {
146 return readArtifactFile(directory, RESULT_FILE, validateResult);
147}
148
149// Older runs upload only the result file, so a run without scenarios is still a valid run.
150function readScenarios(directory) {
151 return readArtifactFile(directory, SCENARIOS_FILE, validateScenarios);
152}
153
154function formatBytes(bytes) {
155 if (bytes < 1_024) return `${bytes} B`;
156 if (bytes >= 1_024 * 1_024) return `${(bytes / 1_024 / 1_024).toFixed(2)} MiB`;
157 return `${(bytes / 1_024).toFixed(1)} KiB`;
158}
159
160function formatValue(value, kind) {
161 return kind === "bytes" ? formatBytes(value) : value.toLocaleString("en-US");
162}
163
164function formatImpact(current, baseline, kind) {
165 if (baseline === undefined) return "—";
166 const delta = current - baseline;
167 const prefix = delta > 0 ? "+" : delta < 0 ? "−" : "";
168 const magnitude = formatValue(Math.abs(delta), kind);
169 const percent =
170 baseline === 0 ? "" : ` (${prefix}${Math.abs((delta / baseline) * 100).toFixed(1)}%)`;
171 return `${prefix}${magnitude}${percent}`;
172}
173
174function sameScenario(left, right) {
175 return SCENARIO_KEYS.every((key) => left[key] === right[key]);
176}
177
178const METRICS = [
179 { key: "totalWireBytes", label: "Total thread wire", kind: "bytes" },
180 { key: "threadSnapshotWireBytes", label: "Thread snapshot wire", kind: "bytes" },
181 {
182 key: "measuredTurnWebSocketWireBytes",
183 label: "Live turn WebSocket wire",
184 kind: "bytes",
185 },
186 {
187 key: "measuredTurnWebSocketDecodedBytes",
188 label: "Live turn WebSocket decoded",
189 kind: "bytes",
190 },
191 { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
192];
193
194function providerLabel(provider) {
195 return provider === "codex" ? "Codex" : "Claude";
196}
197
198/**
199 * The scenario rows of a run against the same rows on `main`. Rows that fail or differ from
200 * `main` are listed in the open; the full table is collapsed below them.
201 */
202function renderScenarios(current, baseline, ceilingChanges) {
203 if (!current) return { lines: [], failed: false };
204 const header = [
205 "| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |",
206 "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |",
207 ];
208 const all = [];
209 const notable = [];
210 let failed = false;
211 let withoutBaseline = 0;
212
213 for (const provider of PROVIDERS) {
214 const baselineRows = new Map(
215 (baseline?.providers[provider] ?? []).map((row) => [scenarioKey(row), row]),
216 );
217 for (const row of current.providers[provider]) {
218 const before = baselineRows.get(scenarioKey(row));
219 const beforeObserved = before?.unit === row.unit ? before.observed : undefined;
220 const pass = row.observed <= row.ceiling;
221 failed ||= !pass;
222 if (beforeObserved === undefined) withoutBaseline += 1;
223 const line = `| ${providerLabel(provider)} | \`${row.phase}\` | \`${row.metric}\` | ${beforeObserved === undefined ? "—" : formatValue(beforeObserved, row.unit)} | ${formatValue(row.observed, row.unit)} | ${formatImpact(row.observed, beforeObserved, row.unit)} | ${formatValue(row.ceiling, row.unit)} | ${pass ? "✅" : "❌"} |`;
224 all.push(line);
225 if (!pass || (beforeObserved !== undefined && beforeObserved !== row.observed)) {
226 notable.push(line);
227 }
228 if (beforeObserved !== undefined && before.ceiling !== row.ceiling) {
229 ceilingChanges.push(
230 `- ${providerLabel(provider)} \`${row.phase}\` \`${row.metric}\`: ${formatValue(before.ceiling, row.unit)} → ${formatValue(row.ceiling, row.unit)}`,
231 );
232 }
233 }
234 }
235
236 const compared = all.length - withoutBaseline;
237 const summary = !baseline
238 ? `No \`main\` baseline exists yet for these ${all.length} scenario measurements.`
239 : [
240 notable.length > 0
241 ? `${notable.length} of ${all.length} scenario measurements changed or failed.`
242 : `${compared === all.length ? "All " : ""}${compared} scenario measurements match \`main\`.`,
243 ...(withoutBaseline > 0 ? [`${withoutBaseline} have no \`main\` baseline.`] : []),
244 ].join(" ");
245 return {
246 failed,
247 lines: [
248 "### Other scenarios",
249 "",
250 summary,
251 ...(notable.length > 0 ? ["", ...header, ...notable] : []),
252 "",
253 "<details>",
254 `<summary>All ${all.length} scenario measurements</summary>`,
255 "",
256 ...header,
257 ...all,
258 "",
259 "</details>",
260 "",
261 ],
262 };
263}
264
265function renderComment(input) {
266 const current = input.current;
267 const baseline = input.baseline;
268 const comparable = baseline !== undefined && sameScenario(current.scenario, baseline.scenario);
269 const rows = [];
270 const ceilingChanges = [];
271 let failed = false;
272
273 for (const provider of PROVIDERS) {
274 for (const metric of METRICS) {
275 const observed = current.providers[provider].observed[metric.key];
276 const ceiling = current.providers[provider].ceiling[metric.key];
277 const baselineObserved = comparable
278 ? baseline.providers[provider].observed[metric.key]
279 : undefined;
280 const pass = observed <= ceiling;
281 failed ||= !pass;
282 rows.push(
283 `| ${providerLabel(provider)} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
284 );
285
286 if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
287 ceilingChanges.push(
288 `- ${providerLabel(provider)} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
289 );
290 }
291 }
292 }
293 const scenarios = renderScenarios(
294 input.currentScenarios,
295 input.baselineScenarios,
296 ceilingChanges,
297 );
298 failed ||= scenarios.failed;
299
300 const baselineLink = input.baselineRun
301 ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
302 : "unavailable";
303 const currentLink = `[\`${input.currentRun.sha.slice(0, 7)}\`](${input.currentRun.url})`;
304 const notices = [];
305 if (!baseline) {
306 notices.push(
307 "> ℹ️ No successful `main` baseline artifact is available yet. This run establishes the initial measurement.",
308 );
309 } else if (!comparable) {
310 notices.push(
311 "> ⚠️ The thread fixture changed, so impact percentages are not directly comparable to the `main` baseline.",
312 );
313 } else if (!input.baselineRun.matchesBase) {
314 notices.push(
315 "> ℹ️ The exact PR base did not have a successful artifact. Baseline uses the latest successful `main` measurement shown below.",
316 );
317 }
318 if (ceilingChanges.length > 0) {
319 notices.push(
320 `> ⚠️ **This PR changes transfer ceilings:**\n>\n${ceilingChanges.map((line) => `> ${line}`).join("\n")}`,
321 );
322 }
323
324 return [
325 COMMENT_MARKER,
326 resultShaMarker(input.currentRun.sha),
327 "## Thread transfer impact",
328 "",
329 failed
330 ? "❌ One or more thread transfer ceilings were exceeded."
331 : "✅ Thread transfer remains within every enforced ceiling.",
332 ...(notices.length > 0 ? ["", ...notices] : []),
333 "",
334 "| Provider | Metric | Main baseline | This PR | Impact | PR ceiling | |",
335 "| --- | --- | ---: | ---: | ---: | ---: | --- |",
336 ...rows,
337 "",
338 ...scenarios.lines,
339 `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
340 "",
341 "<details>",
342 "<summary>Scenario and decoded snapshot size</summary>",
343 "",
344 `${current.scenario.historyTurns} historical turns, ${current.scenario.historyCommandToolsPerTurn} command tools per turn, ${formatBytes(current.scenario.historyMcpResultBytes)} retained MCP result per historical turn, and a ${formatBytes(current.scenario.measuredMcpResultBytes)} retained result in the measured turn.`,
345 "",
346 ...PROVIDERS.map(
347 (provider) =>
348 `- ${providerLabel(provider)} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
349 ),
350 "",
351 "</details>",
352 "",
353 "_Updated in place by a trusted workflow. PR artifacts are strictly validated and never executed._",
354 ].join("\n");
355}
356
357async function artifactsForRun(github, owner, repo, runId) {
358 return github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
359 owner,
360 repo,
361 run_id: runId,
362 per_page: 100,
363 });
364}
365
366function findResultArtifact(artifacts) {
367 return artifacts.find((artifact) => artifact.name === ARTIFACT_NAME && !artifact.expired);
368}
369
370async function resolve({ github, context, core }) {
371 const source = context.payload.workflow_run;
372 const { owner, repo } = context.repo;
373 if (source.event !== "pull_request") {
374 core.setOutput("publish", "false");
375 return;
376 }
377
378 let pullNumber = source.pull_requests?.[0]?.number;
379 if (!pullNumber) {
380 let associated;
381 try {
382 associated = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
383 owner,
384 repo,
385 commit_sha: source.head_sha,
386 per_page: 100,
387 });
388 } catch (error) {
389 // A force-pushed or deleted PR head can disappear before this workflow runs.
390 if (
391 error.status === 422 &&
392 error.response?.data?.message === `No commit found for SHA: ${source.head_sha}`
393 ) {
394 core.info(`Skipping CI result for unavailable commit ${source.head_sha}.`);
395 core.setOutput("publish", "false");
396 return;
397 }
398 throw error;
399 }
400 const matchingPulls = associated.filter(
401 (pull) =>
402 pull.state === "open" &&
403 pull.head.sha === source.head_sha &&
404 pull.head.ref === source.head_branch,
405 );
406 if (matchingPulls.length !== 1) {
407 core.info(
408 `Expected one open pull request for ${source.head_repository?.full_name ?? "unknown repository"}:${source.head_branch ?? "unknown branch"} at ${source.head_sha}; found ${matchingPulls.length}.`,
409 );
410 core.setOutput("publish", "false");
411 return;
412 }
413 pullNumber = matchingPulls[0].number;
414 }
415 if (!pullNumber) {
416 core.info("No open pull request is associated with the completed CI run.");
417 core.setOutput("publish", "false");
418 return;
419 }
420
421 const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: pullNumber });
422 if (pull.head.sha !== source.head_sha) {
423 core.info(`Skipping stale CI result ${source.head_sha}; PR head is ${pull.head.sha}.`);
424 core.setOutput("publish", "false");
425 return;
426 }
427
428 const sourceArtifacts = await artifactsForRun(github, owner, repo, source.id);
429 const sourceArtifact = findResultArtifact(sourceArtifacts);
430 // Successful server tests require this artifact; its absence means CI skipped them.
431 if (!sourceArtifact && source.conclusion === "success") {
432 core.info("CI skipped the server tests for this change; nothing to report.");
433 core.setOutput("publish", "false");
434 return;
435 }
436 // One page each for the exact base and the newest runs; paging through
437 // every successful main run costs up to ten requests to keep 20.
438 const listRuns = async (filters) => {
439 const { data } = await github.rest.actions.listWorkflowRuns({
440 owner,
441 repo,
442 workflow_id: source.workflow_id,
443 branch: pull.base.ref,
444 event: "push",
445 status: "success",
446 per_page: 20,
447 ...filters,
448 });
449 return data.workflow_runs;
450 };
451 const [baseRuns, recentRuns] = await Promise.all([
452 listRuns({ head_sha: pull.base.sha }),
453 listRuns({}),
454 ]);
455 const orderedRuns = [
456 ...baseRuns,
457 ...recentRuns.filter((run) => run.head_sha !== pull.base.sha),
458 ].slice(0, 20);
459
460 let baselineRun;
461 for (const run of orderedRuns) {
462 const artifacts = await artifactsForRun(github, owner, repo, run.id);
463 if (findResultArtifact(artifacts)) {
464 baselineRun = run;
465 break;
466 }
467 }
468
469 core.setOutput("publish", "true");
470 core.setOutput("pull_number", String(pullNumber));
471 core.setOutput("pr_artifact", sourceArtifact ? "true" : "false");
472 core.setOutput("pr_run_id", String(source.id));
473 core.setOutput("pr_sha", source.head_sha);
474 core.setOutput("pr_conclusion", source.conclusion ?? "unknown");
475 core.setOutput("baseline_artifact", baselineRun ? "true" : "false");
476 core.setOutput("baseline_run_id", baselineRun ? String(baselineRun.id) : "");
477 core.setOutput("baseline_sha", baselineRun?.head_sha ?? "");
478 core.setOutput(
479 "baseline_matches_base",
480 baselineRun?.head_sha === pull.base.sha ? "true" : "false",
481 );
482}
483
484async function upsertComment(github, context, pullNumber, body, options = {}) {
485 const { owner, repo } = context.repo;
486 const comments = await github.paginate(github.rest.issues.listComments, {
487 owner,
488 repo,
489 issue_number: pullNumber,
490 per_page: 100,
491 });
492 const existing = comments.find(
493 (comment) =>
494 comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER),
495 );
496 if (
497 options.preserveResultSha &&
498 existing?.body?.includes(resultShaMarker(options.preserveResultSha))
499 ) {
500 return;
501 }
502 if (existing) {
503 await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
504 } else {
505 await github.rest.issues.createComment({ owner, repo, issue_number: pullNumber, body });
506 }
507}
508
509async function upsertCommentForCurrentHead(
510 github,
511 context,
512 core,
513 pullNumber,
514 expectedSha,
515 body,
516 options,
517) {
518 const { owner, repo } = context.repo;
519 const { data: pull } = await github.rest.pulls.get({
520 owner,
521 repo,
522 pull_number: pullNumber,
523 });
524 if (pull.head.sha !== expectedSha) {
525 core.info(`Skipping stale CI result ${expectedSha}; PR head is ${pull.head.sha}.`);
526 return false;
527 }
528
529 await upsertComment(github, context, pullNumber, body, options);
530 return true;
531}
532
533async function publish({ github, context, core }) {
534 const pullNumber = Number(process.env.PR_NUMBER);
535 if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) {
536 throw new Error("PR_NUMBER is invalid");
537 }
538
539 const current = readResult(process.env.PR_RESULT_DIR);
540 const currentRun = {
541 sha: process.env.PR_SHA,
542 conclusion: process.env.PR_CONCLUSION,
543 url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.PR_RUN_ID}`,
544 };
545 if (!current) {
546 await upsertCommentForCurrentHead(
547 github,
548 context,
549 core,
550 pullNumber,
551 currentRun.sha,
552 [
553 COMMENT_MARKER,
554 "## Thread transfer impact",
555 "",
556 `⚠️ The latest [CI run](${currentRun.url}) did not produce a thread transfer result for \`${currentRun.sha.slice(0, 7)}\`.`,
557 "",
558 "_This comment will update automatically after the next completed run._",
559 ].join("\n"),
560 { preserveResultSha: currentRun.sha },
561 );
562 return;
563 }
564
565 const baseline = readResult(process.env.BASELINE_RESULT_DIR);
566 const baselineRun = baseline
567 ? {
568 sha: process.env.BASELINE_SHA,
569 matchesBase: process.env.BASELINE_MATCHES_BASE === "true",
570 url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
571 }
572 : undefined;
573 const body = renderComment({
574 current,
575 baseline,
576 currentScenarios: readScenarios(process.env.PR_RESULT_DIR),
577 baselineScenarios: readScenarios(process.env.BASELINE_RESULT_DIR),
578 currentRun,
579 baselineRun,
580 });
581 const published = await upsertCommentForCurrentHead(
582 github,
583 context,
584 core,
585 pullNumber,
586 currentRun.sha,
587 body,
588 );
589 if (published) {
590 core.info(`Updated thread transfer report on PR #${pullNumber}.`);
591 }
592}
593
594module.exports = {
595 publish,
596 readResult,
597 renderComment,
598 resolve,
599 upsertCommentForCurrentHead,
600 validateResult,
601 validateScenarios,
602};