Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off globalConsole:off - a synchronous wrapper around one vitest project.
2import * as NodeChildProcess from "node:child_process";
3import * as NodeFS from "node:fs";
4import * as NodeOS from "node:os";
5import * as NodePath from "node:path";
6
7import { vitePlusCommand } from "./check-changed-command.ts";
8
9const HELP = `Usage:
10 node scripts/render-metrics.ts run [--out <file>] [--runs <n>]
11 node scripts/render-metrics.ts compare <before.json> <after.json> [--top <n>]
12
13run Runs the web render scenarios (apps/web/test/renderScenarios.renders.tsx,
14 vitest project "renders") on this checkout and writes one JSON object of
15 { scenario: { commits, renders, components } }. With --runs above 1 (default 3)
16 it fails unless every run produced identical counts.
17compare Prints a markdown table of commit and render deltas per scenario, plus the
18 components whose counts changed most (default top 10).
19
20commits are React commits that rendered at least one component (null where frame
21timing splits them); renders are function and class component renders, counted from the
22fibers React worked on. Every scenario waits for the tree to go quiet before and after
23its counted window, so layout follow-ups are included.
24
25Comparing two refs needs both checkouts, since the scenarios run each tree's code:
26 node scripts/render-metrics.ts run --out ../main-renders.json # on main
27 node scripts/render-metrics.ts run --out ../branch-renders.json # on the branch
28 node scripts/render-metrics.ts compare ../main-renders.json ../branch-renders.json
29A second checkout comes from \`git worktree add ../<name> main\`; copy the harness files
30in if that tree predates them.`;
31
32export interface ScenarioReport {
33 /** null where a scenario's commit count depends on frame timing. */
34 readonly commits: number | null;
35 readonly renders: number;
36 readonly components: Readonly<Record<string, number>>;
37}
38export type RenderMetrics = Readonly<Record<string, ScenarioReport>>;
39
40const root = NodePath.resolve(import.meta.dirname, "..");
41
42export function runOnce(): RenderMetrics {
43 const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-renders-"));
44 try {
45 const [command, commandArgs] = vitePlusCommand(["test", "run", "--project", "renders"]);
46 const result = NodeChildProcess.spawnSync(command, commandArgs, {
47 cwd: NodePath.join(root, "apps/web"),
48 env: { ...process.env, T3CODE_RENDER_REPORT_DIR: directory },
49 stdio: ["ignore", "inherit", "inherit"],
50 });
51 if (result.status !== 0) throw new Error(`render scenarios failed (exit ${result.status})`);
52 const files = NodeFS.readdirSync(directory)
53 .filter((file) => file.endsWith(".json"))
54 .toSorted();
55 return Object.fromEntries(
56 files.map((file) => [
57 file.replace(/\.json$/, ""),
58 JSON.parse(NodeFS.readFileSync(NodePath.join(directory, file), "utf8")) as ScenarioReport,
59 ]),
60 );
61 } finally {
62 NodeFS.rmSync(directory, { recursive: true, force: true });
63 }
64}
65
66/** Scenarios whose counts differ between runs, by name. */
67export function nondeterministic(runs: ReadonlyArray<RenderMetrics>): string[] {
68 const [first, ...rest] = runs;
69 if (!first) return [];
70 return Object.keys(first).filter((scenario) =>
71 rest.some((run) => JSON.stringify(run[scenario]) !== JSON.stringify(first[scenario])),
72 );
73}
74
75const signed = (value: number) => (value > 0 ? `+${value}` : `${value}`);
76const change = (before: number | null, after: number | null) =>
77 before === null || after === null
78 ? "n/a"
79 : before === after
80 ? `${after}`
81 : `${before} → ${after} (${signed(after - before)}${before ? `, ${signed(Math.round(((after - before) / before) * 100))}%` : ""})`;
82
83export function renderComparison(before: RenderMetrics, after: RenderMetrics, top = 10): string {
84 const scenarios = [...new Set([...Object.keys(before), ...Object.keys(after)])].toSorted();
85 const lines = [
86 "### Render scenarios",
87 "",
88 "| scenario | commits | renders |",
89 "| :-- | --: | --: |",
90 ];
91 const empty: ScenarioReport = { commits: 0, renders: 0, components: {} };
92 for (const scenario of scenarios) {
93 const a = before[scenario] ?? empty;
94 const b = after[scenario] ?? empty;
95 lines.push(
96 `| ${scenario} | ${change(a.commits, b.commits)} | ${change(a.renders, b.renders)} |`,
97 );
98 }
99 for (const scenario of scenarios) {
100 const a = before[scenario]?.components ?? {};
101 const b = after[scenario]?.components ?? {};
102 const moved = [...new Set([...Object.keys(a), ...Object.keys(b)])]
103 .map((name) => [name, a[name] ?? 0, b[name] ?? 0] as const)
104 .filter(([, x, y]) => x !== y)
105 .sort(([n, x, y], [m, p, q]) => Math.abs(q - p) - Math.abs(y - x) || n.localeCompare(m))
106 .slice(0, top);
107 if (moved.length === 0) continue;
108 lines.push("", `<details><summary>${scenario}: components that changed most</summary>`, "");
109 lines.push("| component | renders |", "| :-- | --: |");
110 for (const [name, x, y] of moved) lines.push(`| \`${name}\` | ${change(x, y)} |`);
111 lines.push("", "</details>");
112 }
113 return lines.join("\n");
114}
115
116function option(args: ReadonlyArray<string>, name: string): string | undefined {
117 const index = args.indexOf(name);
118 return index === -1 ? undefined : args[index + 1];
119}
120
121if (import.meta.main) {
122 const [command, ...args] = process.argv.slice(2);
123 if (command === "run") {
124 const runs = Number(option(args, "--runs") ?? 3);
125 const results = Array.from({ length: runs }, runOnce);
126 const unstable = nondeterministic(results);
127 if (unstable.length) {
128 for (const scenario of unstable) {
129 const counts = results.map((run) => `${run[scenario]?.commits}/${run[scenario]?.renders}`);
130 console.error(`${scenario} differed between runs (commits/renders): ${counts.join(", ")}`);
131 const first = results[0]?.[scenario]?.components ?? {};
132 for (const [index, run] of results.entries()) {
133 const other = run[scenario]?.components ?? {};
134 const names = [...new Set([...Object.keys(first), ...Object.keys(other)])];
135 const moved = names.filter((name) => first[name] !== other[name]);
136 if (moved.length) {
137 const detail = moved.map((name) => `${name} ${first[name] ?? 0}→${other[name] ?? 0}`);
138 console.error(` run ${index + 1} vs run 1: ${detail.join(", ")}`);
139 }
140 }
141 }
142 process.exitCode = 1;
143 } else {
144 const json = JSON.stringify(results[0], null, 2);
145 const out = option(args, "--out");
146 if (out) NodeFS.writeFileSync(out, json + "\n");
147 else console.log(json);
148 console.error(`${runs} run(s) produced identical counts.`);
149 }
150 } else if (command === "compare" && args[0] && args[1]) {
151 const read = (path: string) => JSON.parse(NodeFS.readFileSync(path, "utf8")) as RenderMetrics;
152 console.log(
153 renderComparison(read(args[0]), read(args[1]), Number(option(args, "--top") ?? 10)),
154 );
155 } else {
156 console.log(HELP);
157 if (command !== "--help" && command !== "-h" && command !== undefined) process.exitCode = 1;
158 }
159}