Skip to content
Source
Sign inSign up
TanStack/router
Open History to see the last commit

stable-runner.ts

269 lines8.4 KB

1import { fork } from 'node:child_process'
2import { createHash } from 'node:crypto'
3import {
4 copyFileSync,
5 mkdirSync,
6 readFileSync,
7 rmSync,
8 writeFileSync,
9} from 'node:fs'
10import { dirname, resolve } from 'node:path'
11import { fileURLToPath } from 'node:url'
12import { parseArgs } from 'node:util'
13import { LINK_CASES } from './cases'
14import { createStagingDirectory } from './staging'
15import { classify, mean, summarizeRatios } from './statistics'
16import type { LinkCaseId } from './cases'
17import type {
18 BlockSample,
19 Mode,
20 Variant,
21 WorkerRequest,
22 WorkerResponse,
23} from './worker-protocol'
24​
25const projectRoot = fileURLToPath(new URL('.', import.meta.url))
26const workspaceRoot = resolve(projectRoot, '../../..')
27​
28function startWorker() {
29 const child = fork(
30 fileURLToPath(new URL('./stable-worker.ts', import.meta.url)),
31 {
32 cwd: workspaceRoot,
33 env: { ...process.env, NODE_ENV: 'production' },
34 execArgv: [
35 '--import=@swc-node/register/esm-register',
36 '--random-seed=42',
37 '--hash-seed=42',
38 ],
39 stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
40 },
41 )
42 let stderr = ''
43 child.stderr?.on('data', (data: Buffer) => {
44 stderr = (stderr + data.toString()).slice(-4_096)
45 })
46​
47 function request(message: WorkerRequest): Promise<WorkerResponse> {
48 return new Promise((resolveResponse, reject) => {
49 const timeout = setTimeout(() => {
50 finish(
51 new Error(`Sampling worker timed out: ${message.kind}\n${stderr}`),
52 )
53 }, 60_000)
54 const onExit = (code: number | null) =>
55 finish(new Error(`Sampling worker exited (${code})\n${stderr}`))
56 const onError = (error: Error) => finish(error)
57 const onMessage = (response: WorkerResponse) => {
58 if (response.kind === 'error') {
59 finish(new Error(response.message))
60 } else {
61 finish(undefined, response)
62 }
63 }
64 function finish(error?: Error, response?: WorkerResponse) {
65 clearTimeout(timeout)
66 child.off('message', onMessage)
67 child.off('error', onError)
68 child.off('exit', onExit)
69 if (error) {
70 reject(error)
71 } else if (response) {
72 resolveResponse(response)
73 }
74 }
75 child.once('message', onMessage)
76 child.once('error', onError)
77 child.once('exit', onExit)
78 child.send(message, (error) => {
79 if (error) {
80 finish(error)
81 }
82 })
83 })
84 }
85​
86 return {
87 request,
88 kill: () => child.kill(),
89 }
91​
92async function sampleReplica(
93 mode: Mode,
94 caseId: LinkCaseId,
95 baseline: string,
96 current: string,
97 replica: number,
98) {
99 const worker = startWorker()
100 const samples: Array<Array<BlockSample>> = [[], []]
101 const ratios: Array<{ cpu: number; wall: number }> = []
102 try {
103 const starts: Array<number> = []
104 // Balance module initialization and which variant finishes warming last.
105 const startupOrder: Array<Variant> = replica % 2 ? [1, 0] : [0, 1]
106 for (const index of startupOrder) {
107 const ready = await worker.request({
108 kind: 'init',
109 mode,
110 caseId,
111 bundle: index === 0 ? baseline : current,
112 variant: index,
113 })
114 if (ready.kind !== 'ready') {
115 throw new Error('Expected a ready sampling worker')
116 }
117 starts[index] = ready.batchMs
118 }
119 const iterations = Math.max(1, Math.ceil(500 / Math.min(...starts)))
120 for (let round = 0; round < 2; round++) {
121 const block: Array<Array<BlockSample>> = [[], []]
122 const order: Array<Variant> =
123 (round + replica) % 2 ? [1, 0, 0, 1] : [0, 1, 1, 0]
124 for (const index of order) {
125 const result = await worker.request({
126 kind: 'measure',
127 iterations,
128 variant: index,
129 })
130 if (result.kind !== 'sample') {
131 throw new Error('Expected a block measurement')
132 }
133 samples[index]!.push(result.sample)
134 block[index]!.push(result.sample)
135 }
136 ratios.push({
137 cpu:
138 mean(block[1]!.map((s) => s.cpuMs)) /
139 mean(block[0]!.map((s) => s.cpuMs)),
140 wall:
141 mean(block[1]!.map((s) => s.wallMs)) /
142 mean(block[0]!.map((s) => s.wallMs)),
143 })
144 }
145 const response = await worker.request({ kind: 'stop' })
146 if (response.kind !== 'stopped') {
147 throw new Error('Expected successful post-measurement assertions')
148 }
149 return {
150 iterations,
151 baseline: samples[0]!,
152 current: samples[1]!,
153 cpuRatio: Math.exp(mean(ratios.map((r) => Math.log(r.cpu)))),
154 wallRatio: Math.exp(mean(ratios.map((r) => Math.log(r.wall)))),
155 }
156 } finally {
157 worker.kill()
158 }
160​
161async function main() {
162 const { values } = parseArgs({
163 options: {
164 baseline: { type: 'string' },
165 current: { type: 'string', default: resolve(projectRoot, 'dist') },
166 mode: { type: 'string', default: 'all' },
167 repeats: { type: 'string', default: '4' },
168 outputJson: { type: 'string' },
169 testNamePattern: { type: 'string', short: 't' },
170 },
171 })
172 if (!values.baseline || !values.outputJson) {
173 throw new Error(
174 'Specify --baseline <dist directory> and --outputJson <file>',
175 )
176 }
177 const repeats = Number(values.repeats)
178 if (!Number.isInteger(repeats) || repeats < 3 || repeats > 11) {
179 throw new Error('--repeats must be an integer between 3 and 11')
180 }
181 if (!['all', 'client', 'ssr'].includes(values.mode)) {
182 throw new Error('--mode must be all, client, or ssr')
183 }
184 const modes: Array<Mode> =
185 values.mode === 'all'
186 ? ['client', 'ssr']
187 : values.mode === 'client'
188 ? ['client']
189 : ['ssr']
190 const pattern = values.testNamePattern
191 ? new RegExp(values.testNamePattern)
192 : undefined
193 const cases = LINK_CASES.filter(
194 (c) => !pattern || pattern.test(`${c.label} ${c.id}`),
195 )
196 if (!cases.length) {
197 throw new Error('No Link workloads match the requested pattern')
198 }
199 const report: {
200 protocol: string
201 node: string
202 complete: boolean
203 results: Array<{
204 mode: Mode
205 caseId: LinkCaseId
206 baselineSha256: string
207 currentSha256: string
208 replicas: Array<Awaited<ReturnType<typeof sampleReplica>>>
209 cpu: ReturnType<typeof summarizeRatios>
210 wall: ReturnType<typeof summarizeRatios>
211 verdict: ReturnType<typeof classify>
212 }>
213 } = {
214 protocol:
215 'fresh process per case/replica; shared React runtime; separately loaded router variants; 2 ABBA/BAAB rounds; ~500ms calibrated fixed-work blocks; >=2s/100 batches warmup; main-thread CPU and wall time; fixed V8 random/hash seeds',
216 node: process.version,
217 complete: false,
218 results: [],
219 }
220 const output = resolve(values.outputJson)
221 mkdirSync(dirname(output), { recursive: true })
222 const staging = createStagingDirectory()
223 try {
224 for (const mode of modes) {
225 const baselineInput = resolve(values.baseline, mode, 'app.js')
226 const currentInput = resolve(values.current, mode, 'app.js')
227 // Both snapshots resolve React from the same package installation.
228 const baseline = resolve(staging, `baseline-${mode}.mjs`)
229 const current = resolve(staging, `current-${mode}.mjs`)
230 copyFileSync(baselineInput, baseline)
231 copyFileSync(currentInput, current)
232 const digest = (file: string) =>
233 createHash('sha256').update(readFileSync(file)).digest('hex')
234 const baselineSha256 = digest(baseline)
235 const currentSha256 = digest(current)
236 for (const { id } of cases) {
237 const replicas = []
238 for (let replica = 0; replica < repeats; replica++) {
239 replicas.push(
240 await sampleReplica(mode, id, baseline, current, replica),
241 )
242 }
243 const cpu = summarizeRatios(replicas.map((r) => r.cpuRatio))
244 const wall = summarizeRatios(replicas.map((r) => r.wallRatio))
245 const verdict = classify(cpu, wall)
246 report.results.push({
247 mode,
248 caseId: id,
249 baselineSha256,
250 currentSha256,
251 replicas,
252 cpu,
253 wall,
254 verdict,
255 })
256 writeFileSync(output, JSON.stringify(report, null, 2))
257 console.log(
258 `${mode} ${id}: CPU ${cpu.changePercent.toFixed(2)}% [${cpu.low95.toFixed(2)}, ${cpu.high95.toFixed(2)}], wall ${wall.changePercent.toFixed(2)}% [${wall.low95.toFixed(2)}, ${wall.high95.toFixed(2)}] ${verdict}`,
259 )
260 }
261 }
262 report.complete = true
263 writeFileSync(output, JSON.stringify(report, null, 2))
264 } finally {
265 rmSync(staging, { recursive: true })
266 }
268​
269await main()