Skip to content
Source
Sign inSign up
TanStack/router
Open History to see the last commit

run.test.mjs

434 lines12.8 KB

1import assert from 'node:assert/strict'
2import { spawnSync } from 'node:child_process'
3import fs from 'node:fs'
4import os from 'node:os'
5import path from 'node:path'
6import test from 'node:test'
7import { fileURLToPath } from 'node:url'
8import { run } from './run.mjs'
9​
10const scriptPath = fileURLToPath(new URL('./run.mjs', import.meta.url))
11const measurePath = fileURLToPath(new URL('./measure.mjs', import.meta.url))
12​
13/** @param {string} filePath */
14function writeCurrent(
15 filePath,
16 gzipBytes = 100,
17 ids = ['react-router.minimal'],
18) {
19 fs.mkdirSync(path.dirname(filePath), { recursive: true })
20 fs.writeFileSync(
21 filePath,
22 JSON.stringify({
23 metrics: ids.map((id) => ({
24 id,
25 gzipBytes,
26 initialGzipBytes: gzipBytes,
27 rawBytes: gzipBytes * 3,
28 brotliBytes: gzipBytes - 10,
29 })),
30 }),
31 )
33​
34/** @param {import('node:test').TestContext} t */
35function fixture(t, measuredIds = ['react-router.minimal']) {
36 const root = fs.mkdtempSync(path.join(os.tmpdir(), 'bundle-runner-'))
37 t.after(() => fs.rmSync(root, { recursive: true, force: true }))
38 /** @type {Array<{command: string, args: string[], options: import('./run.mjs').CommandOptions}>} */
39 const calls = []
40 /** @type {string[]} */
41 const messages = []
42 /** @type {string[]} */
43 const reports = []
44 t.mock.method(
45 console,
46 'error',
47 /** @param {string} message */
48 (message) => messages.push(message),
49 )
50​
51 /** @type {import('./run.mjs').Execute} */
52 const execute = (command, args, options) => {
53 calls.push({ command, args, options })
54 if (args[0] === measurePath) {
55 const outputDir = args[args.indexOf('--results-dir') + 1]
56 assert.ok(outputDir)
57 writeCurrent(path.join(outputDir, 'current.json'), 90, measuredIds)
58 }
59 if (options.stdio !== 'inherit') {
60 fs.writeSync(options.stdio[1], 'noisy stdout\n')
61 fs.writeSync(options.stdio[2], 'noisy stderr\n')
62 return { status: 0 }
63 }
64 const result = spawnSync(command, args, {
65 ...options,
66 stdio: 'pipe',
67 encoding: 'utf8',
68 })
69 reports.push(result.stdout)
70 return result
71 }
72​
73 return { root, calls, messages, reports, execute }
75​
76test('runs targeted tests before measurement and saves named results and logs', (t) => {
77 const { root, calls, messages, reports, execute } = fixture(t)
78 const status = run(
79 [
80 '--results-dir',
81 root,
82 '--name',
83 'baseline',
84 '--scenario',
85 'react-router.minimal,react-router.full',
86 '--test-projects',
87 '@tanstack/router-core,@tanstack/react-router',
88 '--',
89 'tests/path.test.ts',
90 'tests/link.test.tsx',
91 '-t',
92 'links with spaces',
93 ],
94 execute,
95 )
96​
97 assert.equal(status, 0)
98 assert.equal(calls.length, 3)
99 const [tests, measure] = calls
100 assert.ok(tests && measure)
101 assert.ok(tests.args.includes('--target=test:unit'))
102 assert.ok(tests.args.includes('--parallel=1'))
103 assert.ok(tests.args.includes('--skipRemoteCache'))
104 assert.ok(
105 tests.args.includes(
106 '--projects=@tanstack/router-core,@tanstack/react-router',
107 ),
108 )
109 assert.deepEqual(tests.args.slice(-5), [
110 '--',
111 'tests/path.test.ts',
112 'tests/link.test.tsx',
113 '-t',
114 'links with spaces',
115 ])
116 assert.equal(measure.args[0], measurePath)
117 assert.ok(measure.args.includes('react-router.minimal,react-router.full'))
118 assert.ok(!measure.args.includes('--skip-package-builds'))
119 assert.ok(!measure.args.includes('--timings'))
120 assert.ok(!measure.args.includes('tests/path.test.ts'))
121 for (const { options } of calls) {
122 assert.equal(options.env.CI, '1')
123 assert.equal(options.env.NX_DAEMON, 'false')
124 assert.ok(!('shell' in options))
125 }
126 const runDir = path.join(root, 'runs/baseline')
127 assert.ok(fs.existsSync(path.join(runDir, 'current.json')))
128 for (const step of ['tests', 'measure']) {
129 assert.equal(
130 fs.readFileSync(path.join(runDir, `${step}.log`), 'utf8'),
131 'noisy stdout\nnoisy stderr\n',
132 )
133 }
134 assert.ok(messages.every((message) => !message.includes('noisy')))
135 assert.match(reports.join(''), /react-router\.minimal gzip=90/)
136})
137​
138test('compares a named baseline against current results without changing the baseline', (t) => {
139 const { root, reports, execute } = fixture(t)
140 const baseline = path.join(root, 'runs/baseline/current.json')
141 writeCurrent(baseline)
142​
143 assert.equal(
144 run(['--results-dir', root, '--baseline', 'baseline'], execute),
145 0,
146 )
147 assert.match(
148 reports.join(''),
149 /react-router\.minimal 100 -> 90 \(-10\) initial=-10 raw=-30 brotli=-10/,
150 )
151 assert.equal(
152 JSON.parse(fs.readFileSync(baseline, 'utf8')).metrics[0].gzipBytes,
153 100,
154 )
155})
156​
157test('accepts an external baseline and forwards measurement options', (t) => {
158 const { root, calls, reports, execute } = fixture(t)
159 const baseline = path.join(root, 'external baseline.json')
160 writeCurrent(baseline)
161​
162 assert.equal(
163 run(
164 [
165 '--results-dir',
166 root,
167 '--baseline',
168 baseline,
169 '--dist-dir',
170 path.join(root, 'dist with spaces'),
171 '--analysis',
172 '--sourcemap',
173 '--timings',
174 '--skip-package-builds',
175 ],
176 execute,
177 ),
178 0,
179 )
180 assert.deepEqual(calls[0]?.args.slice(3), [
181 '--dist-dir',
182 path.join(root, 'dist with spaces'),
183 '--analysis',
184 '--sourcemap',
185 '--timings',
186 '--skip-package-builds',
187 ])
188 assert.ok(calls.slice(1).every(({ args }) => !args.includes('--timings')))
189 assert.match(reports.join(''), /100 -> 90 \(-10\)/)
190})
191​
192test('targeted comparisons omit unrequested scenarios from a broader baseline', async (t) => {
193 for (const [
194 scenario,
195 measuredIds,
196 ] of /** @type {Array<[string, string[]]>} */ ([
197 ['react-router.minimal', ['react-router.minimal']],
198 ['react-router-minimal', ['react-router.minimal']],
199 [
200 ' solid-router.minimal, react-router.minimal ',
201 ['solid-router.minimal', 'react-router.minimal'],
202 ],
203 ])) {
204 await t.test(scenario, (t) => {
205 const { root, reports, execute } = fixture(t, measuredIds)
206 writeCurrent(path.join(root, 'runs/baseline/current.json'), 100, [
207 'react-router.full',
208 'react-router.minimal',
209 'solid-router.minimal',
210 ])
211​
212 assert.equal(
213 run(
214 [
215 '--results-dir',
216 root,
217 '--baseline',
218 'baseline',
219 '--scenario',
220 scenario,
221 ],
222 execute,
223 ),
224 0,
225 )
226 assert.equal(
227 reports.join(''),
228 [...measuredIds]
229 .sort()
230 .map((id) => `${id} 100 -> 90 (-10) initial=-10 raw=-30 brotli=-10\n`)
231 .join(''),
232 )
233 })
234 }
235})
236​
237test('targeted comparisons keep measured scenarios missing from the baseline', (t) => {
238 const { root, reports, execute } = fixture(t)
239 writeCurrent(path.join(root, 'runs/baseline/current.json'), 100, [
240 'react-router.full',
241 ])
242​
243 assert.equal(
244 run(
245 [
246 '--results-dir',
247 root,
248 '--baseline',
249 'baseline',
250 '--scenario',
251 'react-router.minimal',
252 ],
253 execute,
254 ),
255 0,
256 )
257 assert.equal(
258 reports.join(''),
259 'react-router.minimal n/a -> 90 (n/a) initial=n/a raw=n/a brotli=n/a\n',
260 )
261})
262​
263test('unfiltered comparisons retain baseline-only scenarios', (t) => {
264 const { root, reports, execute } = fixture(t)
265 writeCurrent(path.join(root, 'runs/baseline/current.json'), 100, [
266 'react-router.full',
267 ])
268​
269 assert.equal(
270 run(['--results-dir', root, '--baseline', 'baseline'], execute),
271 0,
272 )
273 assert.equal(
274 reports.join(''),
275 'react-router.full 100 -> n/a (n/a) initial=n/a raw=n/a brotli=n/a\n' +
276 'react-router.minimal n/a -> 90 (n/a) initial=n/a raw=n/a brotli=n/a\n',
277 )
278})
279​
280test('current-only diffs filter JSON output and preserve explicit ID selection', (t) => {
281 const { root } = fixture(t)
282 const baseline = path.join(root, 'baseline.json')
283 const current = path.join(root, 'current.json')
284 writeCurrent(baseline, 100, ['react-router.full', 'react-router.minimal'])
285 writeCurrent(current, 90, ['solid-router.minimal', 'react-router.minimal'])
286 const args = [
287 fileURLToPath(new URL('./diff.mjs', import.meta.url)),
288 '--baseline',
289 baseline,
290 '--current',
291 current,
292 '--current-only',
293 '--json',
294 ]
295 const result = spawnSync(process.execPath, args, { encoding: 'utf8' })
296 assert.equal(result.status, 0, result.stderr)
297 assert.deepEqual(JSON.parse(result.stdout), [
298 {
299 id: 'react-router.minimal',
300 baseline: 100,
301 current: 90,
302 delta: -10,
303 initialDelta: -10,
304 rawDelta: -30,
305 brotliDelta: -10,
306 },
307 { id: 'solid-router.minimal', current: 90 },
308 ])
309​
310 const selected = spawnSync(
311 process.execPath,
312 [...args, '--id', 'react-router.full'],
313 {
314 encoding: 'utf8',
315 },
316 )
317 assert.equal(selected.status, 0, selected.stderr)
318 assert.deepEqual(JSON.parse(selected.stdout), [
319 { id: 'react-router.full', baseline: 100 },
320 ])
321})
322​
323test('stops on test failure, preserves its exit code, and prints only a bounded log tail', (t) => {
324 const { root, messages } = fixture(t)
325 let callCount = 0
326 const log = `early output\n${'x'.repeat(10_000)}\n${'later output\n'.repeat(60)}last error\n`
327 const status = run(
328 ['--results-dir', root, '--test-projects', '@tanstack/react-router'],
329 (command, args, options) => {
330 callCount++
331 assert.ok(options.stdio !== 'inherit')
332 fs.writeSync(options.stdio[1], log)
333 return { status: 42 }
334 },
335 )
336​
337 assert.equal(status, 42)
338 assert.equal(callCount, 1)
339 assert.equal(fs.readFileSync(path.join(root, 'tests.log'), 'utf8'), log)
340 assert.ok(!fs.existsSync(path.join(root, 'measure.log')))
341 assert.match(messages.join('\n'), /tests failed: exit 42/)
342 assert.match(messages.join('\n'), /last error/)
343 assert.doesNotMatch(messages.join('\n'), /early output|xxx/)
344 const tail = messages.at(-1)
345 assert.ok(tail)
346 assert.ok(tail.split('\n').length <= 40)
347 assert.ok(tail.length <= 8000)
348})
349​
350test('bounds failure output even when a log contains one very long line', (t) => {
351 const { root, messages } = fixture(t)
352 assert.equal(
353 run(['--results-dir', root], (command, args, options) => {
354 assert.ok(options.stdio !== 'inherit')
355 fs.writeSync(options.stdio[1], 'x'.repeat(20_000))
356 return { status: 1 }
357 }),
358 1,
359 )
360 assert.equal(messages.at(-1), 'x'.repeat(8000))
361})
362​
363test('does not report stale results after a failed or interrupted measurement', async (t) => {
364 for (const result of [
365 { status: 17 },
366 { status: null, signal: 'SIGTERM' },
367 { status: null, error: new Error('spawn failed') },
368 ]) {
369 await t.test(
370 result.error?.message || result.signal || `exit ${result.status}`,
371 (t) => {
372 const { root, messages } = fixture(t)
373 let callCount = 0
374 writeCurrent(path.join(root, 'current.json'))
375 assert.equal(
376 run(['--results-dir', root], () => {
377 callCount++
378 return result
379 }),
380 result.status || 1,
381 )
382 assert.equal(callCount, 1)
383 assert.match(messages.join('\n'), /measure failed:/)
384 },
385 )
386 }
387})
388​
389test('rejects invalid inputs and protects saved results before running commands', (t) => {
390 const { root, calls, execute } = fixture(t)
391 const current = path.join(root, 'current.json')
392 writeCurrent(current)
393 writeCurrent(path.join(root, 'runs/baseline/current.json'))
394 fs.writeFileSync(path.join(root, 'empty.json'), '{"metrics":[]}')
395 fs.writeFileSync(path.join(root, 'invalid.json'), '{')
396 /** @type {Array<[string[], RegExp]>} */
397 const cases = [
398 [['--unknown'], /Unknown option/],
399 [['--', 'tests/path.test.ts'], /require --test-projects/],
400 [['--test-projects', ' , '], /at least one project/],
401 [['--scenario', ' , '], /at least one scenario/],
402 [['--name', '../escape'], /Run names/],
403 [['--name', 'baseline'], /Named run already exists/],
404 [['--baseline', 'missing'], /ENOENT/],
405 [['--baseline', current], /baseline cannot be the current output/],
406 [['--baseline', path.join(root, 'empty.json')], /no metrics/],
407 [['--baseline', path.join(root, 'invalid.json')], /JSON/],
408 ]
409 for (const [args, error] of cases) {
410 assert.throws(() => run(['--results-dir', root, ...args], execute), error)
411 }
412 assert.equal(calls.length, 0)
413 assert.equal(
414 JSON.parse(fs.readFileSync(current, 'utf8')).metrics[0].gzipBytes,
415 100,
416 )
417})
418​
419test('reports help and argument errors through the CLI entry point', () => {
420 const help = spawnSync(process.execPath, [scriptPath, '--help'], {
421 encoding: 'utf8',
422 })
423 assert.equal(help.status, 0)
424 assert.match(help.stdout, /Usage: pnpm benchmark:bundle-size:run/)
425 assert.match(
426 help.stdout,
427 /--timings\s+Record fresh phase timings in measure\.log/,
428 )
429 const invalid = spawnSync(process.execPath, [scriptPath, '--unknown'], {
430 encoding: 'utf8',
431 })
432 assert.equal(invalid.status, 1)
433 assert.match(invalid.stderr, /Unknown option/)
434})