Skip to content
Source
Sign inSign up
TanStack/db
Open History to see the last commit

compare-incremental-update.ts

534 lines15.2 KB

1/**
2 * Compares two JSON reports produced by `scripts/bench/incremental-update.ts`
3 * and prints a markdown summary of the differences.
4 *
5 * Usage:
6 * tsx scripts/bench/compare-incremental-update.ts \
7 * --base=.tmp/perf/base.json \
8 * --candidate=.tmp/perf/candidate.json \
9 * [--outFile=.tmp/perf/comparison.md] \
10 * [--threshold=0.20] \
11 * [--failOnRegression=false]
12 */
13import { readFileSync, writeFileSync } from 'node:fs'
14​
15type Summary = {
16 iterations: number
17 medianMs: number
18 p75Ms: number
19 p95Ms: number
20 minMs: number
21 maxMs: number
22 stddevMs: number
24​
25type FixtureScale = {
26 label: string
27 issueCount: number
28 userCount: number
29 commentCount: number
31​
32type RunResult = {
33 query: string
34 scenario: string
35 scale: FixtureScale
36 sourceIndexMode: string
37 mutationMode: string
38 // Older reports may lack cold hydrate data.
39 coldHydrateMs?: number
40 writeSummary: Summary
42​
43type Report = {
44 metadata: {
45 node: string
46 platform: string
47 cpu: string
48 gitSha: string
49 iterations: number
50 warmup: number
51 }
52 results: Array<RunResult>
54​
55type Comparison = {
56 key: string
57 query: string
58 scenario: string
59 scale: FixtureScale
60 sourceIndexMode: string
61 mutationMode: string
62 base: RunResult
63 candidate: RunResult
64 medianRatio: number
65 p95Ratio: number
67​
68/**
69 * A per-case timing that the report compares. Writes use the per-write median
70 * of many samples. Cold hydrate is one sample per case, so same-code runs vary
71 * by up to about ±50% on small cases; its flags need a larger relative change
72 * and a larger absolute floor.
73 */
74type Metric = {
75 label: string
76 floorMs: number
77 minRelativeThreshold: number
78 value: (result: RunResult) => number | undefined
80​
81// Relative change below which a result is considered noise, and an absolute
82// floor so sub-hundredth-of-a-ms jitter never counts as a change.
83const defaultThreshold = 0.2
84const absoluteFloorMs = 0.05
85const coldAbsoluteFloorMs = 5
86const coldRelativeThreshold = 0.5
87​
88const writeMetric: Metric = {
89 label: `median write time`,
90 floorMs: absoluteFloorMs,
91 minRelativeThreshold: 0,
92 value: (result) => result.writeSummary.medianMs,
94const coldMetric: Metric = {
95 label: `cold hydrate time`,
96 floorMs: coldAbsoluteFloorMs,
97 minRelativeThreshold: coldRelativeThreshold,
98 value: (result) => result.coldHydrateMs,
100​
101const args = parseArgs(process.argv.slice(2))
102​
103const base = readReport(args.base)
104const candidate = readReport(args.candidate)
105​
106const comparisons = compareReports(base, candidate)
107const markdown = formatMarkdown(base, candidate, comparisons, args.threshold)
108​
109if (args.outFile) {
110 writeFileSync(args.outFile, markdown)
112console.log(markdown)
113​
114const regressions = comparisons.filter((comparison) =>
115 isRegression(comparison, args.threshold),
117if (args.failOnRegression && regressions.length > 0) {
118 console.error(`\n${regressions.length} benchmark regression(s) found`)
119 process.exit(1)
121​
122function readReport(path: string): Report {
123 return JSON.parse(readFileSync(path, `utf8`)) as Report
125​
126function resultKey(result: RunResult): string {
127 return [
128 result.query,
129 result.scenario,
130 result.scale.label,
131 result.scale.issueCount,
132 result.scale.userCount,
133 result.scale.commentCount,
134 result.sourceIndexMode,
135 result.mutationMode,
136 ].join(`|`)
138​
139function compareReports(
140 baseReport: Report,
141 candidateReport: Report,
142): Array<Comparison> {
143 const baseByKey = new Map(
144 baseReport.results.map((result) => [resultKey(result), result]),
145 )
146​
147 const matched: Array<Comparison> = []
148 for (const result of candidateReport.results) {
149 const key = resultKey(result)
150 const baseResult = baseByKey.get(key)
151 if (!baseResult) continue
152​
153 matched.push({
154 key,
155 query: result.query,
156 scenario: result.scenario,
157 scale: result.scale,
158 sourceIndexMode: result.sourceIndexMode,
159 mutationMode: result.mutationMode,
160 base: baseResult,
161 candidate: result,
162 medianRatio: ratio(
163 baseResult.writeSummary.medianMs,
164 result.writeSummary.medianMs,
165 ),
166 p95Ratio: ratio(baseResult.writeSummary.p95Ms, result.writeSummary.p95Ms),
167 })
168 }
169​
170 return matched
172​
173function ratio(baseMs: number, candidateMs: number): number {
174 if (baseMs <= 0) return candidateMs <= 0 ? 1 : Number.POSITIVE_INFINITY
175 return candidateMs / baseMs
177​
178function isSignificant(
179 baseMs: number,
180 candidateMs: number,
181 threshold: number,
182 floorMs: number,
183): boolean {
184 const relativeChange = Math.abs(ratio(baseMs, candidateMs) - 1)
185 const absoluteChange = Math.abs(candidateMs - baseMs)
186 return relativeChange > threshold && absoluteChange > floorMs
188​
189/** The candidate/base ratio of a metric, or undefined when a side lacks it. */
190function metricRatio(
191 comparison: Comparison,
192 metric: Metric,
193): number | undefined {
194 const baseMs = metric.value(comparison.base)
195 const candidateMs = metric.value(comparison.candidate)
196 if (baseMs === undefined || candidateMs === undefined) return undefined
197 return ratio(baseMs, candidateMs)
199​
200function changeDirection(
201 comparison: Comparison,
202 threshold: number,
203 metric: Metric,
204): -1 | 0 | 1 {
205 const baseMs = metric.value(comparison.base)
206 const candidateMs = metric.value(comparison.candidate)
207 if (baseMs === undefined || candidateMs === undefined) return 0
208 const relative = Math.max(threshold, metric.minRelativeThreshold)
209 if (!isSignificant(baseMs, candidateMs, relative, metric.floorMs)) return 0
210 return candidateMs > baseMs ? 1 : -1
212​
213function isRegression(
214 comparison: Comparison,
215 threshold: number,
216 metric: Metric = writeMetric,
217): boolean {
218 return changeDirection(comparison, threshold, metric) === 1
220​
221function isImprovement(
222 comparison: Comparison,
223 threshold: number,
224 metric: Metric = writeMetric,
225): boolean {
226 return changeDirection(comparison, threshold, metric) === -1
228​
229function marker(
230 comparison: Comparison,
231 threshold: number,
232 metric: Metric = writeMetric,
233): string {
234 if (isRegression(comparison, threshold, metric)) return `🔴`
235 if (isImprovement(comparison, threshold, metric)) return `🟢`
236 return ``
238​
239// Individual rows are noisy at sub-ms timings, but a consistent shift across
240// many rows is a real change even when no single row clears the per-row
241// significance bar — the geometric mean of the ratios captures that.
242function geomeanRatio(
243 group: Array<Comparison>,
244 metric: Metric = writeMetric,
245): number {
246 const finite = group
247 .map((comparison) => metricRatio(comparison, metric))
248 .filter(
249 (value): value is number =>
250 value !== undefined && Number.isFinite(value) && value > 0,
251 )
252 if (finite.length === 0) return 1
253 return Math.exp(
254 finite.reduce((sum, value) => sum + Math.log(value), 0) / finite.length,
255 )
257​
258function formatRatio(value: number): string {
259 return `${value.toFixed(2)}×`
261​
262function formatMarkdown(
263 baseReport: Report,
264 candidateReport: Report,
265 allComparisons: Array<Comparison>,
266 threshold: number,
267): string {
268 const lines: Array<string> = []
269 const coldComparisons = allComparisons.filter(
270 (comparison) => metricRatio(comparison, coldMetric) !== undefined,
271 )
272​
273 lines.push(`## Incremental update benchmark`)
274 lines.push(``)
275 lines.push(
276 `Comparing \`${candidateReport.metadata.gitSha}\` (this PR) against \`${baseReport.metadata.gitSha}\` (base). ` +
277 `Write times are per-write medians over ${candidateReport.metadata.iterations} iterations ` +
278 `(${candidateReport.metadata.warmup} warmup writes). Cold hydrate is one timed first load of each query.`,
279 )
280 lines.push(``)
281 const coldSummary =
282 coldComparisons.length > 0
283 ? ` · cold hydrate time: **${formatRatio(
284 geomeanRatio(coldComparisons, coldMetric),
285 )}**`
286 : ` · cold hydrate time: n/a (a report has no cold data)`
287 lines.push(
288 `Overall median write time vs base: **${formatRatio(
289 geomeanRatio(allComparisons),
290 )}**${coldSummary} (geometric mean of per-case ratios; lower is faster).`,
291 )
292 lines.push(``)
293 lines.push(
294 formatFlagSummary(`Writes`, allComparisons, threshold, writeMetric),
295 )
296 if (coldComparisons.length > 0) {
297 lines.push(``)
298 lines.push(
299 formatFlagSummary(`Cold hydrate`, coldComparisons, threshold, coldMetric),
300 )
301 }
302 lines.push(``)
303 lines.push(
304 `<sub>Per-case flags are noisy on shared runners. Read the geometric means first.</sub>`,
305 )
306 lines.push(``)
307​
308 const byQuery = new Map<string, Array<Comparison>>()
309 for (const comparison of allComparisons) {
310 const group = byQuery.get(comparison.query) ?? []
311 group.push(comparison)
312 byQuery.set(comparison.query, group)
313 }
314​
315 lines.push(`### Writes`)
316 lines.push(``)
317 pushQueryRollup(lines, byQuery, threshold, writeMetric)
318 lines.push(``)
319 if (coldComparisons.length > 0) {
320 lines.push(`### Cold hydrate`)
321 lines.push(``)
322 pushQueryRollup(lines, byQuery, threshold, coldMetric)
323 lines.push(``)
324 }
325 lines.push(
326 `Each row aggregates the ${allComparisons.length / byQuery.size || 0} ` +
327 `scale/index/write-mode configurations of that query; per-configuration ` +
328 `tables below.`,
329 )
330 lines.push(``)
331​
332 const groups = new Map<string, Array<Comparison>>()
333 for (const comparison of allComparisons) {
334 const groupKey = `${formatScale(comparison.scale)} | source indexes: ${
335 comparison.sourceIndexMode
336 } | ${comparison.mutationMode} writes`
337 const group = groups.get(groupKey) ?? []
338 group.push(comparison)
339 groups.set(groupKey, group)
340 }
341​
342 for (const [groupKey, group] of groups) {
343 lines.push(`<details>`)
344 const changed = group.filter(
345 (comparison) =>
346 marker(comparison, threshold) !== `` ||
347 marker(comparison, threshold, coldMetric) !== ``,
348 ).length
349 const changedSuffix = changed > 0 ? `, ${changed} change(s)` : ``
350 const coldSuffix = group.some(
351 (comparison) => metricRatio(comparison, coldMetric) !== undefined,
352 )
353 ? `, cold ${formatRatio(geomeanRatio(group, coldMetric))}`
354 : ``
355 lines.push(
356 `<summary><b>${groupKey}</b> — geomean ${formatRatio(
357 geomeanRatio(group),
358 )}${coldSuffix}${changedSuffix}</summary>`,
359 )
360 lines.push(``)
361 lines.push(
362 `| Query | Base median | PR median | Δ median | Base p95 | PR p95 | Δ p95 | | Base cold | PR cold | Δ cold | |`,
363 )
364 lines.push(
365 `| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | --- |`,
366 )
367 for (const comparison of group) {
368 lines.push(
369 `| ${comparison.query} | ${formatMs(
370 comparison.base.writeSummary.medianMs,
371 )} | ${formatMs(comparison.candidate.writeSummary.medianMs)} | ${formatDelta(
372 comparison.medianRatio,
373 )} | ${formatMs(comparison.base.writeSummary.p95Ms)} | ${formatMs(
374 comparison.candidate.writeSummary.p95Ms,
375 )} | ${formatDelta(comparison.p95Ratio)} | ${marker(
376 comparison,
377 threshold,
378 )} | ${formatOptionalMs(comparison.base.coldHydrateMs)} | ${formatOptionalMs(
379 comparison.candidate.coldHydrateMs,
380 )} | ${formatDelta(
381 metricRatio(comparison, coldMetric) ?? Number.NaN,
382 )} | ${marker(comparison, threshold, coldMetric)} |`,
383 )
384 }
385 lines.push(``)
386 lines.push(`</details>`)
387 lines.push(``)
388 }
389​
390 lines.push(
391 `<sub>Runner: node ${candidate.metadata.node}, ${candidate.metadata.platform}, ${candidate.metadata.cpu}. ` +
392 `Timings on shared CI runners are noisy; treat small deltas as indicative only.</sub>`,
393 )
394​
395 return lines.join(`\n`)
397​
398function formatFlagSummary(
399 label: string,
400 group: Array<Comparison>,
401 threshold: number,
402 metric: Metric,
403): string {
404 const regressed = group.filter((comparison) =>
405 isRegression(comparison, threshold, metric),
406 ).length
407 const improved = group.filter((comparison) =>
408 isImprovement(comparison, threshold, metric),
409 ).length
410 const relative = Math.max(threshold, metric.minRelativeThreshold)
411 const rule = `threshold: ±${Math.round(relative * 100)}% and >${metric.floorMs}ms`
412 if (regressed === 0 && improved === 0) {
413 return `${label}: **No significant changes** (${rule}).`
414 }
415 return `${label}: **${regressed} regression(s), ${improved} improvement(s)** (${rule}).`
417​
418function pushQueryRollup(
419 lines: Array<string>,
420 byQuery: Map<string, Array<Comparison>>,
421 threshold: number,
422 metric: Metric,
423): void {
424 lines.push(
425 `| Query | Δ ${metric.label} (geomean) | Best case | Worst case | Flags |`,
426 )
427 lines.push(`| --- | ---: | ---: | ---: | :-- |`)
428 for (const [query, group] of byQuery) {
429 const ratios = group
430 .map((comparison) => metricRatio(comparison, metric))
431 .filter(
432 (value): value is number =>
433 value !== undefined && Number.isFinite(value),
434 )
435 if (ratios.length === 0) continue
436 const redCount = group.filter((comparison) =>
437 isRegression(comparison, threshold, metric),
438 ).length
439 const greenCount = group.filter((comparison) =>
440 isImprovement(comparison, threshold, metric),
441 ).length
442 const flags =
443 [
444 redCount > 0 ? `${redCount} 🔴` : ``,
445 greenCount > 0 ? `${greenCount} 🟢` : ``,
446 ]
447 .filter(Boolean)
448 .join(` `) || `—`
449 lines.push(
450 `| ${query} | ${formatDelta(geomeanRatio(group, metric))} | ${formatDelta(
451 Math.min(...ratios),
452 )} | ${formatDelta(Math.max(...ratios))} | ${flags} |`,
453 )
454 }
456​
457function formatOptionalMs(value: number | undefined): string {
458 return value === undefined ? `n/a` : formatMs(value)
460​
461function formatScale(scale: FixtureScale): string {
462 if (
463 scale.issueCount === scale.userCount &&
464 scale.userCount === scale.commentCount
465 ) {
466 return `${scale.issueCount.toLocaleString(`en-US`)} rows/collection`
467 }
468 return `issues:${scale.issueCount} users:${scale.userCount} comments:${scale.commentCount}`
470​
471function formatMs(value: number): string {
472 return `${value.toFixed(3)}ms`
474​
475function formatDelta(value: number): string {
476 if (!Number.isFinite(value)) return `n/a`
477 const percent = (value - 1) * 100
478 const sign = percent > 0 ? `+` : ``
479 return `${sign}${percent.toFixed(1)}%`
481​
482function parseArgs(argv: Array<string>): {
483 base: string
484 candidate: string
485 outFile?: string
486 threshold: number
487 failOnRegression: boolean
488} {
489 let basePath: string | undefined
490 let candidatePath: string | undefined
491 let outFile: string | undefined
492 let threshold = defaultThreshold
493 let failOnRegression = false
494​
495 for (const arg of argv) {
496 const [name, value] = arg.replace(/^--/, ``).split(`=`)
497 if (!name || value === undefined) continue
498​
499 switch (name) {
500 case `base`:
501 basePath = value
502 break
503 case `candidate`:
504 candidatePath = value
505 break
506 case `outFile`:
507 outFile = value
508 break
509 case `threshold`: {
510 const parsed = Number(value)
511 if (!Number.isFinite(parsed) || parsed <= 0) {
512 throw new Error(`--threshold must be a positive number`)
513 }
514 threshold = parsed
515 break
516 }
517 case `failOnRegression`:
518 failOnRegression = value === `true`
519 break
520 }
521 }
522​
523 if (!basePath || !candidatePath) {
524 throw new Error(`--base and --candidate report paths are required`)
525 }
526​
527 return {
528 base: basePath,
529 candidate: candidatePath,
530 outFile,
531 threshold,
532 failOnRegression,
533 }