Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

chart.mjs

195 lines6.4 KB

1// Render the findings as PNG bar charts. Headless Chromium (Playwright, already
2// in the repo) draws an HTML chart and screenshots it -- no new deps, real color
3// and fonts (ImageMagick's SVG path mangles both here).
4//
5// node chart.mjs # writes results/charts/*.png from results/*.jsonl
6//
7// Charts:
8// A terminal-bench input tokens per arm (the Strands-regime replication)
9// B optimize() effect vs baseline across three workloads (the honest story)
10// C code mode vs tool-calls input (code mode is the big lever)
11​
12import { readFileSync, mkdirSync } from 'node:fs'
13import path from 'node:path'
14import { fileURLToPath } from 'node:url'
15// playwright-core is a declared devDependency of this package; import by name so
16// the path survives a lockfile bump (no hardcoded pnpm store directory).
17import pwDefault from 'playwright-core'
18​
19const HERE = path.dirname(fileURLToPath(import.meta.url))
20​
21function avgByArm(file) {
22 const rows = readFileSync(path.join(HERE, file), 'utf8')
23 .trim()
24 .split('\n')
25 .map((l) => JSON.parse(l))
26 const by = {}
27 for (const r of rows) (by[r.arm] ??= []).push(r)
28 const out = {}
29 for (const [arm, g] of Object.entries(by)) {
30 out[arm] = {
31 input: Math.round(
32 g.reduce((s, r) => s + (r.tokens?.input ?? 0), 0) / g.length,
33 ),
34 solved: g.filter((r) => r.outcome === 'solved').length,
35 trials: g.length,
36 }
37 }
38 return out
40​
41// --- HTML chart template (horizontal bars) ---------------------------------
42function barChartHTML(
43 title,
44 subtitle,
45 rows,
46 { unit = 'input tokens / task', max } = {},
47) {
48 const top = max ?? Math.max(...rows.map((r) => Math.abs(r.value)))
49 const bars = rows
50 .map((r) => {
51 const w = Math.max(0.4, (Math.abs(r.value) / top) * 100)
52 const label =
53 r.value >= 1000 ? `${(r.value / 1000).toFixed(0)}k` : `${r.value}`
54 return `
55 <div class="row">
56 <div class="name">${r.label}</div>
57 <div class="track"><div class="bar" style="width:${w}%;background:${r.color}"></div>
58 <div class="val">${label}${r.note ? ` <span class="note">${r.note}</span>` : ''}</div>
59 </div>
60 </div>`
61 })
62 .join('')
63 return `<!doctype html><meta charset="utf8">
64 <style>
65 body{margin:0;font:14px -apple-system,Helvetica,Arial,sans-serif;color:#0f172a;background:#fff}
66 .card{width:760px;padding:28px 32px}
67 h1{font-size:19px;margin:0 0 2px}
68 .sub{color:#64748b;font-size:13px;margin:0 0 20px}
69 .row{display:flex;align-items:center;margin:9px 0}
70 .name{width:150px;font-weight:600;font-size:13px;text-align:right;padding-right:14px;color:#334155}
71 .track{position:relative;flex:1;display:flex;align-items:center}
72 .bar{height:26px;border-radius:5px}
73 .val{margin-left:10px;font-variant-numeric:tabular-nums;font-weight:600;white-space:nowrap}
74 .note{color:#64748b;font-weight:500;font-size:12px}
75 .unit{color:#94a3b8;font-size:12px;margin-top:16px}
76 </style>
77 <div class="card"><h1>${title}</h1><p class="sub">${subtitle}</p>${bars}
78 <div class="unit">${unit} · gemini-3.8-flash</div></div>`
80​
81async function render(browser, html, outPath) {
82 const pg = await browser.newPage({
83 viewport: { width: 760, height: 10 },
84 deviceScaleFactor: 2,
85 })
86 await pg.setContent(html, { waitUntil: 'networkidle' })
87 const el = await pg.$('.card')
88 await el.screenshot({ path: outPath })
89 await pg.close()
91​
92const BLUE = '#2563eb',
93 GRAY = '#94a3b8',
94 GREEN = '#16a34a',
95 RED = '#dc2626',
96 AMBER = '#d97706'
97​
98async function main() {
99 const outDir = path.join(HERE, 'results', 'charts')
100 mkdirSync(outDir, { recursive: true })
101​
102 const tb = avgByArm('results/terminal-bench.jsonl')
103 const mw = avgByArm('results/multi-write.jsonl')
104 const cm = avgByArm('results/code-mode.jsonl')
105​
106 const pw = pwDefault
107 const browser = await pw.chromium.launch()
108​
109 // A. terminal-bench per-arm input (the replication)
110 await render(
111 browser,
112 barChartHTML(
113 'Disposable shell logs: size-based truncation replicates the Strands win',
114 'Terminal-bench-style suite. Answer is a summary line in the tail, so clipping is nearly free.',
115 [
116 { label: 'baseline', value: tb.baseline.input, color: GRAY },
117 { label: 'global (Strands)', value: tb.global.input, color: BLUE },
118 { label: 'universal', value: tb.universal.input, color: BLUE },
119 { label: 'adorned', value: tb.adorned.input, color: GREEN },
120 ],
121 ),
122 path.join(outDir, 'terminal-bench.png'),
123 )
124​
125 // B. the honest story: optimize effect flips sign with the workload
126 const pct = (base, opt) => Math.round(((opt - base) / base) * 100)
127 const tbPct = pct(tb.baseline.input, tb.adorned.input)
128 const cmPct = pct(cm['cm-baseline'].input, cm['cm-adorned'].input)
129 const mwPct = pct(mw.baseline.input, mw.adorned.input)
130 await render(
131 browser,
132 barChartHTML(
133 'optimize() helps or hurts depending on the workload',
134 'Change in input tokens vs baseline, best adornment per regime. Negative = cheaper. The sign flips.',
135 [
136 {
137 label: 'shell logs (disposable)',
138 value: tbPct,
139 color: GREEN,
140 note: '← huge win',
141 },
142 {
143 label: 'code-mode logs',
144 value: cmPct,
145 color: GREEN,
146 note: '← huge win',
147 },
148 {
149 label: 'file edits (load-bearing)',
150 value: mwPct,
151 color: RED,
152 note: '← backfires',
153 },
154 ],
155 {
156 unit: '% change in input tokens vs baseline',
157 max: Math.max(100, Math.abs(tbPct), Math.abs(cmPct), Math.abs(mwPct)),
158 },
159 ),
160 path.join(outDir, 'effect-by-workload.png'),
161 )
162​
163 // C. code mode vs tool calls on the same multi-file edit
164 await render(
165 browser,
166 barChartHTML(
167 'Code mode is the bigger lever than truncation',
168 'Same multi-file edit. Code mode keeps file contents in the sandbox, out of the model context.',
169 [
170 { label: 'tool-calls baseline', value: mw.baseline.input, color: GRAY },
171 {
172 label: 'tool-calls adorned',
173 value: mw.adorned.input,
174 color: RED,
175 note: '← truncation backfired',
176 },
177 {
178 label: 'code mode',
179 value: mw['cm-baseline'].input,
180 color: GREEN,
181 note: '← ~10x cheaper',
182 },
183 ],
184 ),
185 path.join(outDir, 'code-mode-vs-tools.png'),
186 )
187​
188 await browser.close()
189 console.log('wrote', outDir)
191​
192main().catch((e) => {
193 console.error(e)
194 process.exit(1)
195})