Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

run-matrix.mjs

155 lines5.4 KB

1// Matrix runner: suite x arms x tasks x trials -> one JSONL under results/.
2// Runs sequentially (arms share task output paths, and this is kinder to rate
3// limits). Resume-friendly: appends; re-running overwrites the named file.
4//
5// node run-matrix.mjs \
6// --tasks "suites/multi-write/out/task-*.json" \
7// --arms baseline,adorned,universal,global,cm-baseline,cm-adorned \
8// --model gemini-3.8-flash --trials 3 --out results/multi-write.jsonl
9//
10// Then: node report.mjs results/multi-write.jsonl
11​
12import {
13 readFileSync,
14 writeFileSync,
15 mkdirSync,
16 appendFileSync,
17 globSync,
18} from 'node:fs'
19import path from 'node:path'
20import { runTask } from './tanstack_ai_runner.mjs'
21import { DEFAULT_RATES } from './report.mjs'
22​
23function arg(flag, dflt) {
24 const i = process.argv.indexOf(flag)
25 return i === -1 ? dflt : process.argv[i + 1]
27function hasFlag(flag) {
28 return process.argv.includes(flag)
30​
31/** Est. USD for one trial from the rate card ($/MTok). null if the rate is unknown. */
32function trialCost(rec) {
33 const r = DEFAULT_RATES[rec.model]
34 if (!r) return null
35 return (rec.tokens.input / 1e6) * r.in + (rec.tokens.output / 1e6) * r.out
37​
38async function main() {
39 // minimal .env.local loader (mirror the runner; keys live at the repo root)
40 const { existsSync } = await import('node:fs')
41 let dir = process.cwd()
42 for (let i = 0; i < 6; i++) {
43 const p = path.join(dir, '.env.local')
44 if (existsSync(p)) {
45 for (const line of readFileSync(p, 'utf8').split('\n')) {
46 const m = /^([A-Z0-9_]+)=(.*)$/.exec(line.trim())
47 if (m && process.env[m[1]] === undefined)
48 process.env[m[1]] = m[2].replace(/^["']|["']$/g, '')
49 }
50 break
51 }
52 dir = path.dirname(dir)
53 }
54​
55 const taskGlob = arg('--tasks')
56 const arms = arg('--arms', 'baseline,adorned')
57 .split(',')
58 .map((s) => s.trim())
59 .filter(Boolean)
60 const model = arg('--model', 'gemini-3.8-flash')
61 const trials = Number(arg('--trials', '3'))
62 const outFile = arg('--out', 'results/matrix.jsonl')
63 // Hard spend cap in USD (estimated from the rate card). Stops before the next
64 // paid request once exceeded, keeping the partial results written so far.
65 const maxUsd = arg('--max-usd') ? Number(arg('--max-usd')) : null
66 if (!taskGlob) {
67 console.error(
68 'usage: node run-matrix.mjs --tasks "glob" --arms a,b,c --model M --trials N --out results/x.jsonl [--max-usd 5] [--fable]',
69 )
70 process.exit(1)
71 }
72 // Fable is the most expensive cell; require an explicit opt-in to spend on it.
73 if (/fable/i.test(model) && !hasFlag('--fable')) {
74 console.error(
75 `model "${model}" is a Fable cell; re-run with --fable to confirm the spend.`,
76 )
77 process.exit(1)
78 }
79 // A spend cap only works if we can price the model. With an unknown rate,
80 // trialCost() returns null, spentUsd never moves, and --max-usd silently
81 // stops nothing. Refuse to start rather than run paid trials uncapped.
82 if (maxUsd !== null && !DEFAULT_RATES[model]) {
83 console.error(
84 `--max-usd is set but model "${model}" has no rate in DEFAULT_RATES, ` +
85 `so the cap cannot be enforced. Add a rate for it in report.mjs, or ` +
86 `drop --max-usd to run uncapped.`,
87 )
88 process.exit(1)
89 }
90​
91 const taskFiles = globSync(taskGlob).sort()
92 if (taskFiles.length === 0) {
93 console.error(`no task files matched ${taskGlob}`)
94 process.exit(1)
95 }
96 mkdirSync(path.dirname(outFile), { recursive: true })
97 writeFileSync(outFile, '') // fresh file for this run
98​
99 const total = taskFiles.length * arms.length * trials
100 let done = 0
101 let spentUsd = 0
102 console.error(
103 `matrix: ${taskFiles.length} tasks x ${arms.length} arms x ${trials} trials = ${total} runs on ${model}` +
104 (maxUsd !== null ? ` (cap $${maxUsd})` : ''),
105 )
106​
107 for (const tf of taskFiles) {
108 const task = JSON.parse(readFileSync(tf, 'utf8'))
109 for (const arm of arms) {
110 for (let trial = 0; trial < trials; trial++) {
111 if (maxUsd !== null && spentUsd >= maxUsd) {
112 console.error(
113 `budget cap $${maxUsd} reached (~$${spentUsd.toFixed(2)} spent); stopping with ${done}/${total} runs done. Partial results are in ${outFile}.`,
114 )
115 console.error(`wrote ${outFile}`)
116 return
117 }
118 done += 1
119 const tag = `[${done}/${total}] ${task.id} ${arm} t${trial}`
120 try {
121 const rec = await runTask(task, { arm, model })
122 rec.trial = trial
123 const cost = trialCost(rec)
124 if (cost !== null) spentUsd += cost
125 appendFileSync(outFile, JSON.stringify(rec) + '\n')
126 console.error(
127 `${tag} -> ${rec.outcome} in=${rec.tokens.input} proj=${rec.truncation.projections} retr=${rec.truncation.retrievals}` +
128 (cost !== null ? ` ~$${spentUsd.toFixed(2)}` : ''),
129 )
130 } catch (err) {
131 const rec = {
132 taskId: task.id,
133 suite: task.suite,
134 arm,
135 model,
136 trial,
137 outcome: 'error',
138 failureClass: 'runner-error',
139 error: String(err?.message ?? err),
140 tokens: { input: 0, output: 0 },
141 truncation: { projections: 0, retrievals: 0 },
142 }
143 appendFileSync(outFile, JSON.stringify(rec) + '\n')
144 console.error(`${tag} -> ERROR ${rec.error}`)
145 }
146 }
147 }
148 }
149 console.error(`wrote ${outFile}`)
151​
152main().catch((err) => {
153 console.error(err)
154 process.exit(1)
155})