Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

generate.mjs

115 lines4.1 KB

1// Code-mode suite: the agent works entirely through execute_typescript. Code
2// mode already keeps file contents in the sandbox, so the model context is tiny
3// -- that's code mode's own win. This suite isolates what optimize() adds ON TOP
4// of code mode: the `logs`. Real agents console.log liberally to debug ("let me
5// see what I'm working with"), and those dumps are pure noise once the step is
6// done. The task tells the agent to log each file's content as it processes it;
7// `cm-adorned` clips those logs (head_tail), `cm-baseline` carries every dump
8// for the rest of the run. The returned count stays whole, so no accuracy loss.
9//
10// node generate.mjs --count 3 --files 3 --out ./out [--seed 42]
11//
12// Grading: final line `REPLACEMENTS=<total>`.
13​
14import { mkdirSync, writeFileSync } from 'node:fs'
15import path from 'node:path'
16​
17function arg(flag, dflt) {
18 const i = process.argv.indexOf(flag)
19 return i === -1 ? dflt : process.argv[i + 1]
21​
22function mulberry32(seed) {
23 let a = seed >>> 0
24 return () => {
25 a |= 0
26 a = (a + 0x6d2b79f5) | 0
27 let t = Math.imul(a ^ (a >>> 15), 1 | a)
28 t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
29 return ((t ^ (t >>> 14)) >>> 0) / 4294967296
30 }
32​
33function sourceLine(rng, i, carriesToken) {
34 const fns = [
35 'handleRequest',
36 'parseConfig',
37 'buildIndex',
38 'flushQueue',
39 'retryWithBackoff',
40 ]
41 const fn = fns[Math.floor(rng() * fns.length)]
42 if (carriesToken)
43 return ` const client = new ACME_OLD(config.endpoint); // line ${i}, wires ${fn}`
44 return ` // ${fn}: step ${i} handled in ${Math.floor(rng() * 500)}ms, retries=${Math.floor(rng() * 4)}`
46​
47function makeSource(rng, outDir, tag) {
48 const totalLines = 200 + Math.floor(rng() * 160)
49 const k = 10 + Math.floor(rng() * 16)
50 const positions = new Set()
51 while (positions.size < k) positions.add(Math.floor(rng() * totalLines))
52 const lines = []
53 for (let i = 0; i < totalLines; i++)
54 lines.push(sourceLine(rng, i, positions.has(i)))
55 const srcPath = path.join(outDir, `cm-${tag}-src.txt`)
56 const outPath = path.join(outDir, `cm-${tag}-out.txt`)
57 writeFileSync(srcPath, lines.join('\n') + '\n')
58 return { srcPath: path.resolve(srcPath), outPath: path.resolve(outPath), k }
60​
61function makeTask(n, files, rng, outDir) {
62 const id = `cm-${String(n).padStart(3, '0')}`
63 const specs = []
64 for (let f = 0; f < files; f++)
65 specs.push(makeSource(rng, outDir, `${String(n).padStart(3, '0')}-${f}`))
66 const total = specs.reduce((s, x) => s + x.k, 0)
67 const fileList = specs
68 .map(
69 (s, i) => ` ${i + 1}. read ${s.srcPath}, write result to ${s.outPath}`,
70 )
71 .join('\n')
72​
73 const prompt =
74 `Process these ${files} files. For EACH file: read it, replace every occurrence of "ACME_OLD" ` +
75 `with "ACME_NEW", and write the COMPLETE result to its output path:\n${fileList}\n` +
76 `IMPORTANT: for debugging, console.log the FULL content of each file (both before and after the ` +
77 `replacement) as you process it, so the work is auditable. When all files are done, output exactly ` +
78 `one line and nothing else: REPLACEMENTS=<n>, the TOTAL replacements across all files.`
79​
80 return {
81 id,
82 suite: 'code-mode',
83 prompt,
84 cwd: path.resolve(outDir),
85 answer: `REPLACEMENTS=${total}`,
86 disableShell: true,
87 codeModeOnly: true, // meaningful only under code-mode arms
88 files: specs.map((s) => ({ src: s.srcPath, out: s.outPath, k: s.k })),
89 replacements: total,
90 }
92​
93function main() {
94 const count = Number(arg('--count', '3'))
95 const files = Number(arg('--files', '3'))
96 const outDir = arg(
97 '--out',
98 path.join(path.dirname(new URL(import.meta.url).pathname), 'out'),
99 )
100 const seed = Number(arg('--seed', '77'))
101 mkdirSync(outDir, { recursive: true })
102 const rng = mulberry32(seed)
103 for (let n = 0; n < count; n++) {
104 const task = makeTask(n, files, rng, outDir)
105 writeFileSync(
106 path.join(outDir, `task-${String(n).padStart(3, '0')}.json`),
107 JSON.stringify(task, null, 2) + '\n',
108 )
109 }
110 console.log(
111 `Wrote ${count} code-mode tasks (${files} files each) to ${outDir} (seed ${seed}).`,
112 )
114​
115main()