Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

generate.mjs

115 lines3.8 KB

1// Agentic-coding suite: read a large file, transform it, write a large file.
2// This exercises the realistic coding loop the feature targets:
3// - read_file content is load-bearing while editing -> keep it whole
4// - write_file content is redundant once on disk -> drop the completed call's
5// args from history (D3), AFTER the write has gone back to the model.
6//
7// node generate.mjs --count 3 --out ./out [--seed 42]
8//
9// Each task writes:
10// out/source-NNN.txt a large source file with K occurrences of ACME_OLD
11// out/task-NNN.json { id, suite, prompt, cwd, answer, disableShell, ... }
12//
13// Grading: the agent must report `REPLACEMENTS=<K>` where K is the exact count,
14// which requires seeing the WHOLE file. An arm that truncates the read result
15// (global / universal) loses middle occurrences and reports the wrong count.
16​
17import { mkdirSync, writeFileSync } from 'node:fs'
18import path from 'node:path'
19​
20function arg(flag, dflt) {
21 const i = process.argv.indexOf(flag)
22 return i === -1 ? dflt : process.argv[i + 1]
24​
25function mulberry32(seed) {
26 let a = seed >>> 0
27 return () => {
28 a |= 0
29 a = (a + 0x6d2b79f5) | 0
30 let t = Math.imul(a ^ (a >>> 15), 1 | a)
31 t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
32 return ((t ^ (t >>> 14)) >>> 0) / 4294967296
33 }
35​
36// A line of plausible source, occasionally carrying the token to replace.
37function sourceLine(rng, i, carriesToken) {
38 const fns = [
39 'handleRequest',
40 'parseConfig',
41 'buildIndex',
42 'flushQueue',
43 'retryWithBackoff',
44 ]
45 const fn = fns[Math.floor(rng() * fns.length)]
46 if (carriesToken) {
47 return ` const client = new ACME_OLD(config.endpoint); // line ${i}, wires ${fn}`
48 }
49 return ` // ${fn}: step ${i} handled in ${Math.floor(rng() * 500)}ms, retries=${Math.floor(rng() * 4)}`
51​
52function makeTask(n, rng, outDir) {
53 const id = `ac-${String(n).padStart(3, '0')}`
54 const totalLines = 260 + Math.floor(rng() * 220)
55 // Scatter K occurrences across the whole file (including the middle).
56 const k = 14 + Math.floor(rng() * 20)
57 const positions = new Set()
58 while (positions.size < k) positions.add(Math.floor(rng() * totalLines))
59​
60 const lines = []
61 for (let i = 0; i < totalLines; i++)
62 lines.push(sourceLine(rng, i, positions.has(i)))
63 const sourcePath = path.join(
64 outDir,
65 `source-${String(n).padStart(3, '0')}.txt`,
66 )
67 const outputPath = path.join(
68 outDir,
69 `output-${String(n).padStart(3, '0')}.txt`,
70 )
71 writeFileSync(sourcePath, lines.join('\n') + '\n')
72​
73 const prompt =
74 `Using ONLY the read_file and write_file tools (no shell), read the file at ` +
75 `${path.resolve(sourcePath)}. Replace every occurrence of the exact string ` +
76 `"ACME_OLD" with "ACME_NEW". Write the COMPLETE modified file to ` +
77 `${path.resolve(outputPath)}. Then, as your final message, output exactly one ` +
78 `line and nothing else: REPLACEMENTS=<n>, where <n> is how many replacements you made.`
79​
80 return {
81 id,
82 suite: 'agentic-coding',
83 prompt,
84 cwd: path.resolve(outDir),
85 answer: `REPLACEMENTS=${k}`,
86 disableShell: true, // force content through read_file / write_file, not grep/sed
87 replacements: k,
88 totalLines,
89 sourcePath: path.resolve(sourcePath),
90 outputPath: path.resolve(outputPath),
91 }
93​
94function main() {
95 const count = Number(arg('--count', '3'))
96 const outDir = arg(
97 '--out',
98 path.join(path.dirname(new URL(import.meta.url).pathname), 'out'),
99 )
100 const seed = Number(arg('--seed', '42'))
101 mkdirSync(outDir, { recursive: true })
102 const rng = mulberry32(seed)
103 for (let n = 0; n < count; n++) {
104 const task = makeTask(n, rng, outDir)
105 writeFileSync(
106 path.join(outDir, `task-${String(n).padStart(3, '0')}.json`),
107 JSON.stringify(task, null, 2) + '\n',
108 )
109 }
110 console.log(
111 `Wrote ${count} agentic-coding tasks + sources to ${outDir} (seed ${seed}).`,
112 )
114​
115main()