Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

tanstack_ai_runner.mjs

277 lines9.8 KB

1// E2 — the Node agent Harbor uploads and runs inside a task container, also
2// runnable standalone for a `--smoke` test. It builds a chat() agent whose
3// tools are a shell tool and a file-editor tool, selects an arm by env, runs
4// the task, and writes a result + events record with the full §7 token split.
5//
6// Arms are selected by env/config, nothing else changes. System prompt and
7// turn limits are held constant across arms.
8//
9// Usage:
10// ARM=adorned MODEL=claude-sonnet-4-6 node tanstack_ai_runner.mjs --smoke task.json
11// (Harbor path: reads the task from argv/stdin per the BaseInstalledAgent
12// contract — see tanstack_ai.py — and writes result.json + events.jsonl.)
13​
14import { readFile } from 'node:fs/promises'
15import { existsSync, readFileSync } from 'node:fs'
16import path from 'node:path'
17import { pathToFileURL } from 'node:url'
18import { chat, maxIterations } from '@tanstack/ai'
19import { optimize } from '@tanstack/ai-optimize'
20import { createCodeMode } from '@tanstack/ai-code-mode'
21import { createQuickJSIsolateDriver } from '@tanstack/ai-isolate-quickjs'
22import { armSpec } from './arms.mjs'
23import { makeTools } from './tools.mjs'
24​
25const SYSTEM_PROMPT =
26 'You are a precise command-line agent. Use the tools to inspect files and run ' +
27 'commands. When you have the answer, state it plainly as the final message. ' +
28 'If a tool result shows a "[Truncated for context: ...]" or "[Dropped for ' +
29 'context: ...]" marker and you need the elided part, call retrieve_context ' +
30 'with the ref in the marker.'
31​
32const TURN_LIMIT = Number(process.env.TURN_LIMIT ?? 30)
33​
34// --- minimal .env.local loader (keys live at the repo root) ----------------
35function loadEnvLocal() {
36 // walk up to find .env.local
37 let dir = process.cwd()
38 for (let i = 0; i < 6; i++) {
39 const p = path.join(dir, '.env.local')
40 if (existsSync(p)) {
41 for (const line of readFileSync(p, 'utf8').split('\n')) {
42 const m = /^([A-Z0-9_]+)=(.*)$/.exec(line.trim())
43 if (m && process.env[m[1]] === undefined)
44 process.env[m[1]] = m[2].replace(/^["']|["']$/g, '')
45 }
46 return
47 }
48 dir = path.dirname(dir)
49 }
51​
52// --- adapter resolution (model ladder; never Fable-anchored) ---------------
53async function resolveAdapter(model) {
54 if (model.startsWith('gemini-')) {
55 // geminiText() reads GOOGLE_API_KEY, then GEMINI_API_KEY, from the env.
56 const { geminiText } = await import('@tanstack/ai-gemini')
57 return geminiText(model)
58 }
59 if (model.startsWith('gpt-') || model.startsWith('o')) {
60 const { openaiText } = await import('@tanstack/ai-openai')
61 return openaiText(model)
62 }
63 // claude-* and claude-fable-5 are served by the Anthropic adapter.
64 const { anthropicText } = await import('@tanstack/ai-anthropic')
65 return anthropicText(model)
67​
68// --- usage + event collection ----------------------------------------------
69function makeUsageMiddleware(collector) {
70 return {
71 name: 'eval-usage',
72 onIteration() {
73 collector.steps += 1
74 },
75 onUsage(_ctx, usage) {
76 collector.input += usage.inputTokens ?? usage.promptTokens ?? 0
77 collector.output += usage.outputTokens ?? usage.completionTokens ?? 0
78 collector.cacheRead +=
79 usage.cacheReadTokens ?? usage.cachedInputTokens ?? 0
80 collector.cacheWrite +=
81 usage.cacheWriteTokens ?? usage.cacheCreationTokens ?? 0
82 },
83 }
85​
86function buildAgent(arm, cwd, events, includeShell) {
87 const { codeMode, config } = armSpec(arm)
88 const systemPrompts = [SYSTEM_PROMPT]
89​
90 // In code mode the agent sees one tool, execute_typescript, that runs its
91 // program in a QuickJS sandbox; the file tools become external_* functions.
92 let agentTools
93 if (codeMode) {
94 const fileTools = makeTools(cwd, { includeShell: false })
95 const cm = createCodeMode({
96 driver: createQuickJSIsolateDriver(),
97 tools: fileTools,
98 })
99 systemPrompts.push(cm.systemPrompt)
100 agentTools = cm.tools
101 } else {
102 agentTools = makeTools(cwd, { includeShell })
103 }
104​
105 if (!config) return { tools: agentTools, middleware: [], systemPrompts }
106 const opt = optimize(agentTools, {
107 ...config,
108 onEvent: (e) => events.push(e),
109 })
110 return { tools: opt.tools, middleware: [opt.middleware], systemPrompts }
112​
113// --- grading + truncation-attributed failure classification ----------------
114function classify(task, finalText, events) {
115 const solved =
116 typeof task.answer === 'string' &&
117 finalText.toLowerCase().includes(task.answer.toLowerCase())
118 if (solved) return { solved: true, failureClass: null }
119​
120 const retrievals = events.filter((e) => e.type === 'retrieved')
121 const projections = events.filter((e) => e.type === 'projected')
122 // Heuristic classification (the brief's named failure metric):
123 // - missing-middle: a field was truncated, the needle was likely in it, and
124 // it was never retrieved.
125 // - failed-retrieval: retrieval happened but the task still failed.
126 let failureClass = 'other'
127 if (projections.length > 0 && retrievals.length === 0)
128 failureClass = 'missing-middle'
129 else if (retrievals.length > 0) failureClass = 'failed-retrieval'
130 return { solved: false, failureClass }
132​
133export async function runTask(task, { arm, model }) {
134 const cwd = task.cwd ?? process.cwd()
135 const events = []
136 const usage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, steps: 0 }
137 const { tools, middleware, systemPrompts } = buildAgent(
138 arm,
139 cwd,
140 events,
141 task.disableShell !== true,
142 )
143 const adapter = await resolveAdapter(model)
144​
145 let finalText = ''
146 // Wall-clock: total time, and time-to-first-token (round-trip latency floor).
147 const startedAt = performance.now()
148 let firstTokenMs
149 const stream = chat({
150 adapter,
151 model,
152 systemPrompts,
153 messages: [{ role: 'user', content: task.prompt }],
154 threadId: `eval-${task.id}`,
155 tools,
156 middleware: [makeUsageMiddleware(usage), ...middleware],
157 agentLoopStrategy: maxIterations(TURN_LIMIT),
158 })
159​
160 // Catch a mid-stream failure so the record still carries the tokens the run
161 // consumed before it threw (the usage middleware has already counted them).
162 let runError
163 try {
164 for await (const chunk of stream) {
165 // chat() can end the stream with a RUN_ERROR chunk instead of throwing.
166 // Treat it as an error so the run is graded 'error', not from partial
167 // text. Tokens are already counted by the usage middleware.
168 if (chunk.type === 'RUN_ERROR') {
169 runError ??= new Error(chunk.message || 'Run failed')
170 continue
171 }
172 // Time-to-first-token = first *content* token, not the first stream
173 // event. A TEXT_MESSAGE_START (or any non-content chunk) before content
174 // would otherwise report stream-event latency as token latency.
175 if (
176 firstTokenMs === undefined &&
177 chunk.type === 'TEXT_MESSAGE_CONTENT' &&
178 typeof chunk.delta === 'string' &&
179 chunk.delta.length > 0
180 )
181 firstTokenMs = performance.now() - startedAt
182 // Grade only the LAST assistant message: reset at each new text message so
183 // an answer mentioned before a tool call doesn't count as the final answer.
184 if (chunk.type === 'TEXT_MESSAGE_START') finalText = ''
185 if (
186 chunk.type === 'TEXT_MESSAGE_CONTENT' &&
187 typeof chunk.delta === 'string'
188 )
189 finalText += chunk.delta
190 }
191 } catch (err) {
192 runError = err
193 }
194 const durationMs = performance.now() - startedAt
195​
196 const { solved, failureClass } = runError
197 ? { solved: false, failureClass: 'runner-error' }
198 : classify(task, finalText, events)
199 const cacheHitRate = usage.input > 0 ? usage.cacheRead / usage.input : 0
200​
201 return {
202 taskId: task.id,
203 suite: task.suite ?? 'unknown',
204 arm,
205 model,
206 implementation: {
207 language: 'typescript',
208 package: '@tanstack/ai-optimize',
209 version: packageVersion(),
210 },
211 outcome: runError ? 'error' : solved ? 'solved' : 'failed',
212 error: runError ? String(runError?.message ?? runError) : undefined,
213 failureClass,
214 tokens: {
215 input: usage.input,
216 output: usage.output,
217 cache_read: usage.cacheRead,
218 cache_write: usage.cacheWrite,
219 steps: usage.steps,
220 cache_hit_rate: Number(cacheHitRate.toFixed(4)),
221 },
222 timing: {
223 duration_ms: Math.round(durationMs),
224 first_token_ms:
225 firstTokenMs === undefined ? null : Math.round(firstTokenMs),
226 ms_per_step:
227 usage.steps > 0 ? Math.round(durationMs / usage.steps) : null,
228 },
229 truncation: {
230 projections: events.filter((e) => e.type === 'projected').length,
231 retrievals: events.filter((e) => e.type === 'retrieved').length,
232 },
233 }
235​
236function packageVersion() {
237 try {
238 const p = new URL(
239 '../../packages/ai-optimize/package.json',
240 import.meta.url,
241 )
242 return JSON.parse(readFileSync(p, 'utf8')).version
243 } catch {
244 return 'unknown'
245 }
247​
248async function main() {
249 loadEnvLocal()
250 const arm = process.env.ARM ?? 'baseline'
251 const model = process.env.MODEL ?? 'claude-sonnet-4-6'
252 const smokeIdx = process.argv.indexOf('--smoke')
253 if (smokeIdx === -1) {
254 // TODO(harbor): read the task from Harbor's container contract (argv/stdin)
255 // and write the trajectory files Harbor converts. See tanstack_ai.py.
256 console.error(
257 'Harbor container mode not wired yet. Use --smoke <task.json> for a local run.',
258 )
259 process.exit(2)
260 }
261 const taskPath = process.argv[smokeIdx + 1]
262 const task = JSON.parse(await readFile(taskPath, 'utf8'))
263 const record = await runTask(task, { arm, model })
264 // One line of per-trial JSONL to stdout (the §7 split).
265 process.stdout.write(JSON.stringify(record) + '\n')
267​
268// Only run the CLI when invoked directly (not when imported by run-matrix.mjs).
269if (
270 process.argv[1] &&
271 import.meta.url === pathToFileURL(process.argv[1]).href
272) {
273 main().catch((err) => {
274 console.error(err)
275 process.exit(1)
276 })