Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

run-eval.ts

963 lines30.0 KB

1/**
2 * Code-mode models eval: run each configured model on the database-demo gold prompt,
3 * then judge vs gold report (Anthropic Opus + structured scores).
4 *
5 * Usage (from repo root):
6 * pnpm --filter @tanstack/ai-code-mode-models-eval eval
7 * pnpm --filter @tanstack/ai-code-mode-models-eval eval -- --ollama-only
8 * pnpm --filter @tanstack/ai-code-mode-models-eval eval -- --models openai:gpt-4o,groq:qwen/qwen3-32b
9 * cd packages/ai-code-mode/models-eval && pnpm eval -- --no-judge
10 *
11 * Env:
12 * ANTHROPIC_API_KEY — required for judging (unless --no-judge); auto-loaded from
13 * examples/ts-code-mode-web/.env.local (or repo-root .env.local) if present
14 * OPENAI_API_KEY — for OpenAI eval models
15 * GOOGLE_GENERATIVE_AI_API_KEY — for Gemini eval models
16 * XAI_API_KEY — for Grok (xAI) eval models
17 * GROQ_API_KEY — for Groq eval models
18 *
19 * Each run log JSON under models-eval/log/ includes `messages` (full UIMessage[] from
20 * StreamProcessor: user prompt, assistant text, tool-call parts, tool results).
21 */
22​
23import {
24 existsSync,
25 mkdirSync,
26 readFileSync,
27 readdirSync,
28 writeFileSync,
29} from 'node:fs'
30import { dirname, join, relative } from 'node:path'
31import { fileURLToPath } from 'node:url'
32​
33import { config as loadDotenv } from 'dotenv'
34​
35import {
36 StreamProcessor,
37 chat,
38 generateMessageId,
39 maxIterations,
40 normalizeToUIMessage,
41} from '@tanstack/ai'
42import { anthropicText } from '@tanstack/ai-anthropic'
43import { geminiText } from '@tanstack/ai-gemini'
44import { grokText } from '@tanstack/ai-grok'
45import { groqText } from '@tanstack/ai-groq'
46import { createNodeIsolateDriver } from '@tanstack/ai-isolate-node'
47import { ollamaText } from '@tanstack/ai-ollama'
48import { openaiText } from '@tanstack/ai-openai'
49import { createCodeMode } from '@tanstack/ai-code-mode'
50​
51import { databaseTools, getSchemaInfoTool } from './database-tools'
52import { EVAL_MODELS, getModelCategory, parseModelId } from './eval-config'
53import { judgeReports } from './judge'
54import {
55 computeMetrics,
56 computeStarRating,
57 formatTypescriptEvidence,
58} from './metrics'
59import type { JudgeResult } from './judge'
60import type { AnyTextAdapter, StreamChunk, UIMessage } from '@tanstack/ai'
61import type { EvalProvider, ModelCategory } from './eval-config'
62​
63const __dirname = dirname(fileURLToPath(import.meta.url))
64​
65/** Per-run JSON logs (gitignored); inspect candidate text vs gold when scores look wrong. */
66const LOG_DIR = join(__dirname, 'log')
67​
68const repoRoot = join(__dirname, '../../../')
69​
70function loadEnvLocalFiles() {
71 const paths = [
72 join(repoRoot, 'examples/ts-code-mode-web/.env.local'),
73 join(repoRoot, 'examples/ts-code-mode-web/.env'),
74 join(repoRoot, '.env.local'),
75 join(repoRoot, '.env'),
76 join(repoRoot, 'packages/ai-code-mode/.env.local'),
77 join(__dirname, '.env.local'),
78 ]
79 for (const path of paths) {
80 if (existsSync(path)) {
81 loadDotenv({ path, override: false })
82 }
83 }
85​
86loadEnvLocalFiles()
87​
88/**
89 * Code-mode–accurate copy: top-level tools are only getSchemaInfo + execute_typescript.
90 * The web demo's direct-tools prompt incorrectly names `queryTable` as a top-level tool;
91 * in code mode, querying happens via external_queryTable *inside* execute_typescript.
92 */
93const DATABASE_DEMO_SYSTEM_PROMPT = `You are a helpful data analyst assistant working with an in-memory database: tables **customers**, **products**, and **purchases**.
94​
95## Top-level tools (these are the only tools you can invoke)
96​
971. **getSchemaInfo** — Returns schema and row counts. Prefer **one call with no table argument** so you receive customers, products, and purchases together.
98​
992. **execute_typescript** — Runs TypeScript in a sandbox. Inside the sandbox you call **external_queryTable** and **external_getSchemaInfo** (same behavior as querying the DB). Use this for every multi-table join, filter, aggregation, and analysis.
100​
101## Tables
102​
103- **customers** — id, name, email, city, joined
104- **products** — id, name, category, price, stock
105- **purchases** — id, customer_id, product_id, quantity, total, purchased_at
106​
107**Joins:** \`purchases\` does **not** have \`category\`. Category is only on \`products\`. Load \`purchases\` and \`products\`, then join in code with \`product_id\` → \`products.id\` to know each purchase’s category. If you \`external_queryTable({ table: 'purchases', columns: ['category'] })\`, every row will be \`{}\` — that is wrong.
108​
109## Strategy
110​
111- Call getSchemaInfo once (omit \`table\`) unless you truly need a single table only.
112- Use **execute_typescript** with TypeScript that calls \`external_queryTable\` / \`external_getSchemaInfo\` to load rows, join across tables, and compute answers. Do not try to answer from schema alone.
113- After the script returns, write a clear final answer (tables, numbers, short summary).
114- If something is ambiguous, state assumptions briefly.
115​
116## Critical: \`external_queryTable\` (read this — common failures)
117​
118This is **not** SQL and **not** a raw string. You must pass a single **object** with a \`table\` field:
119​
120\`\`\`typescript
121const purchases = await external_queryTable({ table: 'purchases' })
122// Result shape: { rows: Array<Record<string, unknown>>, totalMatchingRows: number }
123for (const row of purchases.rows) { /* ... */ }
124\`\`\`
125​
126- Valid \`table\` values: \`customers\` | \`products\` | \`purchases\` only.
127- **Never** pass SQL, \`query:\`, \`sql:\`, or iterate the return value directly — use \`.rows\`.
128- \`external_getSchemaInfo\` takes \`{}\` for all tables or \`{ table: 'products' }\` for one table.
129​
130Put **only valid TypeScript** in \`typescriptCode\`: no markdown fences (\`\`\`), no \`**\` headings inside the code string.`
131​
132interface GoldMessagePart {
133 type: string
134 content?: string
136​
137interface GoldMessage {
138 role: string
139 parts?: Array<GoldMessagePart>
141​
142interface GoldFile {
143 prompt: string
144 messages: Array<GoldMessage>
146​
147function loadGoldFile(): GoldFile {
148 const path = join(__dirname, 'gold-results', 'db-gold-messages-1.json')
149 const raw = readFileSync(path, 'utf8')
150 return JSON.parse(raw) as GoldFile
152​
153function extractReportTextFromGold(messages: Array<GoldMessage>): string {
154 return messages
155 .filter((m) => m.role === 'assistant')
156 .flatMap((m) =>
157 (m.parts ?? [])
158 .filter((p) => p.type === 'text' && p.content)
159 .map((p) => p.content as string),
160 )
161 .join('\n\n')
163​
164/** Same text extraction as the web demo / gold file, from StreamProcessor UIMessages. */
165function extractAssistantReportText(messages: Array<UIMessage>): string {
166 return messages
167 .filter((m) => m.role === 'assistant')
168 .flatMap((m) =>
169 m.parts.filter((p) => p.type === 'text').map((p) => p.content),
170 )
171 .join('\n\n')
173​
174let codeModeCache: {
175 tool: ReturnType<typeof createCodeMode>['tool']
176 systemPrompt: string
177} | null = null
178​
179function getCodeModeTools() {
180 if (!codeModeCache) {
181 const driver = createNodeIsolateDriver()
182 const { tool, systemPrompt } = createCodeMode({
183 driver,
184 tools: databaseTools,
185 timeout: 60000,
186 memoryLimit: 128,
187 })
188 codeModeCache = { tool, systemPrompt }
189 }
190 return codeModeCache
192​
193function getTextAdapter(
194 provider: EvalProvider,
195 modelId: string,
196): AnyTextAdapter {
197 switch (provider) {
198 case 'ollama':
199 return ollamaText(modelId)
200 case 'openai':
201 return openaiText(modelId as 'gpt-4o-mini')
202 case 'anthropic':
203 return anthropicText(modelId as 'claude-haiku-4-5')
204 case 'gemini':
205 return geminiText(modelId as 'gemini-2.5-flash')
206 case 'grok':
207 return grokText(modelId as 'grok-build-0.1')
208 case 'groq':
209 return groqText(modelId as 'llama-3.3-70b-versatile')
210 }
212​
213/**
214 * The max-output-tokens cap lives in provider-native `modelOptions` keys (the
215 * root `maxTokens` field was removed from core `TextOptions`). The key name
216 * differs per provider. Ollama also keeps its larger context window via
217 * `num_ctx`.
218 */
219function maxTokensModelOptions(
220 provider: EvalProvider,
221 maxTokens: number,
222): Record<string, unknown> {
223 switch (provider) {
224 case 'ollama':
225 // Ollama reads sampling/runner params from the nested `options` key, not
226 // the top level of `modelOptions`. Flat keys here would be silently
227 // dropped at the wire.
228 return { options: { num_predict: maxTokens, num_ctx: 32768 } }
229 case 'openai':
230 case 'grok':
231 return { max_output_tokens: maxTokens }
232 case 'anthropic':
233 return { max_tokens: maxTokens }
234 case 'gemini':
235 return { maxOutputTokens: maxTokens }
236 case 'groq':
237 return { max_completion_tokens: maxTokens }
238 }
240​
241interface EvalRow {
242 name: string
243 /** Full `provider:model` id */
244 model: string
245 provider: EvalProvider
246 modelId: string
247 modelCategory: ModelCategory
248 durationMs: number
249 ttftMs?: number
250 promptTokens: number
251 completionTokens: number
252 totalTokens: number
253 tokenEfficiency?: number
254 speedTier: number
255 tokenEfficiencyTier: number
256 stabilityTier: number
257 stabilityRate: number
258 stabilitySampleSize: number
259 totalToolCalls: number
260 totalExecuteCalls: number
261 successfulExecuteCalls: number
262 compilationFailures: number
263 runtimeFailures: number
264 redundantSchemaChecks: number
265 stars?: 1 | 2 | 3
266 weightedScore?: number
267 error?: string
268 judge?: JudgeResult
270​
271function parseArgs(argv: Array<string>): {
272 ollamaOnly: boolean
273 noJudge: boolean
274 captureOnly: boolean
275 judgeLatest: boolean
276 rejudge: boolean
277 /** If set, only run entries whose \`model\` is in this list */
278 modelFilter: Set<string> | null
279} {
280 let ollamaOnly = false
281 let noJudge = false
282 let captureOnly = false
283 let judgeLatest = false
284 let rejudge = false
285 let modelFilter: Set<string> | null = null
286​
287 for (let i = 0; i < argv.length; i++) {
288 const a = argv[i]
289 if (a === '--ollama-only' || a === '--ollama') ollamaOnly = true
290 if (a === '--no-judge') noJudge = true
291 if (a === '--capture-only') captureOnly = true
292 if (a === '--judge-latest') judgeLatest = true
293 if (a === '--rejudge') rejudge = true
294 if (a === '--models' || a === '--model') {
295 const raw = argv[i + 1]
296 if (raw && !raw.startsWith('--')) {
297 i += 1
298 const ids = raw
299 .split(',')
300 .map((s) => s.trim())
301 .filter(Boolean)
302 modelFilter = new Set(ids)
303 }
304 }
305 if (a === '--help' || a === '-h') {
306 console.log(`
307@tanstack/ai-code-mode-models-eval (pnpm eval from this folder, or pnpm --filter @tanstack/ai-code-mode-models-eval eval)
308​
309 --ollama-only Only run Ollama models from eval-config
310 --models <id> Comma-separated provider:model ids (e.g. openai:gpt-4o,groq:qwen/qwen3-32b)
311 --capture-only Run models and write logs without LLM judging
312 --judge-latest Judge latest captured session from logs (no model runs)
313 --rejudge Re-judge logs even if judge fields already exist
314 --no-judge Skip LLM judging (no ANTHROPIC_API_KEY needed)
315 --help This message
316`)
317 process.exit(0)
318 }
319 }
320 return {
321 ollamaOnly,
322 noJudge,
323 captureOnly,
324 judgeLatest,
325 rejudge,
326 modelFilter,
327 }
329​
330function slugForFilename(model: string): string {
331 return model.replace(/[/\\:*?"<>|]/g, '-').replace(/\s+/g, '_')
333​
334interface RunLogPayload {
335 writtenAt: string
336 sessionStartedAt: string
337 entry: { name: string; model: string }
338 query: string
339 durationMs: number
340 ttftMs?: number
341 modelCategory: ModelCategory
342 promptTokens: number
343 completionTokens: number
344 totalTokens: number
345 tokenEfficiency?: number
346 speedTier: number
347 tokenEfficiencyTier: number
348 stabilityTier: number
349 stabilityRate: number
350 stabilitySampleSize: number
351 totalToolCalls: number
352 totalExecuteCalls: number
353 successfulExecuteCalls: number
354 compilationFailures: number
355 runtimeFailures: number
356 redundantSchemaChecks: number
357 stars?: 1 | 2 | 3
358 weightedScore?: number
359 candidateReportChars: number
360 candidateReportEmpty: boolean
361 goldReportChars: number
362 error?: string
363 judge?: JudgeResult
364 noJudge: boolean
365 goldReport: string
366 candidateReport: string
367 /** Full conversation as UIMessage[] (user + assistant turns, tool calls, tool results). */
368 messages: Array<UIMessage>
370​
371interface CanonicalResultsFile {
372 generatedAt: string
373 sessionStartedAt: string
374 methodology: {
375 benchmark: string
376 scoring: Array<string>
377 stability: string
378 starWeights: Record<string, number>
379 }
380 models: Array<EvalRow>
382​
383function writeRunLog(payload: RunLogPayload): string {
384 mkdirSync(LOG_DIR, { recursive: true })
385 const stamp = new Date().toISOString().replace(/[:.]/g, '-')
386 const fileName = `${stamp}_${slugForFilename(payload.entry.model)}.json`
387 const filePath = join(LOG_DIR, fileName)
388 writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8')
389 return filePath
391​
392function toEvalRow(payload: RunLogPayload): EvalRow {
393 const { provider, modelId } = parseModelId(payload.entry.model)
394 return {
395 name: payload.entry.name,
396 model: payload.entry.model,
397 provider,
398 modelId,
399 modelCategory: payload.modelCategory,
400 durationMs: payload.durationMs,
401 ttftMs: payload.ttftMs,
402 promptTokens: payload.promptTokens,
403 completionTokens: payload.completionTokens,
404 totalTokens: payload.totalTokens,
405 tokenEfficiency: payload.tokenEfficiency,
406 speedTier: payload.speedTier,
407 tokenEfficiencyTier: payload.tokenEfficiencyTier,
408 stabilityTier: payload.stabilityTier,
409 stabilityRate: payload.stabilityRate,
410 stabilitySampleSize: payload.stabilitySampleSize,
411 totalToolCalls: payload.totalToolCalls,
412 totalExecuteCalls: payload.totalExecuteCalls,
413 successfulExecuteCalls: payload.successfulExecuteCalls,
414 compilationFailures: payload.compilationFailures,
415 runtimeFailures: payload.runtimeFailures,
416 redundantSchemaChecks: payload.redundantSchemaChecks,
417 stars: payload.stars,
418 weightedScore: payload.weightedScore,
419 error: payload.error,
420 judge: payload.judge,
421 }
423​
424function isStableRun(payload: RunLogPayload): boolean {
425 if (payload.error) return false
426 if (payload.candidateReportEmpty) return false
427 return payload.successfulExecuteCalls > 0
429​
430function applyStability(rows: Array<EvalRow>, allLogs: Array<RunLogPayload>) {
431 const byModel = new Map<string, Array<RunLogPayload>>()
432 for (const log of allLogs) {
433 const list = byModel.get(log.entry.model) ?? []
434 list.push(log)
435 byModel.set(log.entry.model, list)
436 }
437​
438 for (const row of rows) {
439 const history = (byModel.get(row.model) ?? [])
440 .sort((a, b) => b.writtenAt.localeCompare(a.writtenAt))
441 .slice(0, 5)
442 const sampleSize = history.length
443 const stableCount = history.filter(isStableRun).length
444 const rate = sampleSize > 0 ? stableCount / sampleSize : 0
445 const tier = Math.max(1, Math.min(5, Math.round(rate * 4 + 1)))
446 row.stabilitySampleSize = sampleSize
447 row.stabilityRate = Number(rate.toFixed(2))
448 row.stabilityTier = tier
449 }
451​
452function finalizeRows(rows: Array<EvalRow>, allLogs: Array<RunLogPayload>) {
453 applyStability(rows, allLogs)
454​
455 bucketedTiers(
456 rows,
457 (row) => row.durationMs,
458 (row, tier) => {
459 row.speedTier = tier
460 },
461 true,
462 )
463 bucketedTiers(
464 rows,
465 (row) => row.tokenEfficiency,
466 (row, tier) => {
467 row.tokenEfficiencyTier = tier
468 },
469 true,
470 )
471​
472 for (const row of rows) {
473 if (!row.judge) continue
474 const rating = computeStarRating({
475 accuracy: row.judge.accuracy,
476 comprehensiveness: row.judge.comprehensiveness,
477 typescriptQuality: row.judge.typescriptQuality,
478 codeModeEfficiency: row.judge.codeModeEfficiency,
479 speedTier: row.speedTier,
480 tokenEfficiencyTier: row.tokenEfficiencyTier,
481 stabilityTier: row.stabilityTier,
482 compilationFailures: row.compilationFailures,
483 runtimeFailures: row.runtimeFailures,
484 totalExecuteCalls: row.totalExecuteCalls,
485 })
486 row.stars = rating.stars
487 row.weightedScore = rating.weightedScore
488 }
490​
491function listRunLogFiles(): Array<string> {
492 if (!existsSync(LOG_DIR)) return []
493 return readdirSync(LOG_DIR)
494 .filter(
495 (name) => name.endsWith('.json') && !name.startsWith('results-summary-'),
496 )
497 .map((name) => join(LOG_DIR, name))
499​
500function readRunLog(path: string): RunLogPayload | null {
501 try {
502 const raw = readFileSync(path, 'utf8')
503 return JSON.parse(raw) as RunLogPayload
504 } catch {
505 return null
506 }
508​
509async function judgeLatestSession(rejudge: boolean): Promise<void> {
510 const files = listRunLogFiles()
511 const payloads = files
512 .map((path) => ({ path, payload: readRunLog(path) }))
513 .filter(
514 (entry): entry is { path: string; payload: RunLogPayload } =>
515 !!entry.payload,
516 )
517​
518 if (payloads.length === 0) {
519 console.error('[models-eval] No run logs found to judge.')
520 process.exit(1)
521 }
522​
523 const latestSession = payloads
524 .map((entry) => entry.payload.sessionStartedAt)
525 .sort()
526 .at(-1)
527 if (!latestSession) {
528 console.error('[models-eval] Could not determine latest session.')
529 process.exit(1)
530 }
531​
532 const sessionPayloads = payloads.filter(
533 (entry) => entry.payload.sessionStartedAt === latestSession,
534 )
535 const rows: Array<EvalRow> = []
536​
537 for (const entry of sessionPayloads) {
538 const payload = entry.payload
539 if (payload.error) {
540 rows.push(toEvalRow(payload))
541 continue
542 }
543 if (payload.judge && !rejudge) {
544 rows.push(toEvalRow(payload))
545 continue
546 }
547​
548 const computed = computeMetrics(payload.messages)
549 const judged = await judgeReports({
550 query: payload.query,
551 goldReport: payload.goldReport,
552 candidateReport: payload.candidateReport,
553 typescriptEvidence: formatTypescriptEvidence(computed.typeScriptAttempts),
554 })
555 payload.judge = judged
556 payload.noJudge = false
557​
558 writeFileSync(entry.path, `${JSON.stringify(payload, null, 2)}\n`, 'utf8')
559 rows.push(toEvalRow(payload))
560 console.log(` ✓ judged from log: ${payload.entry.model}`)
561 }
562​
563 finalizeRows(
564 rows,
565 payloads.map((p) => p.payload),
566 )
567 const summaryPath = writeResultsSummary(rows, latestSession)
568 const canonicalPath = writeCanonicalResults(rows, latestSession)
569 printTable(rows)
570 console.log(
571 `[models-eval] Summary JSON → ${relative(process.cwd(), summaryPath) || summaryPath}`,
572 )
573 console.log(
574 `[models-eval] Canonical results → ${relative(process.cwd(), canonicalPath) || canonicalPath}`,
575 )
577​
578function writeResultsSummary(
579 rows: Array<EvalRow>,
580 sessionStartedAt: string,
581): string {
582 const path = join(
583 LOG_DIR,
584 `results-summary-${sessionStartedAt.replace(/[:.]/g, '-')}.json`,
585 )
586 writeFileSync(
587 path,
588 `${JSON.stringify(
589 {
590 sessionStartedAt,
591 createdAt: new Date().toISOString(),
592 rows,
593 },
594 null,
595 2,
596 )}\n`,
597 'utf8',
598 )
599 return path
601​
602function writeCanonicalResults(
603 rows: Array<EvalRow>,
604 sessionStartedAt: string,
605): string {
606 const path = join(__dirname, 'results.json')
607 const payload: CanonicalResultsFile = {
608 generatedAt: new Date().toISOString(),
609 sessionStartedAt,
610 methodology: {
611 benchmark:
612 'Single benchmark prompt over in-memory customers/products/purchases dataset with code-mode execute_typescript tooling.',
613 scoring: [
614 'accuracy (1-10): factual correctness vs gold reference',
615 'comprehensiveness (1-10): complete coverage of user request',
616 'typescriptQuality (1-10): quality and clarity of generated TypeScript',
617 'codeModeEfficiency (1-10): effectiveness and step efficiency in tool/code usage',
618 'speedTier (1-5): relative wall-clock performance bucketed by local/cloud',
619 'tokenEfficiencyTier (1-5): relative tokens-per-successful-execution bucketed by local/cloud',
620 'stabilityTier (1-5): success consistency over latest 5 logged runs per model',
621 ],
622 stability:
623 'Stable run means: no top-level error, non-empty candidate report, and at least one successful execute_typescript call.',
624 starWeights: {
625 accuracy: 0.25,
626 comprehensiveness: 0.15,
627 typescriptQuality: 0.15,
628 codeModeEfficiency: 0.1,
629 speedTier: 0.1,
630 tokenEfficiencyTier: 0.1,
631 stabilityTier: 0.15,
632 },
633 },
634 models: rows,
635 }
636 writeFileSync(path, `${JSON.stringify(payload, null, 2)}\n`, 'utf8')
637 return path
639​
640function teeStream(
641 stream: AsyncIterable<StreamChunk>,
642 onChunk: (chunk: StreamChunk) => void,
643): AsyncIterable<StreamChunk> {
644 return {
645 async *[Symbol.asyncIterator]() {
646 for await (const chunk of stream) {
647 onChunk(chunk)
648 yield chunk
649 }
650 },
651 }
653​
654function bucketedTiers(
655 rows: Array<EvalRow>,
656 getter: (row: EvalRow) => number | undefined,
657 setter: (row: EvalRow, tier: number) => void,
658 lowerIsBetter: boolean,
659) {
660 for (const category of ['local', 'cloud'] as const) {
661 const bucket = rows.filter((r) => r.modelCategory === category)
662 const values = bucket
663 .map((r) => ({ row: r, value: getter(r) }))
664 .filter(
665 (r): r is { row: EvalRow; value: number } =>
666 typeof r.value === 'number',
667 )
668​
669 if (values.length === 0) continue
670 const min = Math.min(...values.map((v) => v.value))
671 const max = Math.max(...values.map((v) => v.value))
672​
673 for (const entry of values) {
674 if (max === min) {
675 setter(entry.row, 5)
676 continue
677 }
678 const normalized = lowerIsBetter
679 ? (max - entry.value) / (max - min)
680 : (entry.value - min) / (max - min)
681 const tier = Math.round(1 + normalized * 4)
682 setter(entry.row, Math.max(1, Math.min(5, tier)))
683 }
684 }
686​
687function printTable(rows: Array<EvalRow>) {
688 console.log('\n=== Results ===\n')
689 const nameW = 22
690 const modelW = 38
691 const starsW = 5
692 const accW = 4
693 const compW = 4
694 const tsW = 3
695 const cmW = 3
696 const msW = 8
697​
698 const header = `${'Model'.padEnd(nameW)} ${'Id'.padEnd(modelW)} ${'★'.padStart(starsW)} ${'ms'.padStart(msW)} ${'Acc'.padStart(accW)} ${'Cmp'.padStart(compW)} ${'TS'.padStart(tsW)} ${'CME'.padStart(cmW)} ${'Spd'.padStart(3)} ${'Tok'.padStart(3)} ${'Stb'.padStart(3)} Error / summary`
699 console.log(header)
700 console.log('-'.repeat(header.length + 10))
701​
702 for (const r of rows) {
703 const acc = r.judge?.accuracy !== undefined ? String(r.judge.accuracy) : '-'
704 const comp =
705 r.judge?.comprehensiveness !== undefined
706 ? String(r.judge.comprehensiveness)
707 : '-'
708 const tsq =
709 r.judge?.typescriptQuality !== undefined
710 ? String(r.judge.typescriptQuality)
711 : '-'
712 const cme =
713 r.judge?.codeModeEfficiency !== undefined
714 ? String(r.judge.codeModeEfficiency)
715 : '-'
716 const stars = r.stars
717 ? `${'★'.repeat(r.stars)}${'☆'.repeat(3 - r.stars)}`
718 : '-'
719 const err = r.error || ''
720 const summary = r.judge?.summary
721 ? r.judge.summary.replace(/\s+/g, ' ').slice(0, 120)
722 : ''
723 const tail = err || summary
724 console.log(
725 `${r.name.slice(0, nameW - 1).padEnd(nameW)} ${r.model.slice(0, modelW - 1).padEnd(modelW)} ${stars.padStart(starsW)} ${String(r.durationMs).padStart(msW)} ${acc.padStart(accW)} ${comp.padStart(compW)} ${tsq.padStart(tsW)} ${cme.padStart(cmW)} ${String(r.speedTier).padStart(3)} ${String(r.tokenEfficiencyTier).padStart(3)} ${String(r.stabilityTier).padStart(3)} ${tail}`,
726 )
727 }
728 console.log('')
730​
731async function main() {
732 const {
733 ollamaOnly,
734 noJudge,
735 captureOnly,
736 judgeLatest,
737 rejudge,
738 modelFilter,
739 } = parseArgs(process.argv.slice(2))
740​
741 mkdirSync(LOG_DIR, { recursive: true })
742 if (judgeLatest) {
743 await judgeLatestSession(rejudge)
744 return
745 }
746​
747 const effectiveNoJudge = noJudge || captureOnly
748 const sessionStartedAt = new Date().toISOString()
749 console.log(
750 `[models-eval] Run logs → ${relative(process.cwd(), LOG_DIR) || LOG_DIR} (gitignored)`,
751 )
752​
753 if (!effectiveNoJudge && !process.env.ANTHROPIC_API_KEY) {
754 console.warn(
755 '[models-eval] ANTHROPIC_API_KEY is not set; judging will fail. Use --no-judge to skip, or set the key.',
756 )
757 }
758​
759 const gold = loadGoldFile()
760 const query = gold.prompt
761 const goldReport = extractReportTextFromGold(gold.messages)
762​
763 const { tool, systemPrompt } = getCodeModeTools()
764 const tools = [tool, getSchemaInfoTool]
765 const systemPrompts = [DATABASE_DEMO_SYSTEM_PROMPT, systemPrompt]
766​
767 let models = ollamaOnly
768 ? EVAL_MODELS.filter((m) => m.model.startsWith('ollama:'))
769 : EVAL_MODELS
770​
771 if (modelFilter && modelFilter.size > 0) {
772 models = models.filter((m) => modelFilter.has(m.model))
773 if (models.length === 0) {
774 console.error(
775 '[models-eval] No models matched --models filter. Check eval-config model ids.',
776 )
777 process.exit(1)
778 }
779 }
780​
781 const rows: Array<EvalRow> = []
782​
783 for (const entry of models) {
784 const { provider, modelId } = parseModelId(entry.model)
785 const modelCategory = getModelCategory(entry.model)
786 console.log(`\n→ Running: ${entry.name} (${entry.model})`)
787 const adapter = getTextAdapter(provider, modelId)
788 const t0 = Date.now()
789 let durationMs = 0
790 let ttftMs: number | undefined
791 let promptTokens = 0
792 let completionTokens = 0
793 let totalTokens = 0
794 let candidateReport = ''
795 let messagesLog: Array<UIMessage> = []
796 let error: string | undefined
797​
798 const userMessage = normalizeToUIMessage(
799 { role: 'user', content: query },
800 generateMessageId,
801 )
802 const processor = new StreamProcessor({
803 initialMessages: [userMessage],
804 })
805​
806 try {
807 const stream = chat({
808 adapter,
809 messages: [
810 {
811 role: 'user',
812 content: query,
813 },
814 ],
815 tools,
816 systemPrompts,
817 agentLoopStrategy: maxIterations(15),
818 // Sampling now lives in provider-native `modelOptions` keys rather
819 // than the removed root `maxTokens`. The max-output-tokens key name
820 // differs per provider.
821 modelOptions: maxTokensModelOptions(provider, 8192),
822 })
823 await processor.process(
824 teeStream(stream, (chunk) => {
825 if (chunk.type === 'TEXT_MESSAGE_CONTENT' && ttftMs === undefined) {
826 ttftMs = Date.now() - t0
827 }
828 if (chunk.type === 'RUN_FINISHED' && chunk.usage) {
829 promptTokens += chunk.usage.promptTokens
830 completionTokens += chunk.usage.completionTokens
831 totalTokens += chunk.usage.totalTokens
832 }
833 }),
834 )
835 messagesLog = processor.getMessages()
836 candidateReport = extractAssistantReportText(messagesLog)
837 durationMs = Date.now() - t0
838 } catch (e) {
839 durationMs = Date.now() - t0
840 messagesLog = processor.getMessages()
841 error = e instanceof Error ? e.message : String(e)
842 console.error(` ✗ ${error}`)
843 }
844​
845 const computed = computeMetrics(messagesLog)
846 const tokenEfficiency =
847 computed.successfulExecuteCalls > 0
848 ? Number((totalTokens / computed.successfulExecuteCalls).toFixed(2))
849 : undefined
850​
851 let judge: JudgeResult | undefined
852 if (!error && !effectiveNoJudge) {
853 try {
854 judge = await judgeReports({
855 query,
856 goldReport,
857 candidateReport,
858 typescriptEvidence: formatTypescriptEvidence(
859 computed.typeScriptAttempts,
860 ),
861 })
862 console.log(
863 ` ✓ judged: accuracy=${judge.accuracy} comprehensiveness=${judge.comprehensiveness} typescriptQuality=${judge.typescriptQuality} codeModeEfficiency=${judge.codeModeEfficiency} (${durationMs}ms)`,
864 )
865 } catch (e) {
866 const msg = e instanceof Error ? e.message : String(e)
867 error = error ? `${error}; judge: ${msg}` : `judge: ${msg}`
868 console.error(` ✗ judge failed: ${msg}`)
869 }
870 } else if (!error && effectiveNoJudge) {
871 console.log(
872 ` ✓ done (${durationMs}ms, ${captureOnly ? '--capture-only' : '--no-judge'})`,
873 )
874 }
875​
876 const logPayload: RunLogPayload = {
877 writtenAt: new Date().toISOString(),
878 sessionStartedAt,
879 entry: {
880 name: entry.name,
881 model: entry.model,
882 },
883 query,
884 durationMs,
885 ttftMs,
886 modelCategory,
887 promptTokens,
888 completionTokens,
889 totalTokens,
890 tokenEfficiency,
891 speedTier: 1,
892 tokenEfficiencyTier: 1,
893 stabilityTier: 1,
894 stabilityRate: 0,
895 stabilitySampleSize: 0,
896 totalToolCalls: computed.totalToolCalls,
897 totalExecuteCalls: computed.totalExecuteCalls,
898 successfulExecuteCalls: computed.successfulExecuteCalls,
899 compilationFailures: computed.compilationFailures,
900 runtimeFailures: computed.runtimeFailures,
901 redundantSchemaChecks: computed.redundantSchemaChecks,
902 candidateReportChars: candidateReport.length,
903 candidateReportEmpty: candidateReport.trim().length === 0,
904 goldReportChars: goldReport.length,
905 error,
906 judge,
907 noJudge: effectiveNoJudge,
908 goldReport,
909 candidateReport,
910 messages: messagesLog,
911 }
912​
913 const logPath = writeRunLog(logPayload)
914 const logRel = relative(process.cwd(), logPath) || logPath
915 console.log(` 📄 ${logRel}`)
916​
917 rows.push({
918 name: entry.name,
919 model: entry.model,
920 provider,
921 modelId,
922 modelCategory,
923 durationMs,
924 ttftMs,
925 promptTokens,
926 completionTokens,
927 totalTokens,
928 tokenEfficiency,
929 speedTier: 1,
930 tokenEfficiencyTier: 1,
931 stabilityTier: 1,
932 stabilityRate: 0,
933 stabilitySampleSize: 0,
934 totalToolCalls: computed.totalToolCalls,
935 totalExecuteCalls: computed.totalExecuteCalls,
936 successfulExecuteCalls: computed.successfulExecuteCalls,
937 compilationFailures: computed.compilationFailures,
938 runtimeFailures: computed.runtimeFailures,
939 redundantSchemaChecks: computed.redundantSchemaChecks,
940 error,
941 judge,
942 })
943 }
944​
945 const allLogs = listRunLogFiles()
946 .map((path) => readRunLog(path))
947 .filter((log): log is RunLogPayload => !!log)
948 finalizeRows(rows, allLogs)
949 const summaryPath = writeResultsSummary(rows, sessionStartedAt)
950 const canonicalPath = writeCanonicalResults(rows, sessionStartedAt)
951 printTable(rows)
952 console.log(
953 `[models-eval] Summary JSON → ${relative(process.cwd(), summaryPath) || summaryPath}`,
954 )
955 console.log(
956 `[models-eval] Canonical results → ${relative(process.cwd(), canonicalPath) || canonicalPath}`,
957 )
959​
960main().catch((e) => {
961 console.error(e)
962 process.exit(1)
963})