Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

eval-config.ts

76 lines2.5 KB

1export type EvalProvider =
2 | 'ollama'
3 | 'openai'
4 | 'anthropic'
5 | 'gemini'
6 | 'grok'
7 | 'groq'
8​
9export interface EvalModel {
10 /** Human-readable label */
11 name: string
12 /** Provider-qualified model id: `provider:model` (e.g. `openai:gpt-4o`) */
13 model: string
15​
16export type ModelCategory = 'local' | 'cloud'
17​
18/** Split a `provider:model` string. First `:` is the delimiter. */
19export function parseModelId(model: string): {
20 provider: EvalProvider
21 modelId: string
22} {
23 const idx = model.indexOf(':')
24 if (idx === -1) {
25 throw new Error(
26 `Invalid model format "${model}". Expected "provider:model" (e.g. "openai:gpt-4o").`,
27 )
28 }
29 return {
30 provider: model.slice(0, idx) as EvalProvider,
31 modelId: model.slice(idx + 1),
32 }
34​
35export function isCloudModel(model: string): boolean {
36 const { provider, modelId } = parseModelId(model)
37 if (provider !== 'ollama') return true
38 return modelId.includes(':cloud')
40​
41export function getModelCategory(model: string): ModelCategory {
42 return isCloudModel(model) ? 'cloud' : 'local'
44​
45/**
46 * Default models to benchmark (see pull-models.sh for Ollama pulls).
47 * Cloud baselines require respective API keys.
48 *
49 * Excluded from default runs (still runnable with `--models <id>` if installed):
50 * - ollama:granite4:3b — emits invalid `typescriptCode` (esbuild parse errors), then hallucinates answers.
51 * - ollama:ministral-3 — calls tools but generated TS often mishandles API (e.g. `.rows`), repeated failures.
52 * - ollama:mistral:7b — answers with markdown "code" only; does not invoke `execute_typescript` reliably.
53 * - ollama:qwen3:8b, ollama:rnj-1:8b, ollama:qwen3.5:9b, ollama:qwen2.5-coder:14b, ollama:qwen3:14b,
54 * ollama:qwen3-coder, ollama:devstral-small-2
55 * — hallucinate results or fail to use external_queryTable correctly.
56 */
57export const EVAL_MODELS: Array<EvalModel> = [
58 // --- Ollama (working) ---
59 { name: 'GPT-OSS 20B', model: 'ollama:gpt-oss:20b' },
60 { name: 'Nemotron Cascade 2', model: 'ollama:nemotron-cascade-2' },
61 { name: 'Gemma 4 31B', model: 'ollama:gemma4:31b' },
62​
63 // --- Cloud baselines ---
64 { name: 'Claude Haiku 4.5', model: 'anthropic:claude-haiku-4-5' },
65 { name: 'GPT-4o Mini', model: 'openai:gpt-4o-mini' },
66​
67 // --- Gemini ---
68 { name: 'Gemini 2.5 Flash', model: 'gemini:gemini-2.5-flash' },
69​
70 // --- Grok (xAI) ---
71 { name: 'Grok Build 0.1', model: 'grok:grok-build-0.1' },
72​
73 // --- Groq ---
74 { name: 'Llama 3.3 70B (Groq)', model: 'groq:llama-3.3-70b-versatile' },
75 { name: 'Qwen3 32B (Groq)', model: 'groq:qwen/qwen3-32b' },