Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

results.json

259 lines10.0 KB

1{
2 "generatedAt": "2026-04-07T20:50:51.586Z",
3 "sessionStartedAt": "2026-04-07T20:37:49.485Z",
4 "methodology": {
5 "benchmark": "Single benchmark prompt over in-memory customers/products/purchases dataset with code-mode execute_typescript tooling.",
6 "scoring": [
7 "accuracy (1-10): factual correctness vs gold reference",
8 "comprehensiveness (1-10): complete coverage of user request",
9 "typescriptQuality (1-10): quality and clarity of generated TypeScript",
10 "codeModeEfficiency (1-10): effectiveness and step efficiency in tool/code usage",
11 "speedTier (1-5): relative wall-clock performance bucketed by local/cloud",
12 "tokenEfficiencyTier (1-5): relative tokens-per-successful-execution bucketed by local/cloud",
13 "stabilityTier (1-5): success consistency over latest 5 logged runs per model"
14 ],
15 "stability": "Stable run means: no top-level error, non-empty candidate report, and at least one successful execute_typescript call.",
16 "starWeights": {
17 "accuracy": 0.25,
18 "comprehensiveness": 0.15,
19 "typescriptQuality": 0.15,
20 "codeModeEfficiency": 0.1,
21 "speedTier": 0.1,
22 "tokenEfficiencyTier": 0.1,
23 "stabilityTier": 0.15
24 }
25 },
26 "models": [
27 {
28 "name": "GPT-OSS 20B",
29 "model": "ollama:gpt-oss:20b",
30 "provider": "ollama",
31 "modelId": "gpt-oss:20b",
32 "modelCategory": "local",
33 "durationMs": 45076,
34 "ttftMs": 43356,
35 "promptTokens": 20413,
36 "completionTokens": 3194,
37 "totalTokens": 23607,
38 "tokenEfficiency": 4721.4,
39 "speedTier": 5,
40 "tokenEfficiencyTier": 5,
41 "stabilityTier": 5,
42 "stabilityRate": 1,
43 "stabilitySampleSize": 1,
44 "totalToolCalls": 10,
45 "totalExecuteCalls": 6,
46 "successfulExecuteCalls": 5,
47 "compilationFailures": 0,
48 "runtimeFailures": 1,
49 "redundantSchemaChecks": 3,
50 "stars": 3,
51 "weightedScore": 0.8561,
52 "judge": {
53 "accuracy": 10,
54 "comprehensiveness": 8,
55 "typescriptQuality": 6,
56 "codeModeEfficiency": 5,
57 "summary": "The candidate report correctly identifies both customers (Bob Smith and Carol Williams) with accurate spend-per-category figures matching the gold standard. However, the presentation uses a long-format table rather than the more readable pivoted format with a total spend column, and the code took 6 attempts including a failure and multiple exploratory queries before arriving at the final in-memory join solution."
58 }
59 },
60 {
61 "name": "Nemotron Cascade 2",
62 "model": "ollama:nemotron-cascade-2",
63 "provider": "ollama",
64 "modelId": "nemotron-cascade-2",
65 "modelCategory": "local",
66 "durationMs": 60421,
67 "ttftMs": 54581,
68 "promptTokens": 8361,
69 "completionTokens": 3372,
70 "totalTokens": 11733,
71 "tokenEfficiency": 5866.5,
72 "speedTier": 5,
73 "tokenEfficiencyTier": 2,
74 "stabilityTier": 5,
75 "stabilityRate": 1,
76 "stabilitySampleSize": 1,
77 "totalToolCalls": 6,
78 "totalExecuteCalls": 3,
79 "successfulExecuteCalls": 2,
80 "compilationFailures": 0,
81 "runtimeFailures": 1,
82 "redundantSchemaChecks": 2,
83 "stars": 3,
84 "weightedScore": 0.7761,
85 "judge": {
86 "accuracy": 10,
87 "comprehensiveness": 9,
88 "typescriptQuality": 5,
89 "codeModeEfficiency": 5,
90 "summary": "The candidate report produces the exact same two customers, cities, and per-category spend figures as the gold standard, so accuracy is perfect and comprehensiveness is strong. However, the TypeScript had a failed first attempt due to incorrect handling of the query response structure, uses liberal `any` types and `as unknown as number` casts, and required three separate tool calls to arrive at the correct result, indicating mediocre code quality and efficiency."
91 }
92 },
93 {
94 "name": "Gemma 4 31B",
95 "model": "ollama:gemma4:31b",
96 "provider": "ollama",
97 "modelId": "gemma4:31b",
98 "modelCategory": "local",
99 "durationMs": 264216,
100 "ttftMs": 250076,
101 "promptTokens": 3835,
102 "completionTokens": 2610,
103 "totalTokens": 6445,
104 "tokenEfficiency": 6445,
105 "speedTier": 1,
106 "tokenEfficiencyTier": 1,
107 "stabilityTier": 5,
108 "stabilityRate": 1,
109 "stabilitySampleSize": 2,
110 "totalToolCalls": 3,
111 "totalExecuteCalls": 2,
112 "successfulExecuteCalls": 1,
113 "compilationFailures": 0,
114 "runtimeFailures": 1,
115 "redundantSchemaChecks": 0,
116 "stars": 2,
117 "weightedScore": 0.6128,
118 "judge": {
119 "accuracy": 10,
120 "comprehensiveness": 8,
121 "typescriptQuality": 4,
122 "codeModeEfficiency": 5,
123 "summary": "The candidate report correctly identifies both customers (Bob Smith and Carol Williams) with accurate spend-per-category figures matching the gold standard. However, the TypeScript code had an initial failed attempt due to misusing the API, and the successful attempt only fetched raw data without performing the actual filtering/aggregation logic in code, suggesting the analysis was done outside the shown code."
124 }
125 },
126 {
127 "name": "Claude Haiku 4.5",
128 "model": "anthropic:claude-haiku-4-5",
129 "provider": "anthropic",
130 "modelId": "claude-haiku-4-5",
131 "modelCategory": "cloud",
132 "durationMs": 9397,
133 "ttftMs": 537,
134 "promptTokens": 7544,
135 "completionTokens": 906,
136 "totalTokens": 8450,
137 "tokenEfficiency": 8450,
138 "speedTier": 4,
139 "tokenEfficiencyTier": 1,
140 "stabilityTier": 5,
141 "stabilityRate": 1,
142 "stabilitySampleSize": 1,
143 "totalToolCalls": 2,
144 "totalExecuteCalls": 1,
145 "successfulExecuteCalls": 1,
146 "compilationFailures": 0,
147 "runtimeFailures": 0,
148 "redundantSchemaChecks": 0,
149 "stars": 3,
150 "weightedScore": 0.825,
151 "judge": {
152 "accuracy": 10,
153 "comprehensiveness": 10,
154 "typescriptQuality": 7,
155 "codeModeEfficiency": 10,
156 "summary": "The candidate report matches the gold standard exactly in all numerical values, rankings, and factual claims, with identical table data for both customers. The TypeScript code accomplishes the task in a single efficient step, though it uses `any` types in several places and could benefit from proper interface definitions."
157 }
158 },
159 {
160 "name": "GPT-4o Mini",
161 "model": "openai:gpt-4o-mini",
162 "provider": "openai",
163 "modelId": "gpt-4o-mini",
164 "modelCategory": "cloud",
165 "durationMs": 19241,
166 "ttftMs": 16775,
167 "promptTokens": 8099,
168 "completionTokens": 603,
169 "totalTokens": 8702,
170 "tokenEfficiency": 8702,
171 "speedTier": 1,
172 "tokenEfficiencyTier": 1,
173 "stabilityTier": 5,
174 "stabilityRate": 1,
175 "stabilitySampleSize": 1,
176 "totalToolCalls": 4,
177 "totalExecuteCalls": 1,
178 "successfulExecuteCalls": 1,
179 "compilationFailures": 0,
180 "runtimeFailures": 0,
181 "redundantSchemaChecks": 2,
182 "stars": 2,
183 "weightedScore": 0.7333,
184 "judge": {
185 "accuracy": 10,
186 "comprehensiveness": 8,
187 "typescriptQuality": 8,
188 "codeModeEfficiency": 10,
189 "summary": "The candidate report correctly identifies both customers (Bob Smith and Carol Williams), their cities, and all per-category spend amounts matching the gold standard exactly. The response is accurate and efficiently produced in a single step, though it lacks a summary table format and the overall total spend figures included in the gold standard; the TypeScript code is functional and clear but uses `any`-implicit Maps without explicit typing."
190 }
191 },
192 {
193 "name": "Gemini 2.5 Flash",
194 "model": "gemini:gemini-2.5-flash",
195 "provider": "gemini",
196 "modelId": "gemini-2.5-flash",
197 "modelCategory": "cloud",
198 "durationMs": 7328,
199 "ttftMs": 6889,
200 "promptTokens": 5791,
201 "completionTokens": 628,
202 "totalTokens": 6865,
203 "tokenEfficiency": 6865,
204 "speedTier": 5,
205 "tokenEfficiencyTier": 2,
206 "stabilityTier": 5,
207 "stabilityRate": 1,
208 "stabilitySampleSize": 1,
209 "totalToolCalls": 2,
210 "totalExecuteCalls": 1,
211 "successfulExecuteCalls": 1,
212 "compilationFailures": 0,
213 "runtimeFailures": 0,
214 "redundantSchemaChecks": 0,
215 "stars": 3,
216 "weightedScore": 0.7839,
217 "judge": {
218 "accuracy": 10,
219 "comprehensiveness": 7,
220 "typescriptQuality": 5,
221 "codeModeEfficiency": 9,
222 "summary": "The candidate correctly identifies both customers (Bob Smith and Carol Williams), their cities, and all spend-per-category figures match the gold standard exactly. However, the report lacks a summary table format, total spend column, and analytical insights present in the gold standard, and the TypeScript code relies heavily on `any` types with no proper interfaces defined, reducing type safety."
223 }
224 },
225 {
226 "name": "Grok 4.1 Fast",
227 "model": "grok:grok-4-1-fast-non-reasoning",
228 "provider": "grok",
229 "modelId": "grok-4-1-fast-non-reasoning",
230 "modelCategory": "cloud",
231 "durationMs": 6998,
232 "ttftMs": 6454,
233 "promptTokens": 0,
234 "completionTokens": 0,
235 "totalTokens": 0,
236 "tokenEfficiency": 0,
237 "speedTier": 5,
238 "tokenEfficiencyTier": 5,
239 "stabilityTier": 5,
240 "stabilityRate": 1,
241 "stabilitySampleSize": 1,
242 "totalToolCalls": 4,
243 "totalExecuteCalls": 1,
244 "successfulExecuteCalls": 1,
245 "compilationFailures": 0,
246 "runtimeFailures": 0,
247 "redundantSchemaChecks": 2,
248 "stars": 3,
249 "weightedScore": 0.9167,
250 "judge": {
251 "accuracy": 10,
252 "comprehensiveness": 9,
253 "typescriptQuality": 6,
254 "codeModeEfficiency": 10,
255 "summary": "The candidate report correctly identifies both customers (Bob Smith and Carol Williams) with perfectly matching spend figures across all three categories. The TypeScript code accomplishes the task in a single step but uses excessive `any` types and lacks proper type definitions, reducing type safety and idiomatic quality."
256 }
257 }
258 ]