Open History to see the last commit
1import { describe, expect, it } from 'vitest'
2
3import { parseLedger } from '../src/lib/completeness.ts'
4import {
5 classify,
6 compare,
7 counterparts,
8 formatProvider,
9 formatTable,
10 main,
11 matchModels,
12} from './compare-models-dev.ts'
13import type { OurRow, TheirCatalog, TheirModel } from './compare-models-dev.ts'
14
15// Hand-written; not a copy of models.dev data.
16const their = (id: string, extra: Partial<TheirModel> = {}): TheirModel => ({
17 id,
18 reasoning: false,
19 tool_call: true,
20 modalities: { input: ['text'], output: ['text'] },
21 limit: { context: 128000, output: 4096 },
22 ...extra,
23})
24
25const catalog = (...models: Array<TheirModel>) => ({
26 models: Object.fromEntries(models.map((model) => [model.id, model])),
27})
28
29const ours = (rawId: string, extra: Partial<OurRow> = {}): OurRow => ({
30 provider: 'acme',
31 rawId,
32 activity: 'chat',
33 ...extra,
34})
35
36const usdCard = (base: Record<string, number>) => ({
37 price: { var: 'input_tokens' },
38 tables: { rate: { base } },
39 source: { url: 'https://acme.example/pricing' },
40})
41
42describe('counterparts', () => {
43 const theirs: TheirCatalog = {
44 xai: catalog(),
45 acme: catalog(),
46 perplexity: catalog(),
47 'perplexity-agent': catalog(),
48 volcengine: catalog(),
49 }
50
51 it('maps a renamed provider, and keeps an identical id', () => {
52 expect(counterparts('grok', theirs)).toEqual(['xai'])
53 expect(counterparts('acme', theirs)).toEqual(['acme'])
54 })
55
56 it('maps one of ours onto both halves of a provider they split', () => {
57 expect(counterparts('perplexity', theirs)).toEqual([
58 'perplexity-agent',
59 'perplexity',
60 ])
61 })
62
63 it('reports no counterpart rather than a look-alike', () => {
64 expect(counterparts('byteplus', theirs)).toEqual([])
65 expect(counterparts('gemini', theirs)).toEqual([])
66 })
67})
68
69describe('matchModels', () => {
70 const index = (...models: Array<TheirModel>) =>
71 new Map(models.map((model) => [model.id, { provider: 'acme', model }]))
72 const how = (rows: Array<OurRow>, ...models: Array<TheirModel>) => {
73 const { matches, unmatched } = matchModels(rows, index(...models))
74 return {
75 matches: matches.map((m) => `${m.row.rawId}→${m.theirs.id} ${m.how}`),
76 unmatched: unmatched.map((row) => row.rawId),
77 }
78 }
79
80 it('matches the exact raw id first', () => {
81 expect(
82 how(
83 [ours('gpt-5', { aliases: ['gpt-5-2025'] })],
84 their('gpt-5'),
85 their('gpt-5-2025'),
86 ),
87 ).toEqual({ matches: ['gpt-5→gpt-5 exact'], unmatched: [] })
88 })
89
90 it('then a stored alias', () => {
91 expect(
92 how(
93 [ours('claude-x-20250101', { aliases: ['claude-x'] })],
94 their('claude-x'),
95 ),
96 ).toEqual({ matches: ['claude-x-20250101→claude-x alias'], unmatched: [] })
97 })
98
99 it('then case and dots-vs-hyphens', () => {
100 expect(how([ours('Org/Model-V3.1')], their('org/model-v3-1'))).toEqual({
101 matches: ['Org/Model-V3.1→org/model-v3-1 normalised'],
102 unmatched: [],
103 })
104 })
105
106 it('never matches a sibling or a dated variant', () => {
107 expect(
108 how(
109 [ours('gpt-5')],
110 their('gpt-5-mini'),
111 their('gpt-5-pro'),
112 their('gpt-5-2025-08-07'),
113 ),
114 ).toEqual({ matches: [], unmatched: ['gpt-5'] })
115 })
116
117 it('leaves an ambiguous normalised id unmatched', () => {
118 expect(
119 how([ours('m-1.5')], their('M-1.5'), their('m-1-5')).matches,
120 ).toEqual([])
121 expect(how([ours('m-1.5'), ours('M-1-5')], their('m-1-5')).matches).toEqual(
122 [],
123 )
124 })
125
126 it('claims each of their models once', () => {
127 expect(how([ours('a'), ours('b', { aliases: ['a'] })], their('a'))).toEqual(
128 { matches: ['a→a exact'], unmatched: ['b'] },
129 )
130 })
131})
132
133describe('classify', () => {
134 it('names each of the five outcomes', () => {
135 expect(classify(1, 1)).toBe('agree')
136 expect(classify('text', 'image,text')).toBe('disagree')
137 expect(classify(true, undefined)).toBe('onlyOurs')
138 expect(classify(undefined, false)).toBe('onlyTheirs')
139 expect(classify(undefined, undefined)).toBe('neither')
140 })
141
142 it('counts a differently rounded label apart from exact and from wrong', () => {
143 const rounding = { rounding: true }
144 expect(classify(128000, 131072, rounding)).toBe('agreeRounding')
145 expect(classify(1048576, 1000000, rounding)).toBe('agreeRounding')
146 expect(classify(200000, 200000, rounding)).toBe('agree')
147 expect(classify(400000, 272000, rounding)).toBe('disagree')
148 // Only limits round; a price 2% off is a different price.
149 expect(classify(1, 1.02)).toBe('disagree')
150 })
151
152 it('does not compare a non-USD price, and never converts it', () => {
153 expect(classify('2.1/8.4', '0.3/1.2', { currency: 'CNY' })).toBe('nonUsd')
154 expect(classify('0.3/1.2', '0.3/1.2', { currency: 'USD' })).toBe('agree')
155 expect(classify(undefined, '0.3/1.2', { currency: 'CNY' })).toBe(
156 'onlyTheirs',
157 )
158 })
159})
160
161describe('compare', () => {
162 const rows: Array<OurRow> = [
163 ours('full', {
164 contextWindow: 131072,
165 maxOutput: 8192,
166 modalities: { input: ['text', 'pdf'], output: ['text'] },
167 pricing: usdCard({
168 input_tokens: 1.25e-6,
169 output_tokens: 1e-5,
170 cache_read_tokens: 1.25e-7,
171 }),
172 capabilities: { tools: true, temperature: true },
173 reasoning: { mode: 'effort', efforts: ['low', 'high'] },
174 }),
175 ours('bare'),
176 ours('yuan', {
177 pricing: {
178 ...usdCard({ prompt: 2.1e-6, completion: 8.4e-6 }),
179 price: { currency: ['CNY', { var: 'input_tokens' }] },
180 },
181 }),
182 ours('borrowed', {
183 pricing: {
184 ...usdCard({ input_tokens: 1e-6, output_tokens: 2e-6 }),
185 source: { url: 'https://models.dev/api.json' },
186 },
187 }),
188 ours('ours-only'),
189 ours('speaker', { activity: 'tts' }),
190 ours('grok-1', { provider: 'grok' }),
191 ours('lonely', { provider: 'byteplus' }),
192 ours('image-only', { provider: 'studio', activity: 'image' }),
193 ]
194 const priced = { cost: { input: 1.25, output: 10, cache_read: 0.125 } }
195 const theirs: TheirCatalog = {
196 acme: catalog(
197 their('full', {
198 ...priced,
199 reasoning: true,
200 reasoning_options: [{ type: 'effort', values: ['high', 'low'] }],
201 temperature: false,
202 modalities: { input: ['text', 'pdf'], output: ['text'] },
203 release_date: '2026-01-01',
204 family: 'full',
205 }),
206 their('bare', priced),
207 their('yuan', priced),
208 their('borrowed', priced),
209 their('theirs-only'),
210 their('speaker'),
211 their('text-embed-1'),
212 their('painter', { modalities: { input: ['text'], output: ['image'] } }),
213 ),
214 xai: catalog(their('grok-1')),
215 volcengine: catalog(their('lonely')),
216 }
217 const ledger = parseLedger('- acme: maxOutput — not published\n')
218 const report = compare(rows, theirs, ledger, new Date(0))
219 const acme = report.providers.find((p) => p.provider === 'acme')
220 const cell = (
221 model: string,
222 fact: 'priced' | 'contextWindow' | 'maxOutput',
223 ) => acme?.models.find((m) => m.ours === model)?.facts[fact]
224
225 it('scopes to our chat rows and lists both sides’ leftovers', () => {
226 expect(report.providers.map((p) => p.provider)).not.toContain('studio')
227 expect(acme).toMatchObject({
228 chat: 5,
229 matched: 4,
230 unmatchedOurs: ['ours-only'],
231 notListed: ['theirs-only'],
232 listedNotChat: ['speaker'],
233 })
234 expect(report.notInModelsDev).toEqual(['byteplus'])
235 expect(report.providers.find((p) => p.provider === 'grok')?.theirs).toEqual(
236 ['xai'],
237 )
238 })
239
240 it('classifies each fact of a matched pair, keeping both values', () => {
241 const full = acme?.models.find((m) => m.ours === 'full')?.facts
242 expect(full).toMatchObject({
243 contextWindow: { status: 'agreeRounding', ours: 131072, theirs: 128000 },
244 maxOutput: { status: 'disagree', ours: 8192, theirs: 4096 },
245 // Their `pdf` and our `file` are one modality.
246 inputModalities: { status: 'agree', ours: 'file,text' },
247 priced: { status: 'agree', ours: '1.25/10', theirs: '1.25/10' },
248 cacheRead: { status: 'agree', ours: 0.125 },
249 cacheWrite: { status: 'neither' },
250 tools: { status: 'agree' },
251 // The map has no `structured_outputs` key: unknown.
252 structuredOutput: { status: 'neither' },
253 reasoning: { status: 'agree' },
254 reasoningOptions: { status: 'agree', ours: 'effort:high,low' },
255 temperature: { status: 'disagree', ours: true, theirs: false },
256 releasedAt: { status: 'onlyTheirs', theirs: '2026-01-01' },
257 knowledgeCutoff: { status: 'neither' },
258 openWeights: { status: 'neither' },
259 })
260 expect(acme?.noField).toMatchObject({ family: 1 })
261 })
262
263 it('reports a non-USD card as held but not compared', () => {
264 expect(cell('yuan', 'priced')).toEqual({
265 status: 'nonUsd',
266 ours: '2.1/8.4',
267 theirs: '1.25/10',
268 })
269 })
270
271 it('does not count a price we took from models.dev as ours', () => {
272 expect(cell('borrowed', 'priced')?.status).toBe('onlyTheirs')
273 })
274
275 it('marks a gap the ledger explains, and only that one', () => {
276 expect(cell('bare', 'maxOutput')).toEqual({
277 status: 'onlyTheirs',
278 theirs: 4096,
279 ledgered: true,
280 })
281 expect(cell('bare', 'contextWindow')).toEqual({
282 status: 'onlyTheirs',
283 theirs: 128000,
284 })
285 expect(acme?.silent).toEqual(['maxOutput'])
286 expect(acme?.facts.maxOutput.ledgered).toBe(3)
287 })
288
289 it('states parity both ways, with and without ledgered facts', () => {
290 const { headline, headlineExcludingLedgered: scored } = report
291 expect(headline.matched).toBe(5)
292 expect(headline.oursOfTheirs.have).toBe(headline.theirsOfOurs.have)
293 expect(headline.oursOfTheirs.of).toBeGreaterThan(headline.oursOfTheirs.have)
294 // The ledgered fact leaves both sides of the ratio: 3 gaps and 1 held.
295 expect(headline.oursOfTheirs.of - scored.oursOfTheirs.of).toBe(4)
296 expect(headline.oursOfTheirs.have - scored.oursOfTheirs.have).toBe(1)
297 expect(scored.oursOfTheirs.ratio).toBeGreaterThan(
298 headline.oursOfTheirs.ratio,
299 )
300 })
301
302 it('prints the table worst first with the caveat, and a provider view', () => {
303 const table = formatTable(report).split('\n')
304 expect(table[1]).toMatch(/^acme\s+acme\s+5\s+4\s+1\s+1\s/)
305 expect(table.join('\n')).toMatch(/byteplus\s+not in models\.dev/)
306 expect(table.join('\n')).toContain('3/4*')
307 expect(table.at(-1)).toContain('not ground truth')
308 const detail = formatProvider(report.providers[0]!)
309 expect(detail).toContain('behind bare: theirs 128000')
310 expect(detail).toContain('disagree full: ours 8192 · theirs 4096')
311 })
312})
313
314describe('what counts as a value', () => {
315 const rows: Array<OurRow> = [
316 ours('thinker', { capabilities: { reasoning: true, tools: false } }),
317 ours('tooler', { capabilities: { tools: true, reasoning: true } }),
318 ours('empty', { capabilities: null }),
319 ours('jingle', {
320 pricing: {
321 price: 0.04,
322 tables: {},
323 source: { url: 'https://acme.example/pricing' },
324 },
325 }),
326 ours('translator', { pricing: usdCard({ audio_tokens: 3.5e-6 }) }),
327 ours('effortless', { reasoning: { mode: 'effort' } }),
328 ours('notext', { modalities: { input: ['image'], output: ['text'] } }),
329 ours('vendor.m', { aliases: ['au.vendor.m-v1', 'us.vendor.m-v1'] }),
330 ]
331 const theirs: TheirCatalog = {
332 acme: catalog(
333 their('thinker', { temperature: true, structured_output: true }),
334 their('tooler'),
335 their('empty'),
336 their('jingle', { cost: { input: 1, output: 2 } }),
337 their('translator', { cost: { input: 1, output: 2 } }),
338 their('effortless', {
339 reasoning: true,
340 reasoning_options: [{ type: 'effort', values: ['low', 'high'] }],
341 cost: { input: 0, output: 0, cache_read: 0, cache_write: 0 },
342 }),
343 their('notext'),
344 their('au.vendor.m-v1'),
345 their('vendor.m-v1'),
346 their('us.vendor.m-v1'),
347 ),
348 }
349 const report = compare(rows, theirs, new Map(), new Date(0))
350 const acme = report.providers[0]!
351 const facts = (model: string) =>
352 acme.models.find((m) => m.ours === model)?.facts
353
354 it('holds a false only where the map states one', () => {
355 // `tools: false` is in the row's own map: a held false.
356 expect(facts('thinker')?.tools).toEqual({
357 status: 'disagree',
358 ours: false,
359 theirs: true,
360 })
361 // No key for these two: unknown, so we are behind. A sibling row that
362 // lists a flag says nothing about this one.
363 expect(facts('thinker')?.temperature.status).toBe('onlyTheirs')
364 expect(facts('thinker')?.structuredOutput.status).toBe('onlyTheirs')
365 expect(facts('thinker')?.reasoning.ours).toBe(true)
366 expect(facts('tooler')?.structuredOutput.status).toBe('neither')
367 })
368
369 it('compares the three dated and stated facts, a month agreeing with its days', () => {
370 const dated = compare(
371 [
372 ours('m', {
373 releasedAt: Date.UTC(2026, 0, 1) / 1000,
374 knowledgeCutoff: '2025-06',
375 openWeights: false,
376 }),
377 ],
378 {
379 acme: catalog(
380 their('m', {
381 release_date: '2026-01-02',
382 knowledge: '2025-06-30',
383 open_weights: false,
384 }),
385 ),
386 },
387 new Map(),
388 new Date(0),
389 ).providers[0]?.models[0]?.facts
390 expect(dated).toMatchObject({
391 releasedAt: { status: 'disagree', ours: '2026-01-01' },
392 knowledgeCutoff: { status: 'agree', ours: '2025-06' },
393 openWeights: { status: 'agree', ours: false },
394 })
395 })
396
397 it('reads no map as unknown', () => {
398 expect(facts('empty')?.tools).toEqual({
399 status: 'onlyTheirs',
400 theirs: true,
401 })
402 expect(facts('empty')?.reasoning.status).toBe('onlyTheirs')
403 })
404
405 it('counts a per-request or audio-token card as priced, uncompared', () => {
406 const nonToken = {
407 status: 'nonToken',
408 ours: 'non-token card',
409 theirs: '1/2',
410 }
411 expect(facts('jingle')?.priced).toEqual(nonToken)
412 expect(facts('translator')?.priced).toEqual(nonToken)
413 expect(acme.facts.priced.nonToken).toBe(2)
414 })
415
416 it('does not hold reasoning options for an effort mode with no names', () => {
417 expect(facts('effortless')?.reasoningOptions).toEqual({
418 status: 'onlyTheirs',
419 theirs: 'effort:high,low',
420 })
421 })
422
423 it('counts models.dev zero rates, discloses them, and can leave them out', () => {
424 expect(report.zeroRates).toEqual({ priced: 1, cacheRead: 1, cacheWrite: 1 })
425 expect(report.note).toContain(
426 'price 0/0 on 1 matched models, cache_read 0 on 1, cache_write 0 on 1',
427 )
428 const { headline, headlineExcludingZeroRates: nonZero } = report
429 expect(headline.oursOfTheirs.of - nonZero.oursOfTheirs.of).toBe(3)
430 expect(nonZero.oursOfTheirs.have).toBe(headline.oursOfTheirs.have)
431 expect(formatTable(report)).toContain('excluding models.dev zero rates:')
432 })
433
434 it('prefers the model itself to a regional profile of it', () => {
435 expect(acme.models.find((m) => m.ours === 'vendor.m')).toMatchObject({
436 theirs: 'acme/vendor.m-v1',
437 how: 'alias',
438 })
439 // The regional ids are aliases of a chat row, not rows we hold as non-chat.
440 expect(acme.aliasOfChatRow).toEqual(['au.vendor.m-v1', 'us.vendor.m-v1'])
441 expect(acme.listedNotChat).toEqual([])
442 expect(acme.notListed).toEqual([])
443 })
444
445 it('keeps the rows whose input modalities lack text visible', () => {
446 expect(report.inputWithoutText).toEqual({
447 total: 1,
448 byProvider: { acme: 1 },
449 })
450 expect(formatTable(report)).toContain(
451 'input modalities lack "text": 1 (acme 1)',
452 )
453 // Compared as stored, not repaired.
454 expect(facts('notext')?.inputModalities).toMatchObject({
455 status: 'disagree',
456 ours: 'image',
457 theirs: 'text',
458 })
459 })
460})
461
462describe('main', () => {
463 it('rejects a --min outside 0..1 before fetching anything', async () => {
464 await expect(main(['--min', '2'])).rejects.toThrow('--min')
465 })
466})