Skip to content
Source
Sign inSign up
TanStack/schemas
Open History to see the last commit

compare-models-dev.ts

986 lines33.7 KB

1/**
2 * COMPARE-ONLY. models.dev is never a source for this catalog (issue #197).
3 * This script reads models.dev to answer one question — "do we have as much
4 * as models.dev for the chat models we support?" — and prints the answer.
5 * Nothing it reads may be written to the DB, an adapter, an ingest fixture
6 * or the source-silent ledger, and nothing under `src/server/ingest/` or
7 * `src/server/providers/` may import this file (nor this file them).
8 *
9 * pnpm run compare:models-dev # per-provider table
10 * pnpm run compare:models-dev -- --json # everything, per model
11 * pnpm run compare:models-dev -- --provider grok # where we are behind / disagree
12 * pnpm run compare:models-dev -- --ours a.json --theirs b.json # saved inputs
13 * pnpm run compare:models-dev -- --check --min 0.9 # exit 1 below 90% parity
14 *
15 * Two requests per run: `GET /v1/models` (anonymous prod is 60 requests an
16 * hour, shared) and `GET https://models.dev/api.json`.
17 *
18 * models.dev is not ground truth. "Only they have it" means a value exists
19 * there, not that a correct value exists there.
20 */
21import { readFileSync } from 'node:fs'
22import { parseArgs } from 'node:util'
23​
24import { cardCurrency } from '@modelschemas/rate-card'
25​
26import { readLedger } from './gap-report.ts'
27import { isCapabilityMap } from '../src/lib/capabilities.ts'
28import type { FactKey as LedgerKey, Ledger } from '../src/lib/completeness.ts'
29​
30const THEIRS_URL = 'https://models.dev/api.json'
31​
32/**
33 * Our provider id → models.dev provider id(s), where the two differ. Any
34 * provider not listed maps to the same id; one with no counterpart is
35 * reported as "not in models.dev". Each pair was checked by reading both
36 * sides' model ids and API hosts (2026-10-07).
37 */
38export const PROVIDER_MAP: Record<string, Array<string>> = {
39 // Google's Gemini API (ai.google.dev); 25 of our ids are theirs verbatim.
40 gemini: ['google'],
41 // xAI; all 8 of our chat ids are theirs verbatim.
42 grok: ['xai'],
43 // Same `Org/Model` ids on api.together.xyz.
44 together: ['togetherai'],
45 // Same `accounts/fireworks/models/…` ids.
46 fireworks: ['fireworks-ai'],
47 // Same `org/model` ids on api.novita.ai.
48 novita: ['novita-ai'],
49 // Ours is the international platform (api.moonshot.ai); the China one is
50 // `moonshotai-cn` on both sides.
51 moonshot: ['moonshotai'],
52 // Our prices come from dashscope-intl.aliyuncs.com, which is their
53 // `alibaba`; `alibaba-cn` is the mainland endpoint with CNY prices.
54 dashscope: ['alibaba'],
55 // They split Perplexity in two: `perplexity-agent` holds the Agent API's
56 // `vendor/model` ids our chat rows use, `perplexity` the bare Sonar ids.
57 perplexity: ['perplexity-agent', 'perplexity'],
58 // Deliberately absent: `byteplus`. Their `volcengine` is the mainland Ark
59 // (ark.cn-beijing.volces.com, CNY) — a different platform that happens to
60 // share some model ids.
62​
63export type OurRow = {
64 provider: string
65 rawId: string
66 activity?: string | null
67 contextWindow?: number | null
68 maxOutput?: number | null
69 modalities?: {
70 input?: Array<string> | null
71 output?: Array<string> | null
72 } | null
73 pricing?: {
74 price?: unknown
75 tables?: { rate?: { base?: Record<string, unknown> } }
76 source?: { url?: string }
77 } | null
78 capabilities?: unknown
79 reasoning?: { mode?: string; efforts?: Array<string> | null } | null
80 aliases?: Array<string> | null
81 releasedAt?: number | null
82 knowledgeCutoff?: string | null
83 openWeights?: boolean | null
85​
86export type TheirModel = {
87 id: string
88 family?: string
89 reasoning?: boolean
90 tool_call?: boolean
91 temperature?: boolean
92 structured_output?: boolean
93 reasoning_options?: Array<{ type: string; values?: Array<string> }>
94 modalities?: { input?: Array<string>; output?: Array<string> }
95 limit?: { context?: number; output?: number; input?: number }
96 cost?: {
97 input?: number
98 output?: number
99 cache_read?: number
100 cache_write?: number
101 }
102 release_date?: string
103 knowledge?: string
104 open_weights?: boolean
106​
107export type TheirCatalog = Record<
108 string,
109 { models: Record<string, TheirModel> }
111​
112/** models.dev provider ids for one of ours; empty when it has none. */
113export function counterparts(
114 provider: string,
115 theirs: TheirCatalog,
116): Array<string> {
117 return (PROVIDER_MAP[provider] ?? [provider]).filter((id) => id in theirs)
119​
120// ── matching ────────────────────────────────────────────────────────────
121​
122export type MatchRule = 'exact' | 'alias' | 'normalised'
123​
124export type Match = {
125 row: OurRow
126 theirs: TheirModel
127 theirProvider: string
128 how: MatchRule
130​
131/** Case and dots-vs-hyphens only: `GPT-4.1` ≡ `gpt-4-1`. Never siblings. */
132export function normaliseId(id: string): string {
133 return id.toLowerCase().replaceAll('.', '-')
135​
136/** A Bedrock-style regional inference-profile prefix: `us.vendor.model`. */
137const REGION_PREFIX = /^(?:us|us-gov|eu|au|apac|global|in|jp|ca)\./
138​
139/**
140 * A row's stored aliases in the order the alias pass tries them: the model
141 * itself before a regional profile of it. A regional alias (`au.vendor.x`)
142 * also names the model it routes to (`vendor.x`), so that id is a candidate
143 * too, ahead of any regional one — otherwise the row would be compared with
144 * one region's limits and prices.
145 */
146export function aliasCandidates(row: OurRow): Array<string> {
147 const aliases = row.aliases ?? []
148 const regional = aliases.filter((alias) => REGION_PREFIX.test(alias))
149 return [
150 ...new Set([
151 ...aliases.filter((alias) => !REGION_PREFIX.test(alias)),
152 ...regional.map((alias) => alias.replace(REGION_PREFIX, '')),
153 ...regional,
154 ]),
155 ]
157​
158/**
159 * Exact raw id, then our stored aliases, then {@link normaliseId} — each
160 * pass over what the earlier ones left, and each of their models claimed at
161 * most once. A normalised key shared by two ids on either side matches
162 * nothing: ambiguity is reported as unmatched, not guessed.
163 */
164export function matchModels(
165 rows: Array<OurRow>,
166 theirs: Map<string, { provider: string; model: TheirModel }>,
167): { matches: Array<Match>; unmatched: Array<OurRow> } {
168 const matches: Array<Match> = []
169 const claimed = new Set<string>()
170 let left = rows
171​
172 const pass = (how: MatchRule, find: (row: OurRow) => string | undefined) => {
173 const next: Array<OurRow> = []
174 for (const row of left) {
175 const id = find(row)
176 const hit = id === undefined ? undefined : theirs.get(id)
177 if (id === undefined || !hit || claimed.has(id)) {
178 next.push(row)
179 continue
180 }
181 claimed.add(id)
182 matches.push({ row, theirs: hit.model, theirProvider: hit.provider, how })
183 }
184 left = next
185 }
186​
187 pass('exact', (row) => row.rawId)
188 pass('alias', (row) =>
189 aliasCandidates(row).find((id) => theirs.has(id) && !claimed.has(id)),
190 )
191​
192 const group = (ids: Array<string>) => {
193 const byKey = new Map<string, Array<string>>()
194 for (const id of ids) {
195 const key = normaliseId(id)
196 byKey.set(key, [...(byKey.get(key) ?? []), id])
197 }
198 return byKey
199 }
200 const theirKeys = group([...theirs.keys()].filter((id) => !claimed.has(id)))
201 const ourKeys = group(left.map((row) => row.rawId))
202 pass('normalised', (row) => {
203 const key = normaliseId(row.rawId)
204 const candidates = theirKeys.get(key)
205 return ourKeys.get(key)?.length === 1 && candidates?.length === 1
206 ? candidates[0]
207 : undefined
208 })
209​
210 return { matches, unmatched: left }
212​
213// ── facts ───────────────────────────────────────────────────────────────
214​
215/** A fact in canonical, directly comparable form. */
216export type Value = number | boolean | string
217​
218export const STATUSES = [
219 'agree',
220 'agreeRounding',
221 'disagree',
222 'nonUsd',
223 'nonToken',
224 'onlyOurs',
225 'onlyTheirs',
226 'neither',
227] as const
228​
229export type Status = (typeof STATUSES)[number]
230​
231/**
232 * Two token limits within 5% of each other are one published label rounded
233 * two ways (128,000 vs 131,072 is 2.3%; 1,000,000 vs 1,048,576 is 4.6%) and
234 * count as `agreeRounding`, apart from exact agreement and from a real
235 * disagreement such as 400,000 vs 272,000.
236 */
237export const ROUNDING_TOLERANCE = 0.05
238​
239/** Our `priced` value for a card with no input and output token rate. */
240export const NON_TOKEN = 'non-token card'
241​
242export function classify(
243 ours: Value | undefined,
244 theirs: Value | undefined,
245 options: { rounding?: boolean; currency?: string; prefix?: boolean } = {},
246): Status {
247 if (ours === undefined) return theirs === undefined ? 'neither' : 'onlyTheirs'
248 if (theirs === undefined) return 'onlyOurs'
249 // A month and a day in that month are one date at two precisions.
250 if (
251 options.prefix &&
252 typeof ours === 'string' &&
253 typeof theirs === 'string' &&
254 (ours.startsWith(theirs) || theirs.startsWith(ours))
255 ) {
256 return 'agree'
257 }
258 // Their prices are USD. Ours in another currency is a value we hold, but
259 // it is never converted, so the two are not compared.
260 if (options.currency !== undefined && options.currency !== 'USD') {
261 return 'nonUsd'
262 }
263 // Likewise a card that bills per request or per audio token: we hold a
264 // price, but there is no per-text-token rate to set beside theirs.
265 if (ours === NON_TOKEN) return 'nonToken'
266 if (ours === theirs) return 'agree'
267 if (
268 options.rounding &&
269 typeof ours === 'number' &&
270 typeof theirs === 'number' &&
271 Math.abs(ours - theirs) / Math.max(ours, theirs) <= ROUNDING_TOLERANCE
272 ) {
273 return 'agreeRounding'
274 }
275 return 'disagree'
277​
278const positive = (value: number | null | undefined): number | undefined =>
279 typeof value === 'number' && value > 0 ? value : undefined
280​
281/** Sorted and de-duplicated; their `pdf` and ours are both read as `file`. */
282function modalitySet(
283 list: Array<string> | null | undefined,
284): string | undefined {
285 if (!list?.length) return undefined
286 const names = list.map((name) => (name === 'pdf' ? 'file' : name))
287 return [...new Set(names)].sort().join(',')
289​
290function isModelsDev(row: OurRow): boolean {
291 try {
292 const { hostname } = new URL(row.pricing?.source?.url ?? '')
293 return hostname === 'models.dev' || hostname.endsWith('.models.dev')
294 } catch {
295 return false
296 }
298​
299/** Per-1M-token rate, 6 significant digits so float noise cannot disagree. */
300const perMillion = (perToken: number): number =>
301 Number((perToken * 1e6).toPrecision(6))
302​
303// Listing-compiled cards use OpenRouter's key names, parsed cards ours. A
304// price that itself came from models.dev would be compared with itself, so
305// it counts as no price (as in the gap report).
306function rate(row: OurRow, ...keys: Array<string>): number | undefined {
307 if (isModelsDev(row)) return undefined
308 const base = row.pricing?.tables?.rate?.base
309 for (const key of keys) {
310 const value = base?.[key]
311 if (typeof value === 'number') return perMillion(value)
312 }
313 return undefined
315​
316/** The card's currency; `unknown` for a card the evaluator cannot read. */
317function currency(row: OurRow): string | undefined {
318 if (!row.pricing) return undefined
319 try {
320 return cardCurrency(row.pricing as Parameters<typeof cardCurrency>[0])
321 } catch {
322 return 'unknown'
323 }
325​
326/**
327 * A capability flag, read from the row's map: `true` and `false` are what
328 * the provider states, and a missing key (or no map) is unknown, never a
329 * held `false`.
330 */
331function flag(row: OurRow, name: string): boolean | undefined {
332 const map = row.capabilities
333 return isCapabilityMap(map) ? map[name] : undefined
335​
336function ourPrice(row: OurRow): Value | undefined {
337 const tokens = pair(
338 rate(row, 'input_tokens', 'prompt'),
339 rate(row, 'output_tokens', 'completion'),
340 )
341 if (tokens !== undefined) return tokens
342 return row.pricing && !isModelsDev(row) ? NON_TOKEN : undefined
344​
345function ourReasoningOptions(row: OurRow): string | undefined {
346 const mode = row.reasoning?.mode
347 if (!mode) return undefined
348 // `adaptive` is effort-driven; they have no separate word for it.
349 if (mode !== 'effort' && mode !== 'adaptive') return mode
350 // An effort mode with no effort names states no options.
351 const efforts = row.reasoning?.efforts ?? []
352 return efforts.length > 0
353 ? `effort:${[...efforts].sort().join(',')}`
354 : undefined
356​
357function theirReasoningOptions(model: TheirModel): string | undefined {
358 if (!model.reasoning_options?.length) return undefined
359 return model.reasoning_options
360 .map((option) =>
361 option.type === 'effort'
362 ? `effort:${[...(option.values ?? [])].sort().join(',')}`
363 : option.type === 'budget_tokens'
364 ? 'budget'
365 : option.type,
366 )
367 .sort()
368 .join('+')
370​
371type Fact = {
372 label: string
373 /** The gap-report key whose ledger entry covers this fact, if any. */
374 ledger?: LedgerKey
375 rounding?: boolean
376 price?: boolean
377 /** Dates: equal when one is the other at a coarser precision. */
378 prefix?: boolean
379 ours: (row: OurRow) => Value | undefined
380 theirs: (model: TheirModel) => Value | undefined
382​
383function pair(a: number | undefined, b: number | undefined) {
384 return a === undefined || b === undefined ? undefined : `${a}/${b}`
386​
387export const FACTS = {
388 contextWindow: {
389 label: 'ctx',
390 ledger: 'contextWindow',
391 rounding: true,
392 ours: (row) => positive(row.contextWindow),
393 theirs: (model) => positive(model.limit?.context),
394 },
395 maxOutput: {
396 label: 'out',
397 ledger: 'maxOutput',
398 rounding: true,
399 ours: (row) => positive(row.maxOutput),
400 theirs: (model) => positive(model.limit?.output),
401 },
402 inputModalities: {
403 label: 'modIn',
404 ledger: 'modalities',
405 ours: (row) => modalitySet(row.modalities?.input),
406 theirs: (model) => modalitySet(model.modalities?.input),
407 },
408 outputModalities: {
409 label: 'modOut',
410 ledger: 'modalities',
411 ours: (row) => modalitySet(row.modalities?.output),
412 theirs: (model) => modalitySet(model.modalities?.output),
413 },
414 // "input/output", USD per 1M tokens.
415 priced: {
416 label: 'price',
417 ledger: 'priced',
418 price: true,
419 ours: ourPrice,
420 theirs: (model) => pair(model.cost?.input, model.cost?.output),
421 },
422 cacheRead: {
423 label: 'cRead',
424 ledger: 'cacheRead',
425 price: true,
426 ours: (row) => rate(row, 'cache_read_tokens', 'input_cache_read'),
427 theirs: (model) => model.cost?.cache_read,
428 },
429 cacheWrite: {
430 label: 'cWrite',
431 price: true,
432 ours: (row) => rate(row, 'cache_write_tokens', 'input_cache_write'),
433 theirs: (model) => model.cost?.cache_write,
434 },
435 tools: {
436 label: 'tools',
437 ledger: 'capabilities',
438 ours: (row) => flag(row, 'tools'),
439 theirs: (model) => model.tool_call,
440 },
441 structuredOutput: {
442 label: 'struct',
443 ledger: 'capabilities',
444 ours: (row) => flag(row, 'structured_outputs'),
445 theirs: (model) => model.structured_output,
446 },
447 reasoning: {
448 label: 'reason',
449 ledger: 'reasoning',
450 ours: (row) => (row.reasoning ? true : flag(row, 'reasoning')),
451 theirs: (model) => model.reasoning,
452 },
453 reasoningOptions: {
454 label: 'opts',
455 ledger: 'efforts',
456 ours: ourReasoningOptions,
457 theirs: theirReasoningOptions,
458 },
459 temperature: {
460 label: 'temp',
461 ledger: 'capabilities',
462 ours: (row) => flag(row, 'temperature'),
463 theirs: (model) => model.temperature,
464 },
465 // The provider's own date as a UTC day, beside their `YYYY-MM-DD`.
466 releasedAt: {
467 label: 'rel',
468 prefix: true,
469 ours: (row) =>
470 typeof row.releasedAt === 'number'
471 ? new Date(row.releasedAt * 1000).toISOString().slice(0, 10)
472 : undefined,
473 theirs: (model) => model.release_date || undefined,
474 },
475 knowledgeCutoff: {
476 label: 'know',
477 prefix: true,
478 ours: (row) => row.knowledgeCutoff ?? undefined,
479 theirs: (model) => model.knowledge || undefined,
480 },
481 openWeights: {
482 label: 'open',
483 ours: (row) => row.openWeights ?? undefined,
484 theirs: (model) => model.open_weights,
485 },
486} satisfies Record<string, Fact>
487​
488export type FactKey = keyof typeof FACTS
489export const FACT_KEYS = Object.keys(FACTS) as Array<FactKey>
490​
491/** Fields models.dev carries that our rows have no column for. */
492export const NO_FIELD = {
493 'limit.input': (model: TheirModel) => positive(model.limit?.input),
494 family: (model: TheirModel) => model.family,
496export type NoFieldKey = keyof typeof NO_FIELD
497const NO_FIELD_KEYS = Object.keys(NO_FIELD) as Array<NoFieldKey>
498​
499// ── report ──────────────────────────────────────────────────────────────
500​
501export type Cell = {
502 status: Status
503 ours?: Value
504 theirs?: Value
505 /** We are behind, and the ledger says the provider does not publish it. */
506 ledgered?: true
508​
509export type FactCounts = Record<Status, number> & { ledgered: number }
510​
511export type ProviderComparison = {
512 provider: string
513 /** models.dev provider ids; empty when it is not in models.dev. */
514 theirs: Array<string>
515 chat: number
516 matched: number
517 matchedBy: Record<MatchRule, number>
518 unmatchedOurs: Array<string>
519 /** Their chat-like models we have no row for at all. */
520 notListed: Array<string>
521 /** Their chat-like models we list, but not as a chat row. */
522 listedNotChat: Array<string>
523 /** Their unmatched ids that one of our chat rows already names as an alias. */
524 aliasOfChatRow: Array<string>
525 /** Our chat rows whose input modalities do not include `text`. */
526 inputWithoutText: number
527 /** Facts the source-silent ledger covers for this provider. */
528 silent: Array<FactKey>
529 facts: Record<FactKey, FactCounts>
530 noField: Record<NoFieldKey, number>
531 models: Array<{
532 ours: string
533 theirs: string
534 how: MatchRule
535 facts: Record<FactKey, Cell>
536 }>
538​
539export type Parity = { have: number; of: number; ratio: number }
540​
541export type Headline = {
542 matched: number
543 /** Of the fact-values models.dev has, how many we have too. */
544 oursOfTheirs: Parity
545 /** Of the fact-values we have, how many models.dev has too. */
546 theirsOfOurs: Parity
548​
549export type Comparison = {
550 generatedAt: string
551 note: string
552 headline: Headline
553 headlineExcludingLedgered: Headline
554 headlineExcludingZeroRates: Headline
555 /** Matched pairs where models.dev's rate is zero; each counts as a value. */
556 zeroRates: Record<PriceFactKey, number>
557 /** Our chat rows, matched or not, whose input modalities lack `text`. */
558 inputWithoutText: { total: number; byProvider: Record<string, number> }
559 noField: Record<NoFieldKey, number>
560 notInModelsDev: Array<string>
561 providers: Array<ProviderComparison>
563​
564export const NOTE =
565 'Compare-only: models.dev is never a source for this catalog. Its values ' +
566 'are not ground truth — "only they have it" (behind) means a value exists ' +
567 'there, not that a correct one does.'
568​
569const PRICE_FACTS = ['priced', 'cacheRead', 'cacheWrite'] as const
570type PriceFactKey = (typeof PRICE_FACTS)[number]
571​
572/** A models.dev rate of zero: often "plan-included" or "unknown", not free. */
573const isZeroRate = (key: FactKey, cell: Cell): boolean =>
574 (PRICE_FACTS as ReadonlyArray<string>).includes(key) &&
575 (cell.theirs === 0 || cell.theirs === '0/0')
576​
577// ponytail: "chat-like" is text out, minus embedding/rerank names; models.dev
578// has no activity field. Read the lists, not just the counts.
579function chatLike(model: TheirModel): boolean {
580 return (
581 (model.modalities?.output ?? []).includes('text') &&
582 !/embed|rerank/i.test(`${model.id} ${model.family ?? ''}`)
583 )
585​
586const BOTH: Array<Status> = [
587 'agree',
588 'agreeRounding',
589 'disagree',
590 'nonUsd',
591 'nonToken',
593​
594/**
595 * Parity both ways over matched pairs. `ledgered` drops the facts a
596 * provider's ledger covers; `zeroRates` drops the cells where models.dev's
597 * rate is zero.
598 */
599export function headline(
600 providers: Array<ProviderComparison>,
601 exclude: { ledgered?: boolean; zeroRates?: boolean } = {},
602): Headline {
603 let both = 0
604 let onlyOurs = 0
605 let onlyTheirs = 0
606 for (const provider of providers) {
607 for (const key of FACT_KEYS) {
608 if (exclude.ledgered && provider.silent.includes(key)) continue
609 for (const model of provider.models) {
610 const cell = model.facts[key]
611 if (exclude.zeroRates && isZeroRate(key, cell)) continue
612 if (BOTH.includes(cell.status)) both += 1
613 else if (cell.status === 'onlyOurs') onlyOurs += 1
614 else if (cell.status === 'onlyTheirs') onlyTheirs += 1
615 }
616 }
617 }
618 const parity = (have: number, of: number): Parity => ({
619 have,
620 of,
621 ratio: of === 0 ? 1 : have / of,
622 })
623 return {
624 matched: providers.reduce((sum, provider) => sum + provider.matched, 0),
625 oursOfTheirs: parity(both, both + onlyTheirs),
626 theirsOfOurs: parity(both, both + onlyOurs),
627 }
629​
630export const behind = (provider: ProviderComparison): number =>
631 FACT_KEYS.reduce((sum, key) => sum + provider.facts[key].onlyTheirs, 0)
632​
633function compareProvider(
634 provider: string,
635 rows: Array<OurRow>,
636 theirs: TheirCatalog,
637 ledger: Ledger,
638): ProviderComparison {
639 const theirIds = counterparts(provider, theirs)
640 // First listed provider wins an id both carry.
641 const theirModels = new Map<string, { provider: string; model: TheirModel }>()
642 for (const id of theirIds) {
643 for (const [modelId, model] of Object.entries(theirs[id]?.models ?? {})) {
644 if (!theirModels.has(modelId))
645 theirModels.set(modelId, { provider: id, model })
646 }
647 }
648​
649 const chat = rows.filter((row) => row.activity === 'chat')
650 const { matches, unmatched } = matchModels(chat, theirModels)
651 const silent = FACT_KEYS.filter((key) => {
652 const ledgerKey = (FACTS[key] as Fact).ledger
653 return ledgerKey !== undefined && ledger.get(provider)?.has(ledgerKey)
654 })
655​
656 const facts = Object.fromEntries(
657 FACT_KEYS.map((key) => [
658 key,
659 { ...Object.fromEntries(STATUSES.map((s) => [s, 0])), ledgered: 0 },
660 ]),
661 ) as Record<FactKey, FactCounts>
662 const noField = Object.fromEntries(
663 NO_FIELD_KEYS.map((key) => [key, 0]),
664 ) as Record<NoFieldKey, number>
665​
666 const models = matches.map((match) => {
667 const cells = {} as Record<FactKey, Cell>
668 for (const key of FACT_KEYS) {
669 const fact: Fact = FACTS[key]
670 const ours = fact.ours(match.row)
671 const their = fact.theirs(match.theirs)
672 const status = classify(ours, their, {
673 rounding: fact.rounding,
674 prefix: fact.prefix,
675 currency: fact.price ? currency(match.row) : undefined,
676 })
677 const ledgered = status === 'onlyTheirs' && silent.includes(key)
678 cells[key] = {
679 status,
680 ...(ours !== undefined && { ours }),
681 ...(their !== undefined && { theirs: their }),
682 ...(ledgered && { ledgered }),
683 }
684 facts[key][status] += 1
685 if (ledgered) facts[key].ledgered += 1
686 }
687 for (const key of NO_FIELD_KEYS) {
688 if (NO_FIELD[key](match.theirs) !== undefined) noField[key] += 1
689 }
690 return {
691 ours: match.row.rawId,
692 theirs: `${match.theirProvider}/${match.theirs.id}`,
693 how: match.how,
694 facts: cells,
695 }
696 })
697​
698 // Every id we list under this provider, chat or not, aliases included.
699 const listed = new Set(
700 rows.flatMap((row) => [row.rawId, ...(row.aliases ?? [])]).map(normaliseId),
701 )
702 // Their id for a model one of our chat rows already names: not a model we
703 // list as something other than chat, and not one we lack.
704 const chatAliases = new Set(chat.flatMap(aliasCandidates))
705 const claimed = new Set(matches.map((match) => match.theirs.id))
706 const unclaimed = [...theirModels]
707 .filter(([id, entry]) => !claimed.has(id) && chatLike(entry.model))
708 .map(([id]) => id)
709 .sort()
710 const extra = unclaimed.filter((id) => !chatAliases.has(id))
711​
712 return {
713 provider,
714 theirs: theirIds,
715 chat: chat.length,
716 matched: matches.length,
717 matchedBy: {
718 exact: matches.filter((match) => match.how === 'exact').length,
719 alias: matches.filter((match) => match.how === 'alias').length,
720 normalised: matches.filter((match) => match.how === 'normalised').length,
721 },
722 unmatchedOurs: unmatched.map((row) => row.rawId).sort(),
723 notListed: extra.filter((id) => !listed.has(normaliseId(id))),
724 listedNotChat: extra.filter((id) => listed.has(normaliseId(id))),
725 aliasOfChatRow: unclaimed.filter((id) => chatAliases.has(id)),
726 inputWithoutText: chat.filter(
727 (row) =>
728 row.modalities?.input?.length && !row.modalities.input.includes('text'),
729 ).length,
730 silent,
731 facts,
732 noField,
733 models,
734 }
736​
737/** Every provider of ours with chat rows, most "behind" first. */
738export function compare(
739 rows: Array<OurRow>,
740 theirs: TheirCatalog,
741 ledger: Ledger = new Map(),
742 now: Date = new Date(),
743): Comparison {
744 const byProvider = new Map<string, Array<OurRow>>()
745 for (const row of rows) {
746 byProvider.set(row.provider, [...(byProvider.get(row.provider) ?? []), row])
747 }
748 const providers = [...byProvider]
749 .filter(([, group]) => group.some((row) => row.activity === 'chat'))
750 .map(([provider, group]) =>
751 compareProvider(provider, group, theirs, ledger),
752 )
753 .sort(
754 (a, b) => behind(b) - behind(a) || a.provider.localeCompare(b.provider),
755 )
756​
757 const zeroRates = Object.fromEntries(
758 PRICE_FACTS.map((key) => [
759 key,
760 providers
761 .flatMap((provider) => provider.models)
762 .filter((model) => isZeroRate(key, model.facts[key])).length,
763 ]),
764 ) as Record<PriceFactKey, number>
765 const withoutText = providers.filter((p) => p.inputWithoutText > 0)
766​
767 return {
768 generatedAt: now.toISOString(),
769 note: `${NOTE} A models.dev rate of zero counts as a value: price 0/0 on ${zeroRates.priced} matched models, cache_read 0 on ${zeroRates.cacheRead}, cache_write 0 on ${zeroRates.cacheWrite}.`,
770 headline: headline(providers),
771 headlineExcludingLedgered: headline(providers, { ledgered: true }),
772 headlineExcludingZeroRates: headline(providers, { zeroRates: true }),
773 zeroRates,
774 inputWithoutText: {
775 total: withoutText.reduce((sum, p) => sum + p.inputWithoutText, 0),
776 byProvider: Object.fromEntries(
777 withoutText.map((p) => [p.provider, p.inputWithoutText]),
778 ),
779 },
780 noField: Object.fromEntries(
781 NO_FIELD_KEYS.map((key) => [
782 key,
783 providers.reduce((sum, provider) => sum + provider.noField[key], 0),
784 ]),
785 ) as Record<NoFieldKey, number>,
786 notInModelsDev: providers
787 .filter((provider) => provider.theirs.length === 0)
788 .map((provider) => provider.provider)
789 .sort(),
790 providers,
791 }
793​
794// ── output ──────────────────────────────────────────────────────────────
795​
796const percent = (parity: Parity) =>
797 `${parity.have} of ${parity.of} (${(parity.ratio * 100).toFixed(1)}%)`
798​
799function headlineLines(report: Comparison): Array<string> {
800 const { headline: all, headlineExcludingLedgered: scored } = report
801 const nonZero = report.headlineExcludingZeroRates
802 const { total, byProvider } = report.inputWithoutText
803 return [
804 `Across ${all.matched} matched chat models and ${FACT_KEYS.length} comparable facts:`,
805 ` we have ${percent(all.oursOfTheirs)} of the fact-values models.dev has`,
806 ` models.dev has ${percent(all.theirsOfOurs)} of the fact-values we have`,
807 ` excluding ledgered facts: we have ${percent(scored.oursOfTheirs)}; models.dev has ${percent(scored.theirsOfOurs)}`,
808 ` excluding models.dev zero rates: we have ${percent(nonZero.oursOfTheirs)}; models.dev has ${percent(nonZero.theirsOfOurs)}`,
809 `They have, we have no field: ${NO_FIELD_KEYS.map((key) => `${key} ${report.noField[key]}`).join(', ')}`,
810 `Not in models.dev: ${report.notInModelsDev.join(', ') || 'none'}`,
811 `Our chat rows whose input modalities lack "text": ${total}${
812 total
813 ? ` (${Object.entries(byProvider)
814 .map(([provider, count]) => `${provider} ${count}`)
815 .join(', ')})`
816 : ''
817 } — a catalog bug, compared as stored`,
818 report.note,
819 ]
821​
822const they = (counts: FactCounts) =>
823 counts.onlyTheirs + BOTH.reduce((sum, status) => sum + counts[status], 0)
824​
825/** One row per provider; a fact cell is `behind/they have`, `*` = ledgered. */
826export function formatTable(report: Comparison): string {
827 const header = [
828 'provider',
829 'models.dev',
830 'chat',
831 'match',
832 'noMatch',
833 'theyOnly',
834 ...FACT_KEYS.map((key) => FACTS[key].label),
835 ]
836 const factCells = (rows: Array<ProviderComparison>, silent: Array<FactKey>) =>
837 FACT_KEYS.map((key) => {
838 const sum = (pick: (counts: FactCounts) => number) =>
839 rows.reduce((total, row) => total + pick(row.facts[key]), 0)
840 return `${sum((c) => c.onlyTheirs)}/${sum(they)}${silent.includes(key) ? '*' : ''}`
841 })
842 const total = (pick: (provider: ProviderComparison) => number) =>
843 String(report.providers.reduce((sum, provider) => sum + pick(provider), 0))
844 const body = [
845 ...report.providers.map((p) =>
846 p.theirs.length === 0
847 ? [p.provider, 'not in models.dev', String(p.chat)]
848 : [
849 p.provider,
850 p.theirs.join('+'),
851 String(p.chat),
852 String(p.matched),
853 String(p.unmatchedOurs.length),
854 String(p.notListed.length),
855 ...factCells([p], p.silent),
856 ],
857 ),
858 [
859 'OVERALL',
860 '',
861 total((p) => p.chat),
862 total((p) => p.matched),
863 total((p) => (p.theirs.length ? p.unmatchedOurs.length : 0)),
864 total((p) => p.notListed.length),
865 ...factCells(report.providers, []),
866 ],
867 ]
868 const width = (i: number) =>
869 Math.max(...[header, ...body].map((line) => line[i]?.length ?? 0))
870 const table = [header, ...body].map((line) =>
871 line
872 .map((cell, i) =>
873 i < 2 ? cell.padEnd(width(i)) : cell.padStart(width(i)),
874 )
875 .join(' ')
876 .trimEnd(),
877 )
878 return [
879 ...table,
880 '',
881 'fact cells: behind/they have — matched pairs where only models.dev has a value, over those where it has one; * = ledgered (provider publishes nothing).',
882 'noMatch: our chat rows with no models.dev model; theyOnly: their chat-like models we do not list at all.',
883 ...headlineLines(report),
884 ].join('\n')
886​
887/** Per fact: where we are behind, and where the two disagree. */
888export function formatProvider(provider: ProviderComparison): string {
889 if (provider.theirs.length === 0) {
890 return `${provider.provider}: not in models.dev (${provider.chat} chat rows)`
891 }
892 const { exact, alias, normalised } = provider.matchedBy
893 const lines = [
894 `${provider.provider} ↔ ${provider.theirs.join('+')}: ${provider.chat} chat rows, ${provider.matched} matched (exact ${exact}, alias ${alias}, normalised ${normalised})`,
895 ]
896 for (const key of FACT_KEYS) {
897 const of = (status: Status) =>
898 provider.models.filter((model) => model.facts[key].status === status)
899 const behindRows = of('onlyTheirs')
900 const disagree = of('disagree')
901 const counts = provider.facts[key]
902 lines.push(
903 '',
904 `${key}${provider.silent.includes(key) ? ' (ledgered)' : ''}: agree ${counts.agree}, rounding ${counts.agreeRounding}, disagree ${counts.disagree}, non-USD ${counts.nonUsd}, non-token ${counts.nonToken}, only ours ${counts.onlyOurs}, only theirs ${counts.onlyTheirs}, neither ${counts.neither}`,
905 ...behindRows.map(
906 (model) =>
907 ` behind ${model.ours}: theirs ${model.facts[key].theirs}`,
908 ),
909 ...disagree.map(
910 (model) =>
911 ` disagree ${model.ours}: ours ${model.facts[key].ours} · theirs ${model.facts[key].theirs}`,
912 ),
913 )
914 }
915 const list = (title: string, ids: Array<string>) =>
916 lines.push('', `${title} (${ids.length}): ${ids.join(', ') || 'none'}`)
917 list('our chat rows with no models.dev model', provider.unmatchedOurs)
918 list('their chat-like models we do not list', provider.notListed)
919 list('their chat-like models we list as non-chat', provider.listedNotChat)
920 list('their ids that alias one of our chat rows', provider.aliasOfChatRow)
921 return lines.join('\n')
923​
924async function load<T>(file: string | undefined, url: string): Promise<T> {
925 if (file) return JSON.parse(readFileSync(file, 'utf8')) as T
926 const response = await fetch(url)
927 if (response.status === 429) {
928 throw new Error(
929 `GET ${url} → 429: rate limited. Stopping; rerun later, or pass a saved copy with --ours / --theirs.`,
930 )
931 }
932 if (!response.ok) throw new Error(`GET ${url} → ${response.status}`)
933 return (await response.json()) as T
935​
936export async function main(
937 argv: Array<string> = process.argv.slice(2),
938): Promise<number> {
939 const { values } = parseArgs({
940 args: argv,
941 options: {
942 base: { type: 'string', default: 'https://modelschemas.com' },
943 ours: { type: 'string' },
944 theirs: { type: 'string' },
945 table: { type: 'boolean', default: false },
946 json: { type: 'boolean', default: false },
947 provider: { type: 'string' },
948 check: { type: 'boolean', default: false },
949 min: { type: 'string', default: '1' },
950 },
951 })
952 const min = Number(values.min)
953 if (!(min >= 0 && min <= 1)) {
954 throw new Error(`--min must be between 0 and 1, got "${values.min}"`)
955 }
956​
957 const { models } = await load<{ models: Array<OurRow> }>(
958 values.ours,
959 new URL('/v1/models?pricing=1&limit=20000', values.base).href,
960 )
961 const theirs = await load<TheirCatalog>(values.theirs, THEIRS_URL)
962 const report = compare(models, theirs, readLedger())
963​
964 if (values.json) {
965 console.log(JSON.stringify(report, null, 2))
966 } else if (values.provider) {
967 const provider = report.providers.find(
968 (p) => p.provider === values.provider,
969 )
970 if (!provider) throw new Error(`no chat rows for "${values.provider}"`)
971 console.log(formatProvider(provider))
972 } else {
973 console.log(formatTable(report))
974 }
975​
976 const { ratio } = report.headline.oursOfTheirs
977 if (values.check && ratio < min) {
978 console.error(`parity ${ratio.toFixed(3)} is below --min ${min}`)
979 return 1
980 }
981 return 0
983​
984if (import.meta.main) {
985 process.exit(await main())