Open History to see the last commit
1/**
2 * Gap report: which facts `@tanstack/ai-models` needs are still missing,
3 * per provider, over chat rows. One HTTP request; no per-model detail calls.
4 *
5 * bun run gap:report # JSON
6 * bun run gap:report --table # readable, worst score first
7 * bun run gap:report --base http://localhost:3100
8 * bun run gap:report --check --providers grok,mistral --target 0.9
9 *
10 * `docs/source-silent/<provider>.md` lists facts a provider does not publish; those
11 * are left out of that provider's score and listed under `silent`. `flags`
12 * lists rows whose stored shape is wrong; flags do not change the score.
13 */
14import { existsSync, readFileSync, readdirSync } from 'node:fs'
15import { join } from 'node:path'
16import { parseArgs } from 'node:util'
17
18import {
19 FACT_KEYS,
20 FLAG_KEYS,
21 buildReport,
22 parseLedger,
23} from '../src/lib/completeness.ts'
24import type { GapReport, Ledger, ModelRow } from '../src/lib/completeness.ts'
25
26// The scoring itself is shared with `GET /v1/status`.
27export * from '../src/lib/completeness.ts'
28
29const LEDGER_DIR = join(import.meta.dirname, '..', 'docs', 'source-silent')
30
31/** The on-disk ledger; the Worker bundles the same files (`source-silent.ts`). */
32export function readLedger(dir = LEDGER_DIR): Ledger | undefined {
33 if (!existsSync(dir)) return undefined
34 return parseLedger(
35 readdirSync(dir)
36 .filter((file) => file.endsWith('.md') && file !== 'README.md')
37 .map((file) => readFileSync(join(dir, file), 'utf8'))
38 .join('\n'),
39 )
40}
41
42/** Providers below `target`. With no `names`, every provider with chat rows. */
43export function failing(
44 report: GapReport,
45 target: number,
46 names?: Array<string>,
47): Array<string> {
48 if (!names) {
49 return report.providers
50 .filter((p) => p.chat > 0 && p.score < target)
51 .map((p) => p.provider)
52 }
53 const scores = new Map(report.providers.map((p) => [p.provider, p.score]))
54 // A named provider with no rows at all fails too.
55 return names.filter((name) => (scores.get(name) ?? 0) < target)
56}
57
58export function formatTable(report: GapReport): string {
59 const header = [
60 'provider',
61 'score',
62 'rows',
63 'chat',
64 'noAct',
65 'm.dev',
66 ...FACT_KEYS,
67 ...FLAG_KEYS,
68 ]
69 const body = report.providers.map((p) => [
70 p.provider,
71 p.score.toFixed(2),
72 String(p.rows),
73 String(p.chat),
74 String(p.noActivity),
75 String(p.fromModelsDev),
76 ...FACT_KEYS.map((key) =>
77 p.silent.includes(key)
78 ? 'silent'
79 : `${p.facts[key].have}/${p.facts[key].need}${p.modelSilent?.[key] ? ` (${p.modelSilent[key]} model-silent)` : ''}`,
80 ),
81 ...FLAG_KEYS.map((key) => String(p.flags[key].length)),
82 ])
83 const width = (i: number) =>
84 Math.max(...[header, ...body].map((line) => line[i]?.length ?? 0))
85 return [header, ...body]
86 .map((line) =>
87 line
88 .map((cell, i) =>
89 i === 0 ? cell.padEnd(width(i)) : cell.padStart(width(i)),
90 )
91 .join(' '),
92 )
93 .join('\n')
94}
95
96export async function main(
97 argv: Array<string> = process.argv.slice(2),
98): Promise<number> {
99 const { values } = parseArgs({
100 args: argv,
101 options: {
102 base: { type: 'string', default: 'https://modelschemas.com' },
103 table: { type: 'boolean', default: false },
104 check: { type: 'boolean', default: false },
105 providers: { type: 'string' },
106 target: { type: 'string', default: '1' },
107 },
108 })
109 const target = Number(values.target)
110 if (!(target >= 0 && target <= 1)) {
111 throw new Error(`--target must be between 0 and 1, got "${values.target}"`)
112 }
113
114 const url = new URL('/v1/models?pricing=1&limit=20000', values.base)
115 const response = await fetch(url)
116 if (!response.ok) throw new Error(`GET ${url.href} → ${response.status}`)
117 const { models } = (await response.json()) as { models: Array<ModelRow> }
118
119 const report = buildReport(models, readLedger())
120
121 console.log(
122 values.table ? formatTable(report) : JSON.stringify(report, null, 2),
123 )
124
125 if (!values.check) return 0
126 const names = values.providers?.split(',').filter(Boolean)
127 const below = failing(report, target, names)
128 if (below.length > 0) {
129 console.error(`below target ${target}: ${below.join(', ')}`)
130 return 1
131 }
132 return 0
133}
134
135if (import.meta.main) {
136 process.exit(await main())
137}