Skip to content
Source
Sign inSign up
TanStack/schemas
Open History to see the last commit

gap-report.ts

129 lines4.0 KB

1/**
2 * Gap report: which facts `@tanstack/ai-models` needs are still missing,
3 * per provider, over chat rows. One HTTP request; no per-model detail calls.
4 *
5 * bun run gap:report # JSON
6 * bun run gap:report --table # readable, worst score first
7 * bun run gap:report --base http://localhost:3100
8 * bun run gap:report --check --providers grok,mistral --target 0.9
9 *
10 * `docs/source-silent/<provider>.md` lists facts a provider does not publish; those
11 * are left out of that provider's score and listed under `silent`.
12 */
13import { existsSync, readFileSync, readdirSync } from 'node:fs'
14import { join } from 'node:path'
15import { parseArgs } from 'node:util'
16​
17import { FACT_KEYS, buildReport, parseLedger } from '../src/lib/completeness.ts'
18import type { GapReport, Ledger, ModelRow } from '../src/lib/completeness.ts'
19​
20// The scoring itself is shared with `GET /v1/status`.
21export * from '../src/lib/completeness.ts'
22​
23const LEDGER_DIR = join(import.meta.dirname, '..', 'docs', 'source-silent')
24​
25/** The on-disk ledger; the Worker bundles the same files (`source-silent.ts`). */
26export function readLedger(dir = LEDGER_DIR): Ledger | undefined {
27 if (!existsSync(dir)) return undefined
28 return parseLedger(
29 readdirSync(dir)
30 .filter((file) => file.endsWith('.md') && file !== 'README.md')
31 .map((file) => readFileSync(join(dir, file), 'utf8'))
32 .join('\n'),
33 )
35​
36/** Providers below `target`. With no `names`, every provider with chat rows. */
37export function failing(
38 report: GapReport,
39 target: number,
40 names?: Array<string>,
41): Array<string> {
42 if (!names) {
43 return report.providers
44 .filter((p) => p.chat > 0 && p.score < target)
45 .map((p) => p.provider)
46 }
47 const scores = new Map(report.providers.map((p) => [p.provider, p.score]))
48 // A named provider with no rows at all fails too.
49 return names.filter((name) => (scores.get(name) ?? 0) < target)
51​
52export function formatTable(report: GapReport): string {
53 const header = [
54 'provider',
55 'score',
56 'rows',
57 'chat',
58 'noAct',
59 'm.dev',
60 ...FACT_KEYS,
61 ]
62 const body = report.providers.map((p) => [
63 p.provider,
64 p.score.toFixed(2),
65 String(p.rows),
66 String(p.chat),
67 String(p.noActivity),
68 String(p.fromModelsDev),
69 ...FACT_KEYS.map((key) =>
70 p.silent.includes(key)
71 ? 'silent'
72 : `${p.facts[key].have}/${p.facts[key].need}${p.modelSilent?.[key] ? ` (${p.modelSilent[key]} model-silent)` : ''}`,
73 ),
74 ])
75 const width = (i: number) =>
76 Math.max(...[header, ...body].map((line) => line[i]?.length ?? 0))
77 return [header, ...body]
78 .map((line) =>
79 line
80 .map((cell, i) =>
81 i === 0 ? cell.padEnd(width(i)) : cell.padStart(width(i)),
82 )
83 .join(' '),
84 )
85 .join('\n')
87​
88export async function main(
89 argv: Array<string> = process.argv.slice(2),
90): Promise<number> {
91 const { values } = parseArgs({
92 args: argv,
93 options: {
94 base: { type: 'string', default: 'https://modelschemas.com' },
95 table: { type: 'boolean', default: false },
96 check: { type: 'boolean', default: false },
97 providers: { type: 'string' },
98 target: { type: 'string', default: '1' },
99 },
100 })
101 const target = Number(values.target)
102 if (!(target >= 0 && target <= 1)) {
103 throw new Error(`--target must be between 0 and 1, got "${values.target}"`)
104 }
105​
106 const url = new URL('/v1/models?pricing=1&limit=20000', values.base)
107 const response = await fetch(url)
108 if (!response.ok) throw new Error(`GET ${url.href} → ${response.status}`)
109 const { models } = (await response.json()) as { models: Array<ModelRow> }
110​
111 const report = buildReport(models, readLedger())
112​
113 console.log(
114 values.table ? formatTable(report) : JSON.stringify(report, null, 2),
115 )
116​
117 if (!values.check) return 0
118 const names = values.providers?.split(',').filter(Boolean)
119 const below = failing(report, target, names)
120 if (below.length > 0) {
121 console.error(`below target ${target}: ${below.join(', ')}`)
122 return 1
123 }
124 return 0
126​
127if (import.meta.main) {
128 process.exit(await main())