Open History to see the last commit
1/**
2 * Completeness: the share of the chat facts `@tanstack/ai-models` needs that
3 * a provider's rows carry. The one definition behind `bun run gap:report`
4 * (`scripts/gap-report.ts`) and `completeness` on `GET /v1/status`, so the
5 * CLI and the site cannot disagree. No Node or Workers dependencies.
6 *
7 * `docs/source-silent/<provider>.md` lists facts a provider does not publish;
8 * those are left out of that provider's score and listed under `silent`.
9 */
10import { isCapabilityMap, supportedFlags } from './capabilities.ts'
11import { modelLedgerScope, parseLedgerEntry } from './source-silence-ledger.ts'
12
13export const FACT_KEYS = [
14 'contextWindow',
15 'maxOutput',
16 'modalities',
17 'priced',
18 'cacheRead',
19 'capabilities',
20 'reasoning',
21 'efforts',
22 'requestMap',
23 'endpoint',
24 'replayReasoningContent',
25] as const
26
27export type FactKey = (typeof FACT_KEYS)[number]
28
29/** The fields of a `/v1/models` row the report reads. */
30export type ModelRow = {
31 provider: string
32 /** Exact native model id, never the slugged catalog id. */
33 rawId?: string
34 activity?: string | null
35 contextWindow?: number | null
36 maxOutput?: number | null
37 modalities?: { input?: unknown } | null
38 pricing?: {
39 tables?: { rate?: { base?: unknown } }
40 source?: { url?: string }
41 } | null
42 capabilities?: unknown
43 reasoning?: {
44 mode?: string
45 mandatory?: boolean | null
46 efforts?: Array<string>
47 } | null
48 requestMap?: unknown
49 schemaEndpointId?: string | null
50}
51
52export type ProviderReport = {
53 provider: string
54 rows: number
55 chat: number
56 noActivity: number
57 fromModelsDev: number
58 facts: Record<FactKey, { have: number; need: number }>
59 silent: Array<FactKey>
60 /** Applicable missing rows excluded by exact model-scoped evidence. */
61 modelSilent?: Partial<Record<FactKey, number>>
62 score: number
63}
64
65export type GapReport = {
66 generatedAt: string
67 providers: Array<ProviderReport>
68}
69
70/** Provider id or provider/exact-raw-id → facts its sources do not publish. */
71export type Ledger = Map<string, Set<FactKey>>
72
73function isFactKey(value: string): value is FactKey {
74 return (FACT_KEYS as ReadonlyArray<string>).includes(value)
75}
76
77/** Ledger entries are list lines: `- <provider>: <fact> — <why>`. */
78export function parseLedger(markdown: string): Ledger {
79 const ledger: Ledger = new Map()
80 for (const line of markdown.split('\n')) {
81 const entry = parseLedgerEntry(line)
82 if (!entry) continue
83 const { scope, fact } = entry
84 if (!isFactKey(fact)) {
85 throw new Error(
86 `source-silent ledger: unknown fact "${fact}" in: ${line}`,
87 )
88 }
89 const facts = ledger.get(scope) ?? new Set<FactKey>()
90 facts.add(fact)
91 ledger.set(scope, facts)
92 }
93 return ledger
94}
95
96function isModelsDev(row: ModelRow): boolean {
97 const url = row.pricing?.source?.url
98 if (!url) return false
99 try {
100 const { hostname } = new URL(url)
101 return hostname === 'models.dev' || hostname.endsWith('.models.dev')
102 } catch {
103 return false
104 }
105}
106
107/** The base rate row, or null when absent or a models.dev price. */
108function baseRate(row: ModelRow): Record<string, unknown> | null {
109 if (isModelsDev(row)) return null
110 const base = row.pricing?.tables?.rate?.base
111 return typeof base === 'object' && base !== null
112 ? (base as Record<string, unknown>)
113 : null
114}
115
116// Listing-compiled cards use OpenRouter's key names, parsed cards ours.
117function hasRate(row: ModelRow, ...keys: Array<string>): boolean {
118 const base = baseRate(row)
119 return base !== null && keys.some((key) => typeof base[key] === 'number')
120}
121
122/** `need`: the fact applies to this row. `have`: the row carries it. */
123const FACTS: Record<
124 FactKey,
125 { need?: (row: ModelRow) => boolean; have: (row: ModelRow) => boolean }
126> = {
127 contextWindow: { have: (row) => row.contextWindow != null },
128 maxOutput: { have: (row) => row.maxOutput != null },
129 modalities: {
130 have: (row) => {
131 const input = row.modalities?.input
132 return Array.isArray(input) && input.length > 0
133 },
134 },
135 priced: {
136 have: (row) =>
137 hasRate(row, 'input_tokens', 'prompt') &&
138 hasRate(row, 'output_tokens', 'completion'),
139 },
140 cacheRead: {
141 have: (row) => hasRate(row, 'cache_read_tokens', 'input_cache_read'),
142 },
143 capabilities: { have: (row) => isCapabilityMap(row.capabilities) },
144 // Filled by any stored object, whatever its `mandatory`: null there means
145 // the source is silent on turning thinking off, which is a stored fact.
146 reasoning: {
147 need: (row) =>
148 row.reasoning != null ||
149 supportedFlags(row.capabilities).includes('reasoning'),
150 have: (row) => row.reasoning != null,
151 },
152 // Only `effort` and `adaptive` rows need effort names; a `toggle` or
153 // `budget` row needs none. `mandatory` plays no part: efforts with
154 // `mandatory: null` is filled.
155 efforts: {
156 need: (row) =>
157 row.reasoning?.mode === 'effort' || row.reasoning?.mode === 'adaptive',
158 have: (row) => (row.reasoning?.efforts?.length ?? 0) > 0,
159 },
160 requestMap: { have: (row) => row.requestMap != null },
161 endpoint: { have: (row) => row.schemaEndpointId != null },
162 replayReasoningContent: {
163 need: (row) =>
164 row.reasoning != null ||
165 supportedFlags(row.capabilities).includes('reasoning'),
166 have: (row) =>
167 typeof row.requestMap === 'object' &&
168 row.requestMap !== null &&
169 !Array.isArray(row.requestMap) &&
170 typeof (row.requestMap as Record<string, unknown>)
171 .replayReasoningContent === 'boolean',
172 },
173}
174
175/** Facts filled and facts needed across a provider, ledgered ones left out. */
176export function scoredFacts(
177 report: Pick<ProviderReport, 'facts' | 'silent' | 'modelSilent'>,
178): {
179 have: number
180 need: number
181} {
182 let have = 0
183 let need = 0
184 for (const key of FACT_KEYS) {
185 if (report.silent.includes(key)) continue
186 have += report.facts[key].have
187 need += report.facts[key].need - (report.modelSilent?.[key] ?? 0)
188 }
189 return { have, need }
190}
191
192export function buildReport(
193 rows: Array<ModelRow>,
194 ledger: Ledger = new Map(),
195 now: Date = new Date(),
196): GapReport {
197 const byProvider = new Map<string, Array<ModelRow>>()
198 for (const row of rows) {
199 const group = byProvider.get(row.provider)
200 if (group) group.push(row)
201 else byProvider.set(row.provider, [row])
202 }
203
204 const providers = [...byProvider].map(([provider, group]): ProviderReport => {
205 const chat = group.filter((row) => row.activity === 'chat')
206 const silent = FACT_KEYS.filter((key) => ledger.get(provider)?.has(key))
207 const modelSilent: Partial<Record<FactKey, number>> = {}
208 const facts = Object.fromEntries(
209 FACT_KEYS.map((key) => {
210 const { need, have } = FACTS[key]
211 const needed = need ? chat.filter(need) : chat
212 const scoped = silent.includes(key)
213 ? 0
214 : needed.filter(
215 (row) =>
216 !have(row) &&
217 row.rawId !== undefined &&
218 ledger.get(modelLedgerScope(provider, row.rawId))?.has(key),
219 ).length
220 if (scoped > 0) modelSilent[key] = scoped
221 return [key, { have: needed.filter(have).length, need: needed.length }]
222 }),
223 ) as ProviderReport['facts']
224 const { have, need } = scoredFacts({ facts, silent, modelSilent })
225
226 return {
227 provider,
228 rows: group.length,
229 chat: chat.length,
230 noActivity: group.filter((row) => row.activity == null).length,
231 fromModelsDev: group.filter(isModelsDev).length,
232 facts,
233 silent,
234 modelSilent,
235 // No chat rows is a gap, not a pass. Chat rows with every needed fact
236 // on the ledger have nothing left to fill.
237 score: need === 0 ? (chat.length > 0 ? 1 : 0) : have / need,
238 }
239 })
240
241 providers.sort(
242 (a, b) => a.score - b.score || a.provider.localeCompare(b.provider),
243 )
244 return { generatedAt: now.toISOString(), providers }
245}