Skip to content
Source
Sign inSign up
TanStack/schemas
Open History to see the last commit

completeness.ts

318 lines10.1 KB

1/**
2 * Completeness: the share of the chat facts `@tanstack/ai-models` needs that
3 * a provider's rows carry. The one definition behind `bun run gap:report`
4 * (`scripts/gap-report.ts`) and `completeness` on `GET /v1/status`, so the
5 * CLI and the site cannot disagree. No Node or Workers dependencies.
6 *
7 * `docs/source-silent/<provider>.md` lists facts a provider does not publish;
8 * those are left out of that provider's score and listed under `silent`.
9 */
10import { isCapabilityMap, supportedFlags } from './capabilities.ts'
11import { modelLedgerScope, parseLedgerEntry } from './source-silence-ledger.ts'
12​
13export const FACT_KEYS = [
14 'contextWindow',
15 'maxOutput',
16 'modalities',
17 'priced',
18 'cacheRead',
19 'capabilities',
20 'reasoning',
21 'efforts',
22 'requestMap',
23 'endpoint',
24 'replayReasoningContent',
25] as const
26​
27export type FactKey = (typeof FACT_KEYS)[number]
28​
29/**
30 * Row shapes `@tanstack/ai-models` cannot read correctly, counted apart from
31 * the score (issue #323). The score counts a fact as filled; a flag says the
32 * stored value still leaves the catalog to guess.
33 */
34export const FLAG_KEYS = [
35 'mandatoryUnknown',
36 'reasoningUnknown',
37 'outputModalitiesEmpty',
38 'mandatoryWithNone',
39] as const
40​
41export type FlagKey = (typeof FLAG_KEYS)[number]
42​
43/** The fields of a `/v1/models` row the report reads. */
44export type ModelRow = {
45 provider: string
46 /** Exact native model id, never the slugged catalog id. */
47 rawId?: string
48 activity?: string | null
49 contextWindow?: number | null
50 maxOutput?: number | null
51 modalities?: { input?: unknown; output?: unknown } | null
52 pricing?: {
53 tables?: { rate?: Record<string, unknown> }
54 source?: { url?: string }
55 } | null
56 capabilities?: unknown
57 reasoning?: {
58 mode?: string
59 mandatory?: boolean | null
60 efforts?: Array<string>
61 } | null
62 requestMap?: unknown
63 schemaEndpointId?: string | null
65​
66export type ProviderReport = {
67 provider: string
68 rows: number
69 chat: number
70 noActivity: number
71 fromModelsDev: number
72 facts: Record<FactKey, { have: number; need: number }>
73 silent: Array<FactKey>
74 /** Applicable missing rows excluded by exact model-scoped evidence. */
75 modelSilent?: Partial<Record<FactKey, number>>
76 score: number
77 /** Raw ids of the chat rows each flag matches. */
78 flags: Record<FlagKey, Array<string>>
80​
81export type GapReport = {
82 generatedAt: string
83 providers: Array<ProviderReport>
85​
86/** Provider id or provider/exact-raw-id → facts its sources do not publish. */
87export type Ledger = Map<string, Set<FactKey>>
88​
89function isFactKey(value: string): value is FactKey {
90 return (FACT_KEYS as ReadonlyArray<string>).includes(value)
92​
93/** Ledger entries are list lines: `- <provider>: <fact> — <why>`. */
94export function parseLedger(markdown: string): Ledger {
95 const ledger: Ledger = new Map()
96 for (const line of markdown.split('\n')) {
97 const entry = parseLedgerEntry(line)
98 if (!entry) continue
99 const { scope, fact } = entry
100 if (!isFactKey(fact)) {
101 throw new Error(
102 `source-silent ledger: unknown fact "${fact}" in: ${line}`,
103 )
104 }
105 const facts = ledger.get(scope) ?? new Set<FactKey>()
106 facts.add(fact)
107 ledger.set(scope, facts)
108 }
109 return ledger
111​
112function isModelsDev(row: ModelRow): boolean {
113 const url = row.pricing?.source?.url
114 if (!url) return false
115 try {
116 const { hostname } = new URL(url)
117 return hostname === 'models.dev' || hostname.endsWith('.models.dev')
118 } catch {
119 return false
120 }
122​
123/**
124 * The rate rows a lever must appear in: the base row, or, for a card with
125 * no base (BytePlus off-peak/peak, picked by a required lever), every row.
126 * Empty when absent or a models.dev price.
127 */
128function rateRows(row: ModelRow): Array<Record<string, unknown>> {
129 if (isModelsDev(row)) return []
130 const rate = row.pricing?.tables?.rate
131 if (!rate) return []
132 const rows = 'base' in rate ? [rate.base] : Object.values(rate)
133 return rows.every((r) => typeof r === 'object' && r !== null)
134 ? (rows as Array<Record<string, unknown>>)
135 : []
137​
138// Listing-compiled cards use OpenRouter's key names, parsed cards ours.
139function hasRate(row: ModelRow, ...keys: Array<string>): boolean {
140 const rows = rateRows(row)
141 return (
142 rows.length > 0 &&
143 rows.every((r) => keys.some((key) => typeof r[key] === 'number'))
144 )
146​
147/** `need`: the fact applies to this row. `have`: the row carries it. */
148const FACTS: Record<
149 FactKey,
150 { need?: (row: ModelRow) => boolean; have: (row: ModelRow) => boolean }
151> = {
152 contextWindow: { have: (row) => row.contextWindow != null },
153 maxOutput: { have: (row) => row.maxOutput != null },
154 modalities: {
155 have: (row) => {
156 const input = row.modalities?.input
157 return Array.isArray(input) && input.length > 0
158 },
159 },
160 priced: {
161 have: (row) =>
162 hasRate(row, 'input_tokens', 'prompt') &&
163 hasRate(row, 'output_tokens', 'completion'),
164 },
165 cacheRead: {
166 have: (row) => hasRate(row, 'cache_read_tokens', 'input_cache_read'),
167 },
168 capabilities: { have: (row) => isCapabilityMap(row.capabilities) },
169 // Filled by any stored object, whatever its `mandatory`: null there means
170 // the source is silent on turning thinking off, which is a stored fact.
171 reasoning: {
172 need: (row) =>
173 row.reasoning != null ||
174 supportedFlags(row.capabilities).includes('reasoning'),
175 have: (row) => row.reasoning != null,
176 },
177 // Only `effort` and `adaptive` rows need effort names; a `toggle` or
178 // `budget` row needs none. `mandatory` plays no part: efforts with
179 // `mandatory: null` is filled, and the `mandatoryUnknown` flag counts it.
180 efforts: {
181 need: (row) =>
182 row.reasoning?.mode === 'effort' || row.reasoning?.mode === 'adaptive',
183 have: (row) => (row.reasoning?.efforts?.length ?? 0) > 0,
184 },
185 requestMap: { have: (row) => row.requestMap != null },
186 endpoint: { have: (row) => row.schemaEndpointId != null },
187 replayReasoningContent: {
188 need: (row) =>
189 row.reasoning != null ||
190 supportedFlags(row.capabilities).includes('reasoning'),
191 have: (row) =>
192 typeof row.requestMap === 'object' &&
193 row.requestMap !== null &&
194 !Array.isArray(row.requestMap) &&
195 typeof (row.requestMap as Record<string, unknown>)
196 .replayReasoningContent === 'boolean',
197 },
199​
200/**
201 * `reasoningUnknown` is the `reasoning` gap, so a `reasoning` ledger entry
202 * for the provider or the exact model clears it. No ledger entry clears
203 * the other flags: each is a stored value in the wrong shape.
204 */
205const FLAGS: Record<
206 FlagKey,
207 { silencedBy?: FactKey; match: (row: ModelRow) => boolean }
208> = {
209 // The catalog cannot tell if it can offer an off level.
210 mandatoryUnknown: {
211 match: (row) =>
212 row.reasoning?.mode === 'effort' && row.reasoning.mandatory === null,
213 },
214 // The catalog has to guess the mode and the levels.
215 reasoningUnknown: {
216 silencedBy: 'reasoning',
217 match: (row) =>
218 FACTS.reasoning.need?.(row) === true && !FACTS.reasoning.have(row),
219 },
220 // The catalog reads `[]` as unknown; an unknown list is stored as null.
221 outputModalitiesEmpty: {
222 match: (row) => {
223 const output = row.modalities?.output
224 return Array.isArray(output) && output.length === 0
225 },
226 },
227 // Thinking cannot be turned off, yet `none` turns it off.
228 mandatoryWithNone: {
229 match: (row) =>
230 row.reasoning?.mandatory === true &&
231 (row.reasoning.efforts?.includes('none') ?? false),
232 },
234​
235/** Facts filled and facts needed across a provider, ledgered ones left out. */
236export function scoredFacts(
237 report: Pick<ProviderReport, 'facts' | 'silent' | 'modelSilent'>,
238): {
239 have: number
240 need: number
241} {
242 let have = 0
243 let need = 0
244 for (const key of FACT_KEYS) {
245 if (report.silent.includes(key)) continue
246 have += report.facts[key].have
247 need += report.facts[key].need - (report.modelSilent?.[key] ?? 0)
248 }
249 return { have, need }
251​
252export function buildReport(
253 rows: Array<ModelRow>,
254 ledger: Ledger = new Map(),
255 now: Date = new Date(),
256): GapReport {
257 const byProvider = new Map<string, Array<ModelRow>>()
258 for (const row of rows) {
259 const group = byProvider.get(row.provider)
260 if (group) group.push(row)
261 else byProvider.set(row.provider, [row])
262 }
263​
264 const providers = [...byProvider].map(([provider, group]): ProviderReport => {
265 const chat = group.filter((row) => row.activity === 'chat')
266 const silent = FACT_KEYS.filter((key) => ledger.get(provider)?.has(key))
267 const modelSilent: Partial<Record<FactKey, number>> = {}
268 const modelLedgered = (row: ModelRow, key: FactKey) =>
269 row.rawId !== undefined &&
270 ledger.get(modelLedgerScope(provider, row.rawId))?.has(key)
271 const facts = Object.fromEntries(
272 FACT_KEYS.map((key) => {
273 const { need, have } = FACTS[key]
274 const needed = need ? chat.filter(need) : chat
275 const scoped = silent.includes(key)
276 ? 0
277 : needed.filter((row) => !have(row) && modelLedgered(row, key)).length
278 if (scoped > 0) modelSilent[key] = scoped
279 return [key, { have: needed.filter(have).length, need: needed.length }]
280 }),
281 ) as ProviderReport['facts']
282 const { have, need } = scoredFacts({ facts, silent, modelSilent })
283 const flags = Object.fromEntries(
284 FLAG_KEYS.map((key) => {
285 const { silencedBy, match } = FLAGS[key]
286 const isSilent = (row: ModelRow) =>
287 silencedBy !== undefined &&
288 (silent.includes(silencedBy) || modelLedgered(row, silencedBy))
289 return [
290 key,
291 chat
292 .filter((row) => match(row) && !isSilent(row))
293 .map((row) => row.rawId ?? ''),
294 ]
295 }),
296 ) as ProviderReport['flags']
297​
298 return {
299 provider,
300 rows: group.length,
301 chat: chat.length,
302 noActivity: group.filter((row) => row.activity == null).length,
303 fromModelsDev: group.filter(isModelsDev).length,
304 facts,
305 silent,
306 modelSilent,
307 // No chat rows is a gap, not a pass. Chat rows with every needed fact
308 // on the ledger have nothing left to fill.
309 score: need === 0 ? (chat.length > 0 ? 1 : 0) : have / need,
310 flags,
311 }
312 })
313​
314 providers.sort(
315 (a, b) => a.score - b.score || a.provider.localeCompare(b.provider),
316 )
317 return { generatedAt: now.toISOString(), providers }