Open History to see the last commit
1import { describe, expect, it } from 'vitest'
2
3import { sourceSilentLedger } from '../src/server/source-silent.ts'
4import {
5 buildReport,
6 failing,
7 parseLedger,
8 readLedger,
9 formatTable,
10} from './gap-report.ts'
11import type { ModelRow } from './gap-report.ts'
12
13const complete: ModelRow = {
14 provider: 'full',
15 activity: 'chat',
16 contextWindow: 200000,
17 maxOutput: 64000,
18 modalities: { input: ['text'] },
19 pricing: {
20 tables: {
21 rate: {
22 base: {
23 input_tokens: 1e-6,
24 output_tokens: 2e-6,
25 cache_read_tokens: 1e-7,
26 },
27 },
28 },
29 source: { url: 'https://docs.example.com/pricing' },
30 },
31 capabilities: { reasoning: true, tools: true },
32 reasoning: { mode: 'effort', efforts: ['low', 'high'] },
33 requestMap: { maxTokensField: 'max_tokens', replayReasoningContent: false },
34 schemaEndpointId: 'full/chat/completions',
35}
36
37// Listing-compiled cards carry OpenRouter's key names.
38const listingTables = {
39 rate: { base: { prompt: 1e-6, completion: 2e-6, input_cache_read: 1e-7 } },
40}
41
42const rows: Array<ModelRow> = [
43 complete,
44 {
45 ...complete,
46 provider: 'borrowed',
47 pricing: {
48 tables: listingTables,
49 source: { url: 'https://models.dev/api.json' },
50 },
51 },
52 { ...complete, provider: 'quiet', maxOutput: null },
53 { provider: 'images', activity: 'image' },
54 { provider: 'images', activity: null },
55]
56
57const ledger = parseLedger(
58 '# Ledger\n\n- quiet: maxOutput — not published (#75)\n',
59)
60const report = buildReport(rows, ledger, new Date(0))
61const provider = (id: string, from = report) => {
62 const found = from.providers.find((p) => p.provider === id)
63 if (!found) throw new Error(`no provider ${id}`)
64 return found
65}
66
67describe('gap report scoring', () => {
68 it('scores a complete row 1', () => {
69 const full = provider('full')
70 expect(full.score).toBe(1)
71 expect(full.facts.priced).toEqual({ have: 1, need: 1 })
72 expect(full.facts.efforts).toEqual({ have: 1, need: 1 })
73 expect(report.generatedAt).toBe('1970-01-01T00:00:00.000Z')
74 })
75
76 it('does not count a models.dev price as filled', () => {
77 const borrowed = provider('borrowed')
78 expect(borrowed.fromModelsDev).toBe(1)
79 expect(borrowed.facts.priced).toEqual({ have: 0, need: 1 })
80 expect(borrowed.facts.cacheRead).toEqual({ have: 0, need: 1 })
81 expect(borrowed.score).toBeCloseTo(9 / 11)
82 })
83
84 it('reads both rate key spellings', () => {
85 const own = buildReport([
86 { ...complete, pricing: { tables: listingTables } },
87 ])
88 expect(provider('full', own).score).toBe(1)
89 })
90
91 it('counts a card with no base row only when every row prices the lever', () => {
92 const rate = {
93 off_peak: {
94 input_tokens: 1e-7,
95 output_tokens: 4e-7,
96 cache_read_tokens: 1e-9,
97 },
98 peak: { input_tokens: 2e-7, output_tokens: 8e-7 },
99 }
100 const periods = provider(
101 'full',
102 buildReport([{ ...complete, pricing: { tables: { rate } } }]),
103 )
104 expect(periods.facts.priced).toEqual({ have: 1, need: 1 })
105 expect(periods.facts.cacheRead).toEqual({ have: 0, need: 1 })
106 })
107
108 it('counts a card in another currency as priced', () => {
109 const yuan = buildReport([
110 {
111 ...complete,
112 pricing: {
113 ...complete.pricing,
114 price: { currency: ['CNY', 0] },
115 } as ModelRow['pricing'],
116 },
117 ])
118 expect(provider('full', yuan).facts.priced).toEqual({ have: 1, need: 1 })
119 expect(provider('full', yuan).facts.cacheRead).toEqual({ have: 1, need: 1 })
120 })
121
122 it('leaves a ledger fact out of the score', () => {
123 const quiet = provider('quiet')
124 expect(quiet.silent).toEqual(['maxOutput'])
125 expect(quiet.facts.maxOutput).toEqual({ have: 0, need: 1 })
126 expect(quiet.score).toBe(1)
127 })
128
129 it('scores chat rows 1 when every needed fact is on the ledger', () => {
130 const all = parseLedger(
131 [
132 'contextWindow',
133 'maxOutput',
134 'modalities',
135 'priced',
136 'cacheRead',
137 'capabilities',
138 'requestMap',
139 'endpoint',
140 ]
141 .map((fact) => `- bare: ${fact} — not published`)
142 .join('\n'),
143 )
144 const bare = buildReport(
145 [{ provider: 'bare', activity: 'chat' }],
146 all,
147 new Date(0),
148 )
149 expect(bare.providers[0]).toMatchObject({ chat: 1, score: 1 })
150 })
151
152 it('treats rows with no chat rows as a gap', () => {
153 const images = provider('images')
154 expect(images).toMatchObject({ rows: 2, chat: 0, noActivity: 1, score: 0 })
155 expect(report.providers[0]?.provider).toBe('images')
156 })
157
158 it('needs reasoning only where a row claims it', () => {
159 const plain = provider(
160 'full',
161 buildReport([
162 {
163 ...complete,
164 capabilities: { tools: true, reasoning: false },
165 reasoning: null,
166 },
167 { ...complete, reasoning: null },
168 ]),
169 )
170 expect(plain.facts.reasoning).toEqual({ have: 0, need: 1 })
171 expect(plain.facts.efforts).toEqual({ have: 0, need: 0 })
172 })
173
174 it('asks a toggle row for reasoning and for no efforts', () => {
175 const toggles = provider(
176 'full',
177 buildReport([
178 { ...complete, reasoning: { mode: 'toggle', mandatory: false } },
179 { ...complete, reasoning: { mode: 'toggle', mandatory: true } },
180 ]),
181 )
182 expect(toggles.facts.reasoning).toEqual({ have: 2, need: 2 })
183 expect(toggles.facts.efforts).toEqual({ have: 0, need: 0 })
184 expect(toggles.score).toBe(1)
185 })
186
187 it('counts efforts with an unstated mandatory as filled', () => {
188 const unstated = provider(
189 'full',
190 buildReport([
191 {
192 ...complete,
193 reasoning: { mode: 'effort', mandatory: null, efforts: ['low'] },
194 },
195 ]),
196 )
197 expect(unstated.facts.reasoning).toEqual({ have: 1, need: 1 })
198 expect(unstated.facts.efforts).toEqual({ have: 1, need: 1 })
199 expect(unstated.score).toBe(1)
200 })
201
202 it('flags no shape on a complete row', () => {
203 expect(provider('full').flags).toEqual({
204 mandatoryUnknown: [],
205 reasoningUnknown: [],
206 outputModalitiesEmpty: [],
207 mandatoryWithNone: [],
208 })
209 })
210
211 it('flags the shapes the catalog cannot read, apart from the score', () => {
212 const flagged = provider(
213 'full',
214 buildReport([
215 {
216 ...complete,
217 rawId: 'unstated',
218 reasoning: { mode: 'effort', mandatory: null, efforts: ['low'] },
219 },
220 {
221 ...complete,
222 rawId: 'adaptive-unstated',
223 reasoning: { mode: 'adaptive', mandatory: null },
224 },
225 { ...complete, rawId: 'no-mode', reasoning: null },
226 {
227 ...complete,
228 rawId: 'not-reasoning',
229 capabilities: { reasoning: false },
230 reasoning: null,
231 },
232 {
233 ...complete,
234 rawId: 'no-output',
235 modalities: { input: ['text'], output: [] },
236 },
237 {
238 ...complete,
239 rawId: 'unknown-output',
240 modalities: { input: ['text'], output: null },
241 },
242 {
243 ...complete,
244 rawId: 'forced-none',
245 reasoning: {
246 mode: 'effort',
247 mandatory: true,
248 efforts: ['none', 'high'],
249 },
250 },
251 {
252 ...complete,
253 rawId: 'image',
254 activity: 'image',
255 modalities: { output: [] },
256 },
257 ]),
258 )
259 expect(flagged.flags).toEqual({
260 mandatoryUnknown: ['unstated'],
261 reasoningUnknown: ['no-mode'],
262 outputModalitiesEmpty: ['no-output'],
263 mandatoryWithNone: ['forced-none'],
264 })
265 expect(flagged.facts.efforts).toEqual({ have: 4, need: 5 })
266 expect(formatTable({ generatedAt: '', providers: [flagged] })).toContain(
267 'mandatoryWithNone',
268 )
269 })
270
271 it('clears reasoningUnknown only for a reasoning ledger entry', () => {
272 const hostRows: Array<ModelRow> = [
273 { ...complete, provider: 'host', rawId: 'a', reasoning: null },
274 {
275 ...complete,
276 provider: 'host',
277 rawId: 'b',
278 reasoning: null,
279 modalities: { input: ['text'], output: [] },
280 },
281 { ...complete, provider: 'host', rawId: 'c', reasoning: null },
282 ]
283 const scoped = provider(
284 'host',
285 buildReport(
286 hostRows,
287 parseLedger(
288 '- host/a: reasoning — own docs silent\n- host/b: modalities — own docs silent',
289 ),
290 ),
291 )
292 expect(scoped.flags.reasoningUnknown).toEqual(['b', 'c'])
293 expect(scoped.flags.outputModalitiesEmpty).toEqual(['b'])
294 const wide = provider(
295 'host',
296 buildReport(hostRows, parseLedger('- host: reasoning — not published')),
297 )
298 expect(wide.flags.reasoningUnknown).toEqual([])
299 })
300
301 it('picks the providers below target', () => {
302 expect(failing(report, 1)).toEqual(['borrowed'])
303 expect(failing(report, 1, ['full', 'images', 'absent'])).toEqual([
304 'images',
305 'absent',
306 ])
307 expect(failing(report, 0.8)).toEqual([])
308 })
309
310 it('rejects an unknown ledger fact', () => {
311 expect(() => parseLedger('- grok: maxOutputs — typo')).toThrow(
312 /unknown fact/,
313 )
314 })
315})
316
317describe('source-silent ledger', () => {
318 it('is the same bundled into the Worker as read from disk', () => {
319 const onDisk = readLedger()
320 expect(onDisk?.size).toBeGreaterThan(0)
321 expect(sourceSilentLedger).toEqual(onDisk)
322 })
323})
324
325describe('model-scoped source silence', () => {
326 it('matches exact native ids with embedded slashes and keeps published facts counted', () => {
327 const scoped = parseLedger(
328 '- host/maker/model: maxOutput — own docs silent\n- host/maker/filled: maxOutput — no longer silent',
329 )
330 const result = provider(
331 'host',
332 buildReport(
333 [
334 {
335 ...complete,
336 provider: 'host',
337 rawId: 'maker/model',
338 maxOutput: null,
339 },
340 {
341 ...complete,
342 provider: 'host',
343 rawId: 'maker/model-snapshot',
344 maxOutput: null,
345 },
346 { ...complete, provider: 'host', rawId: 'maker/filled' },
347 ],
348 scoped,
349 ),
350 )
351 expect(result.silent).toEqual([])
352 expect(result.facts.maxOutput).toEqual({ have: 1, need: 3 })
353 expect(result.modelSilent).toEqual({ maxOutput: 1 })
354 expect(result.score).toBeCloseTo(31 / 32)
355 expect(formatTable({ generatedAt: '', providers: [result] })).toContain(
356 '1/3 (1 model-silent)',
357 )
358 })
359 it('counts both explicit replay booleans, not null or missing fields', () => {
360 const replay = provider(
361 'host',
362 buildReport(
363 [
364 {
365 ...complete,
366 provider: 'host',
367 rawId: 'a',
368 requestMap: { replayReasoningContent: true },
369 },
370 {
371 ...complete,
372 provider: 'host',
373 rawId: 'b',
374 requestMap: { replayReasoningContent: false },
375 },
376 {
377 ...complete,
378 provider: 'host',
379 rawId: 'c',
380 requestMap: { replayReasoningContent: null },
381 },
382 { ...complete, provider: 'host', rawId: 'd', requestMap: null },
383 {
384 ...complete,
385 provider: 'host',
386 rawId: 'e',
387 reasoning: null,
388 capabilities: { reasoning: false },
389 },
390 ],
391 parseLedger('- host/c: replayReasoningContent — checked own source'),
392 ),
393 )
394 expect(replay.facts.replayReasoningContent).toEqual({ have: 2, need: 4 })
395 expect(replay.modelSilent).toEqual({ replayReasoningContent: 1 })
396 })
397 it('does not turn a model-scoped entry into a provider exemption or guess missing raw ids', () => {
398 const scoped = parseLedger('- host/maker/model: maxOutput — absent')
399 const result = provider(
400 'host',
401 buildReport([{ ...complete, provider: 'host', maxOutput: null }], scoped),
402 )
403 expect(result.modelSilent).toEqual({})
404 expect(result.score).toBeLessThan(1)
405 })
406 it('preserves provider-wide grammar and avoids double exclusions', () => {
407 const scoped = parseLedger(
408 '- host: maxOutput — absent\n- `host/@cf/a/model:variant`: maxOutput — absent',
409 )
410 expect(scoped.get('host/@cf/a/model:variant')).toEqual(
411 new Set(['maxOutput']),
412 )
413 const result = provider(
414 'host',
415 buildReport(
416 [
417 {
418 ...complete,
419 provider: 'host',
420 rawId: '@cf/a/model:variant',
421 maxOutput: null,
422 },
423 ],
424 scoped,
425 ),
426 )
427 expect(result.silent).toEqual(['maxOutput'])
428 expect(result.modelSilent).toEqual({})
429 expect(result.score).toBe(1)
430 expect(() => parseLedger('- host/: maxOutput — missing model')).toThrow(
431 'invalid scope',
432 )
433 expect(() => parseLedger('- host/model: ')).toThrow('malformed entry')
434 expect(() => parseLedger('- host/model: `maxOutput')).toThrow(
435 'malformed entry',
436 )
437 expect(() =>
438 parseLedger(
439 '- host/id: requestMap.replayReasoningContent — unsupported spelling',
440 ),
441 ).toThrow('unknown fact')
442 })
443})