Open History to see the last commit
1import { describe, expect, it } from 'vitest'
2
3import { sourceSilentLedger } from '../src/server/source-silent.ts'
4import {
5 buildReport,
6 failing,
7 parseLedger,
8 readLedger,
9 formatTable,
10} from './gap-report.ts'
11import type { ModelRow } from './gap-report.ts'
12
13const complete: ModelRow = {
14 provider: 'full',
15 activity: 'chat',
16 contextWindow: 200000,
17 maxOutput: 64000,
18 modalities: { input: ['text'] },
19 pricing: {
20 tables: {
21 rate: {
22 base: {
23 input_tokens: 1e-6,
24 output_tokens: 2e-6,
25 cache_read_tokens: 1e-7,
26 },
27 },
28 },
29 source: { url: 'https://docs.example.com/pricing' },
30 },
31 capabilities: { reasoning: true, tools: true },
32 reasoning: { mode: 'effort', efforts: ['low', 'high'] },
33 requestMap: { maxTokensField: 'max_tokens', replayReasoningContent: false },
34 schemaEndpointId: 'full/chat/completions',
35}
36
37// Listing-compiled cards carry OpenRouter's key names.
38const listingTables = {
39 rate: { base: { prompt: 1e-6, completion: 2e-6, input_cache_read: 1e-7 } },
40}
41
42const rows: Array<ModelRow> = [
43 complete,
44 {
45 ...complete,
46 provider: 'borrowed',
47 pricing: {
48 tables: listingTables,
49 source: { url: 'https://models.dev/api.json' },
50 },
51 },
52 { ...complete, provider: 'quiet', maxOutput: null },
53 { provider: 'images', activity: 'image' },
54 { provider: 'images', activity: null },
55]
56
57const ledger = parseLedger(
58 '# Ledger\n\n- quiet: maxOutput — not published (#75)\n',
59)
60const report = buildReport(rows, ledger, new Date(0))
61const provider = (id: string, from = report) => {
62 const found = from.providers.find((p) => p.provider === id)
63 if (!found) throw new Error(`no provider ${id}`)
64 return found
65}
66
67describe('gap report scoring', () => {
68 it('scores a complete row 1', () => {
69 const full = provider('full')
70 expect(full.score).toBe(1)
71 expect(full.facts.priced).toEqual({ have: 1, need: 1 })
72 expect(full.facts.efforts).toEqual({ have: 1, need: 1 })
73 expect(report.generatedAt).toBe('1970-01-01T00:00:00.000Z')
74 })
75
76 it('does not count a models.dev price as filled', () => {
77 const borrowed = provider('borrowed')
78 expect(borrowed.fromModelsDev).toBe(1)
79 expect(borrowed.facts.priced).toEqual({ have: 0, need: 1 })
80 expect(borrowed.facts.cacheRead).toEqual({ have: 0, need: 1 })
81 expect(borrowed.score).toBeCloseTo(9 / 11)
82 })
83
84 it('reads both rate key spellings', () => {
85 const own = buildReport([
86 { ...complete, pricing: { tables: listingTables } },
87 ])
88 expect(provider('full', own).score).toBe(1)
89 })
90
91 it('counts a card in another currency as priced', () => {
92 const yuan = buildReport([
93 {
94 ...complete,
95 pricing: {
96 ...complete.pricing,
97 price: { currency: ['CNY', 0] },
98 } as ModelRow['pricing'],
99 },
100 ])
101 expect(provider('full', yuan).facts.priced).toEqual({ have: 1, need: 1 })
102 expect(provider('full', yuan).facts.cacheRead).toEqual({ have: 1, need: 1 })
103 })
104
105 it('leaves a ledger fact out of the score', () => {
106 const quiet = provider('quiet')
107 expect(quiet.silent).toEqual(['maxOutput'])
108 expect(quiet.facts.maxOutput).toEqual({ have: 0, need: 1 })
109 expect(quiet.score).toBe(1)
110 })
111
112 it('scores chat rows 1 when every needed fact is on the ledger', () => {
113 const all = parseLedger(
114 [
115 'contextWindow',
116 'maxOutput',
117 'modalities',
118 'priced',
119 'cacheRead',
120 'capabilities',
121 'requestMap',
122 'endpoint',
123 ]
124 .map((fact) => `- bare: ${fact} — not published`)
125 .join('\n'),
126 )
127 const bare = buildReport(
128 [{ provider: 'bare', activity: 'chat' }],
129 all,
130 new Date(0),
131 )
132 expect(bare.providers[0]).toMatchObject({ chat: 1, score: 1 })
133 })
134
135 it('treats rows with no chat rows as a gap', () => {
136 const images = provider('images')
137 expect(images).toMatchObject({ rows: 2, chat: 0, noActivity: 1, score: 0 })
138 expect(report.providers[0]?.provider).toBe('images')
139 })
140
141 it('needs reasoning only where a row claims it', () => {
142 const plain = provider(
143 'full',
144 buildReport([
145 {
146 ...complete,
147 capabilities: { tools: true, reasoning: false },
148 reasoning: null,
149 },
150 { ...complete, reasoning: null },
151 ]),
152 )
153 expect(plain.facts.reasoning).toEqual({ have: 0, need: 1 })
154 expect(plain.facts.efforts).toEqual({ have: 0, need: 0 })
155 })
156
157 it('asks a toggle row for reasoning and for no efforts', () => {
158 const toggles = provider(
159 'full',
160 buildReport([
161 { ...complete, reasoning: { mode: 'toggle', mandatory: false } },
162 { ...complete, reasoning: { mode: 'toggle', mandatory: true } },
163 ]),
164 )
165 expect(toggles.facts.reasoning).toEqual({ have: 2, need: 2 })
166 expect(toggles.facts.efforts).toEqual({ have: 0, need: 0 })
167 expect(toggles.score).toBe(1)
168 })
169
170 it('counts efforts with an unstated mandatory as filled', () => {
171 const unstated = provider(
172 'full',
173 buildReport([
174 {
175 ...complete,
176 reasoning: { mode: 'effort', mandatory: null, efforts: ['low'] },
177 },
178 ]),
179 )
180 expect(unstated.facts.reasoning).toEqual({ have: 1, need: 1 })
181 expect(unstated.facts.efforts).toEqual({ have: 1, need: 1 })
182 expect(unstated.score).toBe(1)
183 })
184
185 it('picks the providers below target', () => {
186 expect(failing(report, 1)).toEqual(['borrowed'])
187 expect(failing(report, 1, ['full', 'images', 'absent'])).toEqual([
188 'images',
189 'absent',
190 ])
191 expect(failing(report, 0.8)).toEqual([])
192 })
193
194 it('rejects an unknown ledger fact', () => {
195 expect(() => parseLedger('- grok: maxOutputs — typo')).toThrow(
196 /unknown fact/,
197 )
198 })
199})
200
201describe('source-silent ledger', () => {
202 it('is the same bundled into the Worker as read from disk', () => {
203 const onDisk = readLedger()
204 expect(onDisk?.size).toBeGreaterThan(0)
205 expect(sourceSilentLedger).toEqual(onDisk)
206 })
207})
208
209describe('model-scoped source silence', () => {
210 it('matches exact native ids with embedded slashes and keeps published facts counted', () => {
211 const scoped = parseLedger(
212 '- host/maker/model: maxOutput — own docs silent\n- host/maker/filled: maxOutput — no longer silent',
213 )
214 const result = provider(
215 'host',
216 buildReport(
217 [
218 {
219 ...complete,
220 provider: 'host',
221 rawId: 'maker/model',
222 maxOutput: null,
223 },
224 {
225 ...complete,
226 provider: 'host',
227 rawId: 'maker/model-snapshot',
228 maxOutput: null,
229 },
230 { ...complete, provider: 'host', rawId: 'maker/filled' },
231 ],
232 scoped,
233 ),
234 )
235 expect(result.silent).toEqual([])
236 expect(result.facts.maxOutput).toEqual({ have: 1, need: 3 })
237 expect(result.modelSilent).toEqual({ maxOutput: 1 })
238 expect(result.score).toBeCloseTo(31 / 32)
239 expect(formatTable({ generatedAt: '', providers: [result] })).toContain(
240 '1/3 (1 model-silent)',
241 )
242 })
243 it('counts both explicit replay booleans, not null or missing fields', () => {
244 const replay = provider(
245 'host',
246 buildReport(
247 [
248 {
249 ...complete,
250 provider: 'host',
251 rawId: 'a',
252 requestMap: { replayReasoningContent: true },
253 },
254 {
255 ...complete,
256 provider: 'host',
257 rawId: 'b',
258 requestMap: { replayReasoningContent: false },
259 },
260 {
261 ...complete,
262 provider: 'host',
263 rawId: 'c',
264 requestMap: { replayReasoningContent: null },
265 },
266 { ...complete, provider: 'host', rawId: 'd', requestMap: null },
267 {
268 ...complete,
269 provider: 'host',
270 rawId: 'e',
271 reasoning: null,
272 capabilities: { reasoning: false },
273 },
274 ],
275 parseLedger('- host/c: replayReasoningContent — checked own source'),
276 ),
277 )
278 expect(replay.facts.replayReasoningContent).toEqual({ have: 2, need: 4 })
279 expect(replay.modelSilent).toEqual({ replayReasoningContent: 1 })
280 })
281 it('does not turn a model-scoped entry into a provider exemption or guess missing raw ids', () => {
282 const scoped = parseLedger('- host/maker/model: maxOutput — absent')
283 const result = provider(
284 'host',
285 buildReport([{ ...complete, provider: 'host', maxOutput: null }], scoped),
286 )
287 expect(result.modelSilent).toEqual({})
288 expect(result.score).toBeLessThan(1)
289 })
290 it('preserves provider-wide grammar and avoids double exclusions', () => {
291 const scoped = parseLedger(
292 '- host: maxOutput — absent\n- `host/@cf/a/model:variant`: maxOutput — absent',
293 )
294 expect(scoped.get('host/@cf/a/model:variant')).toEqual(
295 new Set(['maxOutput']),
296 )
297 const result = provider(
298 'host',
299 buildReport(
300 [
301 {
302 ...complete,
303 provider: 'host',
304 rawId: '@cf/a/model:variant',
305 maxOutput: null,
306 },
307 ],
308 scoped,
309 ),
310 )
311 expect(result.silent).toEqual(['maxOutput'])
312 expect(result.modelSilent).toEqual({})
313 expect(result.score).toBe(1)
314 expect(() => parseLedger('- host/: maxOutput — missing model')).toThrow(
315 'invalid scope',
316 )
317 expect(() => parseLedger('- host/model: ')).toThrow('malformed entry')
318 expect(() => parseLedger('- host/model: `maxOutput')).toThrow(
319 'malformed entry',
320 )
321 expect(() =>
322 parseLedger(
323 '- host/id: requestMap.replayReasoningContent — unsupported spelling',
324 ),
325 ).toThrow('unknown fact')
326 })
327})