Open History to see the last commit
1import { describe, expect, it, vi } from 'vitest'
2import { env } from 'cloudflare:test'
3import { eq } from 'drizzle-orm'
4
5import { getDb } from '../db/index.ts'
6import {
7 cacheMeta,
8 endpoints,
9 models,
10 providers,
11 schemaVersions,
12} from '../db/schema.ts'
13import { FACT_KEYS, buildReport } from '../lib/completeness.ts'
14import type { Ledger, ModelRow } from '../lib/completeness.ts'
15import { providerRegistry } from '../server/providers/index.ts'
16import { listModelsCatalog } from './catalog.ts'
17import { getServiceStatus, recordCompleteness } from './status.ts'
18
19const NOW = 1_781_150_000
20
21describe('getServiceStatus', () => {
22 it('reports per-provider sync state and counts', async () => {
23 const db = getDb(env)
24 await db.insert(providers).values([
25 {
26 id: 'status-a',
27 displayName: 'Status A',
28 specSourceUrl: 'https://example.com/a.json',
29 lastPolledAt: NOW,
30 lastSyncedAt: NOW - 100,
31 },
32 {
33 id: 'status-b',
34 displayName: 'Status B',
35 specSourceUrl: 'https://example.com/b.json',
36 status: 'degraded',
37 },
38 ])
39 await db.insert(models).values([
40 {
41 id: 'status-a-model',
42 providerId: 'status-a',
43 rawId: 'model',
44 firstSeenAt: NOW,
45 lastSeenAt: NOW,
46 },
47 {
48 // A stored card counts as priced; a null column does not.
49 id: 'status-a-priced',
50 providerId: 'status-a',
51 rawId: 'priced',
52 pricing: { inputs: {}, tables: {}, price: 1, examples: [] },
53 firstSeenAt: NOW,
54 lastSeenAt: NOW,
55 },
56 {
57 // Chat without reasoning metadata: denominator only.
58 id: 'status-a-chat',
59 providerId: 'status-a',
60 rawId: 'chat',
61 activity: 'chat',
62 firstSeenAt: NOW,
63 lastSeenAt: NOW,
64 },
65 {
66 id: 'status-a-reasoner',
67 providerId: 'status-a',
68 rawId: 'reasoner',
69 activity: 'chat',
70 reasoning: { mode: 'effort', mandatory: false },
71 firstSeenAt: NOW,
72 lastSeenAt: NOW,
73 },
74 ])
75 await db.insert(endpoints).values({
76 id: 'status-a/v1/things',
77 providerId: 'status-a',
78 activity: 'chat',
79 method: 'POST',
80 path: '/v1/things',
81 })
82 await db.insert(schemaVersions).values([
83 {
84 id: 'status-a/v1/things:input:current',
85 endpointId: 'status-a/v1/things',
86 kind: 'input',
87 contentHash: 'c'.repeat(64),
88 schema: '{}',
89 createdAt: NOW,
90 },
91 {
92 // Superseded version must not count.
93 id: 'status-a/v1/things:input:old',
94 endpointId: 'status-a/v1/things',
95 kind: 'input',
96 contentHash: 'd'.repeat(64),
97 schema: '{}',
98 createdAt: NOW - 10,
99 supersededAt: NOW,
100 },
101 ])
102
103 const status = await getServiceStatus(db, NOW)
104 expect(status.service).toBe('modelschemas')
105 const a = status.providers.find((p) => p.id === 'status-a')
106 const b = status.providers.find((p) => p.id === 'status-b')
107 expect(a).toMatchObject({
108 status: 'active',
109 lastPolledAt: NOW,
110 lastSyncedAt: NOW - 100,
111 counts: {
112 models: 4,
113 priced: 1,
114 reasoning: 1,
115 chat: 2,
116 endpoints: 1,
117 schemas: 1,
118 },
119 })
120 expect(b).toMatchObject({
121 status: 'degraded',
122 lastPolledAt: null,
123 counts: {
124 models: 0,
125 priced: 0,
126 reasoning: 0,
127 chat: 0,
128 endpoints: 0,
129 schemas: 0,
130 },
131 })
132 })
133
134 it('lists every registered provider even before its first sync', async () => {
135 const db = getDb(env)
136 const status = await getServiceStatus(db, NOW)
137 // Nothing is seeded in the test DB, so every registry provider must
138 // surface as pending — the full roster is always visible.
139 for (const config of providerRegistry) {
140 const listed = status.providers.find((p) => p.id === config.id)
141 expect(listed).toMatchObject({
142 displayName: config.displayName,
143 status: 'pending',
144 counts: {
145 models: 0,
146 priced: 0,
147 reasoning: 0,
148 chat: 0,
149 endpoints: 0,
150 schemas: 0,
151 },
152 })
153 }
154 })
155
156 it('carries docs-failing and refused-clears records, and only those', async () => {
157 const db = getDb(env)
158 const ids = [
159 'rec-none',
160 'rec-docs',
161 'rec-corrupt',
162 'rec-clears',
163 'rec-storm',
164 ]
165 await db.insert(providers).values(
166 ids.map((id) => ({
167 id,
168 displayName: id,
169 specSourceUrl: 'https://example.com/spec.json',
170 })),
171 )
172 const docsFailing = {
173 since: NOW - 1800,
174 polls: 3,
175 lastAt: NOW,
176 failed: 2,
177 skipped: 1,
178 sources: ['https://example.com/docs/a', 'https://example.com/docs/b'],
179 error: 'page changed shape',
180 }
181 const priceClearsRefused = {
182 since: NOW - 900,
183 polls: 2,
184 lastAt: NOW,
185 refused: 6,
186 priced: 8,
187 }
188 const updateStorm = {
189 since: NOW - 60,
190 polls: 1,
191 lastAt: NOW,
192 updated: 21,
193 live: 30,
194 }
195 const row = (key: string, lastError: string | null) => ({
196 key,
197 fetchedAt: NOW,
198 staleTime: 0,
199 lastError,
200 })
201 await db.insert(cacheMeta).values([
202 row('docs-failing:rec-docs', JSON.stringify(docsFailing)),
203 row(
204 'price-clears-refused:rec-clears',
205 JSON.stringify(priceClearsRefused),
206 ),
207 row('update-storm:rec-storm', JSON.stringify(updateStorm)),
208 // Not JSON, JSON of the wrong shape, and no body at all.
209 row('docs-failing:rec-corrupt', '{not json'),
210 row('price-clears-refused:rec-corrupt', '"a string"'),
211 row('docs-failing:rec-none', null),
212 // Another cache_meta row is not a record.
213 row('schema:rec-none', JSON.stringify(docsFailing)),
214 ])
215
216 const status = await getServiceStatus(db, NOW)
217 const [none, docs, corrupt, clears, storm] = ids.map((id) =>
218 status.providers.find((p) => p.id === id),
219 )
220 expect(docs?.docsFailing).toEqual(docsFailing)
221 expect(docs).not.toHaveProperty('priceClearsRefused')
222 expect(docs).not.toHaveProperty('updateStorm')
223 expect(clears?.priceClearsRefused).toEqual(priceClearsRefused)
224 expect(clears).not.toHaveProperty('docsFailing')
225 expect(storm?.updateStorm).toEqual(updateStorm)
226 expect(storm).not.toHaveProperty('docsFailing')
227 for (const healthy of [none, corrupt]) {
228 expect(healthy).toMatchObject({ status: 'active' })
229 expect(healthy).not.toHaveProperty('docsFailing')
230 expect(healthy).not.toHaveProperty('priceClearsRefused')
231 expect(healthy).not.toHaveProperty('updateStorm')
232 }
233 // Neither record moves `status`.
234 expect(docs?.status).toBe('active')
235 expect(clears?.status).toBe('active')
236 })
237
238 const provider = (id: string) => ({
239 id,
240 displayName: id,
241 specSourceUrl: 'https://example.com/spec.json',
242 })
243 const model = (
244 providerId: string,
245 rawId: string,
246 facts: Partial<typeof models.$inferInsert> = {},
247 ) => ({
248 id: `${providerId}-${rawId}`,
249 providerId,
250 rawId,
251 activity: 'chat' as const,
252 firstSeenAt: NOW,
253 lastSeenAt: NOW,
254 ...facts,
255 })
256 const card = {
257 inputs: {},
258 tables: { rate: { base: { input_tokens: 1e-6, output_tokens: 2e-6 } } },
259 price: 1,
260 examples: [],
261 source: {
262 url: 'https://docs.example.com/pricing',
263 hash: 'a'.repeat(64),
264 extractedAt: '2026-01-01T00:00:00.000Z',
265 },
266 }
267
268 it('counts live rows only, and deprecated rows apart', async () => {
269 const db = getDb(env)
270 await db.insert(providers).values([provider('gone'), provider('part')])
271 const retired = { deprecatedAt: NOW - 10, pricing: card }
272 await db
273 .insert(models)
274 .values([
275 model('gone', 'a', retired),
276 model('gone', 'b', { ...retired, reasoning: { mode: 'toggle' } }),
277 model('part', 'old', retired),
278 model('part', 'live', { pricing: card }),
279 model('part', 'image', { activity: 'image' }),
280 ])
281
282 await recordCompleteness(db, NOW)
283 const status = await getServiceStatus(db, NOW)
284 const gone = status.providers.find((p) => p.id === 'gone')
285 const part = status.providers.find((p) => p.id === 'part')
286 // Every row retired: what `/v1/models?provider=gone` lists is nothing.
287 expect(gone?.counts).toMatchObject({
288 models: 0,
289 priced: 0,
290 reasoning: 0,
291 chat: 0,
292 deprecated: 2,
293 })
294 expect(gone?.completeness).toMatchObject({ score: null, chat: 0 })
295 expect(part?.counts).toMatchObject({
296 models: 2,
297 priced: 1,
298 reasoning: 0,
299 chat: 1,
300 deprecated: 1,
301 })
302 expect(part?.completeness.chat).toBe(1)
303 expect((await listModelsCatalog(db, { provider: 'part' })).count).toBe(
304 part?.counts.models,
305 )
306 })
307
308 it('scores completeness as the gap report does', async () => {
309 const db = getDb(env)
310 const ids = ['no-chat', 'all-silent', 'mixed']
311 await db.insert(providers).values(ids.map(provider))
312 await db.insert(models).values([
313 model('no-chat', 'image', { activity: 'image', pricing: card }),
314 model('all-silent', 'bare'),
315 model('mixed', 'full', {
316 contextWindow: 200_000,
317 maxOutput: 64_000,
318 modalities: { input: ['text'], output: ['text'] },
319 pricing: card,
320 capabilities: { reasoning: true, tools: true },
321 reasoning: { mode: 'effort', efforts: ['low', 'high'] },
322 requestMap: {
323 maxTokensField: 'max_tokens',
324 replayReasoningContent: false,
325 },
326 schemaEndpointId: 'v1/chat/completions',
327 }),
328 // Claims reasoning and stores none; a card that does not parse is
329 // served as null, so it is no price here either.
330 model('mixed', 'thin', {
331 contextWindow: 8000,
332 capabilities: { reasoning: true },
333 pricing: { tables: card.tables },
334 }),
335 // A models.dev price does not count.
336 model('mixed', 'borrowed', {
337 pricing: {
338 ...card,
339 source: { ...card.source, url: 'https://models.dev/api.json' },
340 },
341 }),
342 // Retired and of another activity: neither is scored.
343 model('mixed', 'retired', { deprecatedAt: NOW, contextWindow: 1 }),
344 model('mixed', 'image', { activity: 'image' }),
345 ])
346 const ledger: Ledger = new Map([
347 ['all-silent', new Set(FACT_KEYS)],
348 ['mixed', new Set(['cacheRead'] as const)],
349 ])
350
351 // Nothing scored yet (no record, whatever an earlier test stored): no
352 // score anywhere, and the request still answers.
353 await db.delete(cacheMeta)
354 const before = await getServiceStatus(db, NOW)
355 expect(before.completenessComputedAt).toBeNull()
356 expect(before.providers.every((p) => p.completeness.score === null)).toBe(
357 true,
358 )
359
360 expect(await recordCompleteness(db, NOW - 60, ledger)).toBe(true)
361 const status = await getServiceStatus(db, NOW)
362 expect(status.completenessComputedAt).toBe(NOW - 60)
363 const [noChat, allSilent, mixed] = ids.map(
364 (id) => status.providers.find((p) => p.id === id)?.completeness,
365 )
366 // Nothing to score is not a score of zero.
367 expect(noChat).toEqual({
368 score: null,
369 chat: 0,
370 filled: 0,
371 needed: 0,
372 silent: [],
373 })
374 // Chat rows with every fact on the ledger have nothing left to fill.
375 expect(allSilent).toEqual({
376 score: 1,
377 chat: 1,
378 filled: 0,
379 needed: 0,
380 silent: [...FACT_KEYS],
381 })
382 // full: 10 of 10. thin: contextWindow, capabilities of 9. borrowed: 0 of 7.
383 expect(mixed).toEqual({
384 score: 12 / 26,
385 chat: 3,
386 filled: 12,
387 needed: 26,
388 silent: ['cacheRead'],
389 })
390
391 // The same rows as `bun run gap:report` reads them, off the API.
392 const served = await listModelsCatalog(db, { pricing: true })
393 const report = buildReport(served.models as Array<ModelRow>, ledger)
394 for (const id of ['all-silent', 'mixed']) {
395 const scored = report.providers.find((p) => p.provider === id)
396 const listed = status.providers.find((p) => p.id === id)
397 expect(listed?.completeness.score).toBe(scored?.score)
398 expect(listed?.completeness.chat).toBe(scored?.chat)
399 }
400 })
401
402 const KEY = 'status:completeness:v1'
403 const scoreOf = async (db: ReturnType<typeof getDb>, id: string) => {
404 const status = await getServiceStatus(db, NOW)
405 return {
406 computedAt: status.completenessComputedAt,
407 ...status.providers.find((p) => p.id === id)?.completeness,
408 }
409 }
410
411 it('serves the stored score only while it matches the live chat rows', async () => {
412 const db = getDb(env)
413 await db.insert(providers).values(provider('stored'))
414 await db.insert(models).values(model('stored', 'a'))
415 await recordCompleteness(db, NOW)
416 expect(await scoreOf(db, 'stored')).toMatchObject({
417 computedAt: NOW,
418 score: 0,
419 chat: 1,
420 needed: 8,
421 })
422
423 // A chat row the record never scored: no number over a different set.
424 await db.insert(models).values(model('stored', 'b'))
425 expect(await scoreOf(db, 'stored')).toMatchObject({
426 computedAt: NOW,
427 score: null,
428 chat: 0,
429 })
430 // The next poll's record scores both, and says when.
431 await recordCompleteness(db, NOW + 900)
432 expect(await scoreOf(db, 'stored')).toMatchObject({
433 computedAt: NOW + 900,
434 score: 0,
435 chat: 2,
436 })
437 // No live chat row left: null at once, before any rescore.
438 await db.update(models).set({ deprecatedAt: NOW })
439 expect(await scoreOf(db, 'stored')).toMatchObject({ score: null, chat: 0 })
440 })
441
442 it('answers with null scores when the record cannot be used', async () => {
443 const db = getDb(env)
444 await db.insert(providers).values(provider('broken'))
445 await db.insert(models).values(model('broken', 'a'))
446 await recordCompleteness(db, NOW)
447 expect(await scoreOf(db, 'broken')).toMatchObject({ score: 0 })
448 const none = { computedAt: null, score: null, chat: 0 }
449 const errors = vi.spyOn(console, 'error').mockImplementation(() => {})
450
451 // Corrupt JSON, JSON of the wrong shape, an entry of the wrong shape.
452 for (const lastError of ['{not json', '"a string"', 'null']) {
453 await db
454 .update(cacheMeta)
455 .set({ lastError })
456 .where(eq(cacheMeta.key, KEY))
457 expect(await scoreOf(db, 'broken')).toMatchObject(none)
458 }
459 // Entries that are not a whole `Completeness`, though the chat count
460 // matches the live one: never served, and no usable record at all.
461 for (const entry of [
462 7,
463 { chat: 1 },
464 { score: 'high', chat: 1, filled: 0, needed: 8, silent: 'x' },
465 { score: 0, chat: 1, filled: 0, needed: 8, silent: [1] },
466 ]) {
467 await db
468 .update(cacheMeta)
469 .set({ lastError: JSON.stringify({ broken: entry }) })
470 .where(eq(cacheMeta.key, KEY))
471 expect(await scoreOf(db, 'broken')).toEqual({
472 computedAt: null,
473 score: null,
474 chat: 0,
475 filled: 0,
476 needed: 0,
477 silent: [],
478 })
479 }
480 // An array is not a record. One bad entry does not cost the good ones.
481 await db
482 .update(cacheMeta)
483 .set({ lastError: '[{"chat":1}]' })
484 .where(eq(cacheMeta.key, KEY))
485 expect(await scoreOf(db, 'broken')).toMatchObject(none)
486 const good = { score: 0.5, chat: 1, filled: 4, needed: 8, silent: [] }
487 await db
488 .update(cacheMeta)
489 .set({ lastError: JSON.stringify({ broken: good, other: { chat: 1 } }) })
490 .where(eq(cacheMeta.key, KEY))
491 expect(await scoreOf(db, 'broken')).toEqual({ computedAt: NOW, ...good })
492
493 // Deleted.
494 await db.delete(cacheMeta).where(eq(cacheMeta.key, KEY))
495 expect(await scoreOf(db, 'broken')).toMatchObject(none)
496
497 // The read itself throws: the rest of the status is still served.
498 await recordCompleteness(db, NOW)
499 vi.spyOn(db.query.cacheMeta, 'findFirst').mockImplementation(() => {
500 throw new Error('D1 down')
501 })
502 const status = await getServiceStatus(db, NOW)
503 expect(status.completenessComputedAt).toBeNull()
504 expect(status.providers.find((p) => p.id === 'broken')).toMatchObject({
505 counts: { models: 1, chat: 1 },
506 completeness: { score: null },
507 })
508 expect(errors).toHaveBeenCalled()
509 vi.restoreAllMocks()
510 })
511
512 it('keeps the previous record when scoring fails', async () => {
513 const db = getDb(env)
514 await db.insert(providers).values(provider('kept'))
515 await db.insert(models).values(model('kept', 'a'))
516 await recordCompleteness(db, NOW)
517 const errors = vi.spyOn(console, 'error').mockImplementation(() => {})
518 const throwing = {
519 get: () => {
520 throw new Error('scorer broke')
521 },
522 } as unknown as Ledger
523 // Never throws: the cron that calls it goes on to its next step.
524 expect(await recordCompleteness(db, NOW + 900, throwing)).toBe(false)
525 expect(errors).toHaveBeenCalledWith(
526 JSON.stringify({ job: 'completeness', error: 'scorer broke' }),
527 )
528 vi.restoreAllMocks()
529 expect(await scoreOf(db, 'kept')).toMatchObject({
530 computedAt: NOW,
531 score: 0,
532 chat: 1,
533 })
534 })
535})
536
537it('scores only the exact raw model covered by a scoped silence entry', async () => {
538 const db = getDb(env)
539 await db.insert(providers).values({
540 id: 'scoped-status',
541 displayName: 'Scoped status',
542 specSourceUrl: 'https://example.com/docs',
543 })
544 await db.insert(models).values([
545 {
546 id: 'scoped-status-exact',
547 providerId: 'scoped-status',
548 rawId: 'maker/model:variant',
549 activity: 'chat',
550 firstSeenAt: NOW,
551 lastSeenAt: NOW,
552 },
553 {
554 id: 'scoped-status-sibling',
555 providerId: 'scoped-status',
556 rawId: 'maker/model:variant-snapshot',
557 activity: 'chat',
558 firstSeenAt: NOW,
559 lastSeenAt: NOW,
560 },
561 ])
562 const ledger: Ledger = new Map([
563 ['scoped-status/maker/model:variant', new Set(['maxOutput'])],
564 ])
565 await recordCompleteness(db, NOW, ledger)
566 const result = (await getServiceStatus(db, NOW)).providers.find(
567 (entry) => entry.id === 'scoped-status',
568 )?.completeness
569 expect(result?.filled).toBe(0)
570 expect(result?.needed).toBe(15)
571 const served = await listModelsCatalog(db, {
572 provider: 'scoped-status',
573 pricing: true,
574 })
575 const report = buildReport(served.models as Array<ModelRow>, ledger)
576 .providers[0]!
577 expect(report.modelSilent).toEqual({ maxOutput: 1 })
578 expect(result?.score).toBe(report.score)
579})