Skip to content
Source
Sign inSign up
TanStack/schemas
Open History to see the last commit

status.ts

383 lines12.3 KB

1import { and, count, eq, isNull, sql } from 'drizzle-orm'
2​
3import type { Db } from '#/db/index.ts'
4import {
5 cacheMeta,
6 endpoints,
7 models,
8 providers,
9 schemaVersions,
10} from '#/db/schema.ts'
11import { FACT_KEYS, buildReport, scoredFacts } from '#/lib/completeness.ts'
12import type { FactKey, Ledger, ModelRow } from '#/lib/completeness.ts'
13import { errorMessage } from '#/server/errors.ts'
14import { readIngestRecords } from '#/server/ingest/docs-failing.ts'
15import type {
16 DocsFailing,
17 PriceClearsRefused,
18 UpdateStorm,
19} from '#/server/ingest/docs-failing.ts'
20import { providerRegistry } from '#/server/providers/index.ts'
21import { servePricing } from '#/server/rate-card.ts'
22import { resolveSchemaEndpointId } from '#/server/schema-binding.ts'
23import { sourceSilentLedger } from '#/server/source-silent.ts'
24​
25/**
26 * The gap-report score (`#/lib/completeness.ts`) over the provider's live
27 * chat models: `filled / needed`, the same number `bun run gap:report`
28 * prints.
29 */
30export interface Completeness {
31 /**
32 * 0 to 1. Null when the provider has no live chat models and so nothing
33 * to score (the CLI prints 0 there so its loop keeps going), and when
34 * there is no current score: none computed yet, or the provider's chat
35 * models changed since the last poll scored them.
36 */
37 score: number | null
38 /** Live chat models scored. */
39 chat: number
40 /** Facts carried, summed over those models; `silent` facts left out. */
41 filled: number
42 /** Facts that apply, summed over those models; `silent` facts left out. */
43 needed: number
44 /** Facts the provider does not publish (`docs/source-silent`). */
45 silent: Array<FactKey>
47​
48export interface ProviderStatus {
49 id: string
50 displayName: string
51 /** `pending` = registered in code but not yet synced into the DB. */
52 status: 'active' | 'degraded' | 'disabled' | 'pending'
53 lastPolledAt: number | null
54 lastSyncedAt: number | null
55 /**
56 * Present while the provider's docs pages fail to load: polls go on and
57 * rows keep their stored docs-derived facts. Does not change `status`.
58 */
59 docsFailing?: DocsFailing
60 /**
61 * Present while polls refuse a mass price clear: the stored prices stay.
62 * Does not change `status`.
63 */
64 priceClearsRefused?: PriceClearsRefused
65 /**
66 * Present while the latest poll wrote `model.updated` for more than half
67 * of the live rows, and at least 20. The updates are written; this only
68 * marks a catalog-wide rewrite. Does not change `status`.
69 */
70 updateStorm?: UpdateStorm
71 completeness: Completeness
72 /** Model tallies count live rows: what `GET /v1/models` lists by default. */
73 counts: {
74 models: number
75 /** Models with a stored rate card. */
76 priced: number
77 /** Models with stored reasoning metadata. */
78 reasoning: number
79 /** Models whose activity is `chat`. */
80 chat: number
81 /** Models no longer listed upstream; in none of the tallies above. */
82 deprecated: number
83 endpoints: number
84 schemas: number
85 }
87​
88export interface ServiceStatus {
89 service: 'modelschemas'
90 time: number
91 /**
92 * Epoch seconds the `completeness` scores were computed (the last models
93 * poll), or null when there is no record and every score is null.
94 */
95 completenessComputedAt: number | null
96 providers: Array<ProviderStatus>
98​
99/** A tally of live rows, optionally only those matching `where`. */
100const live = (where = sql`1`) =>
101 sql<number>`count(case when ${models.deprecatedAt} is null and ${where} then 1 end)`
102​
103/**
104 * The `cache_meta` row holding every provider's score: `fetched_at` is when
105 * it was computed, `last_error` the JSON. Scoring reads every live chat
106 * row's JSON facts (about 2 MB on the full catalog), so the 15-minute poll
107 * computes it once (`recordCompleteness`) and requests only read the row.
108 * Bump the version when the scorer or the `Completeness` shape changes: the
109 * old row then reads as no record, and scores are null until the next poll.
110 */
111const COMPLETENESS_KEY = 'status:completeness:v1'
112​
113/**
114 * Completeness of every provider with live chat rows. One query, only the
115 * columns the score reads, each row shaped as `GET /v1/models?pricing=1`
116 * serves it (`toApiModel`) so this and the CLI score the same thing.
117 */
118async function scoreCompleteness(
119 db: Db,
120 ledger: Ledger,
121): Promise<Record<string, Completeness>> {
122 const rows = await db
123 .select({
124 providerId: models.providerId,
125 rawId: models.rawId,
126 activity: models.activity,
127 contextWindow: models.contextWindow,
128 maxOutput: models.maxOutput,
129 modalities: models.modalities,
130 pricing: models.pricing,
131 capabilities: models.capabilities,
132 reasoning: models.reasoning,
133 requestMap: models.requestMap,
134 schemaEndpointId: models.schemaEndpointId,
135 })
136 .from(models)
137 .where(and(eq(models.activity, 'chat'), isNull(models.deprecatedAt)))
138 const report = buildReport(
139 rows.map(
140 (row): ModelRow => ({
141 provider: row.providerId,
142 rawId: row.rawId,
143 activity: row.activity,
144 contextWindow: row.contextWindow,
145 maxOutput: row.maxOutput,
146 // JSON columns: the scorer checks what it reads of each.
147 modalities: row.modalities as ModelRow['modalities'],
148 pricing: servePricing(row.pricing, 'full') as ModelRow['pricing'],
149 capabilities: row.capabilities,
150 reasoning: row.reasoning as ModelRow['reasoning'],
151 requestMap: row.requestMap,
152 schemaEndpointId: resolveSchemaEndpointId(row),
153 }),
154 ),
155 ledger,
156 )
157 return Object.fromEntries(
158 report.providers.map((provider) => {
159 const { have, need } = scoredFacts(provider)
160 return [
161 provider.provider,
162 {
163 score: provider.score,
164 chat: provider.chat,
165 filled: have,
166 needed: need,
167 silent: provider.silent,
168 },
169 ]
170 }),
171 )
173​
174const logCompleteness = (error: unknown) => {
175 console.error(
176 JSON.stringify({ job: 'completeness', error: errorMessage(error) }),
177 )
179​
180/** A stored entry is served only if it is wholly a `Completeness`. */
181function isCompleteness(value: unknown): value is Completeness {
182 if (typeof value !== 'object' || value === null) return false
183 const entry = value as Partial<Record<keyof Completeness, unknown>>
184 return (
185 (entry.score === null || typeof entry.score === 'number') &&
186 typeof entry.chat === 'number' &&
187 typeof entry.filled === 'number' &&
188 typeof entry.needed === 'number' &&
189 Array.isArray(entry.silent) &&
190 entry.silent.every((fact) => typeof fact === 'string')
191 )
193​
194/**
195 * Score every provider and store the result, after the models poll. Never
196 * throws: a failure is logged and the previous record stays. Returns
197 * whether a new record was written.
198 */
199export async function recordCompleteness(
200 db: Db,
201 now = Math.floor(Date.now() / 1000),
202 ledger: Ledger = sourceSilentLedger,
203): Promise<boolean> {
204 try {
205 const lastError = JSON.stringify(await scoreCompleteness(db, ledger))
206 await db
207 .insert(cacheMeta)
208 .values({
209 key: COMPLETENESS_KEY,
210 fetchedAt: now,
211 staleTime: 0,
212 lastError,
213 })
214 .onConflictDoUpdate({
215 target: cacheMeta.key,
216 set: { fetchedAt: now, lastError },
217 })
218 return true
219 } catch (error) {
220 logCompleteness(error)
221 return false
222 }
224​
225/**
226 * The stored scores. A missing, unreadable or misshapen record is no
227 * record: every score is then null and the request goes on.
228 */
229async function readCompleteness(db: Db): Promise<{
230 computedAt: number | null
231 scores: Partial<Record<string, Completeness>>
232}> {
233 try {
234 const row = await db.query.cacheMeta.findFirst({
235 where: eq(cacheMeta.key, COMPLETENESS_KEY),
236 })
237 if (row?.lastError) {
238 const stored: unknown = JSON.parse(row.lastError)
239 if (typeof stored === 'object' && stored !== null) {
240 const entries = Object.entries(stored)
241 const valid = entries.filter(([, entry]) => isCompleteness(entry))
242 // Entries and none of them usable is no record either.
243 if (
244 !Array.isArray(stored) &&
245 (valid.length > 0 || entries.length === 0)
246 ) {
247 return {
248 computedAt: row.fetchedAt,
249 scores: Object.fromEntries(valid),
250 }
251 }
252 }
253 }
254 } catch (error) {
255 logCompleteness(error)
256 }
257 return { computedAt: null, scores: {} }
259​
260/**
261 * Public per-provider sync status + row counts (GET /v1/status).
262 *
263 * The provider REGISTRY (code) is the source of truth for which providers
264 * exist; the DB carries runtime state. Registered providers whose first
265 * sync hasn't landed yet still appear — as `pending` with zero counts — so
266 * the listing is always the full roster, for humans and agents alike.
267 */
268export async function getServiceStatus(
269 db: Db,
270 now = Math.floor(Date.now() / 1000),
271): Promise<ServiceStatus> {
272 const [
273 providerRows,
274 records,
275 { computedAt, scores },
276 modelCounts,
277 endpointCounts,
278 schemaCounts,
279 ] = await Promise.all([
280 db.select().from(providers),
281 // One statement for every provider's docs-failing, refused-clears, and update-storm row.
282 readIngestRecords(db),
283 readCompleteness(db),
284 // One pass yields every model tally. Deprecated rows are counted apart:
285 // the catalog hides them, and a provider's count must match its list.
286 db
287 .select({
288 providerId: models.providerId,
289 models: live(),
290 priced: live(sql`${models.pricing} is not null`),
291 reasoning: live(sql`${models.reasoning} is not null`),
292 chat: live(sql`${models.activity} = 'chat'`),
293 deprecated: count(models.deprecatedAt),
294 })
295 .from(models)
296 .groupBy(models.providerId),
297 db
298 .select({ providerId: endpoints.providerId, n: count() })
299 .from(endpoints)
300 .groupBy(endpoints.providerId),
301 // Current (non-superseded) schema versions per provider.
302 db
303 .select({ providerId: endpoints.providerId, n: count() })
304 .from(schemaVersions)
305 .innerJoin(endpoints, eq(schemaVersions.endpointId, endpoints.id))
306 .where(isNull(schemaVersions.supersededAt))
307 .groupBy(endpoints.providerId),
308 ])
309​
310 const toMap = (rows: Array<{ providerId: string; n: number }>) =>
311 new Map(rows.map((r) => [r.providerId, r.n]))
312 const modelsBy = new Map(modelCounts.map((r) => [r.providerId, r]))
313 const endpointsBy = toMap(endpointCounts)
314 const schemasBy = toMap(schemaCounts)
315 // The record is as old as the last poll. It is served only for a provider
316 // whose live chat count still equals the count it scored, so a score never
317 // sits over a different row set; that also makes no chat rows a null, not
318 // a zero.
319 const completeness = (providerId: string): Completeness => {
320 const scored = scores[providerId]
321 const chat = modelsBy.get(providerId)?.chat ?? 0
322 if (chat > 0 && scored?.chat === chat) return scored
323 return {
324 score: null,
325 chat: 0,
326 filled: 0,
327 needed: 0,
328 silent: FACT_KEYS.filter((key) =>
329 sourceSilentLedger.get(providerId)?.has(key),
330 ),
331 }
332 }
333​
334 const byId = new Map<string, ProviderStatus>()
335 for (const p of providerRows) {
336 byId.set(p.id, {
337 id: p.id,
338 displayName: p.displayName,
339 status: p.status,
340 lastPolledAt: p.lastPolledAt,
341 lastSyncedAt: p.lastSyncedAt,
342 ...records.get(p.id),
343 completeness: completeness(p.id),
344 counts: {
345 models: modelsBy.get(p.id)?.models ?? 0,
346 priced: modelsBy.get(p.id)?.priced ?? 0,
347 reasoning: modelsBy.get(p.id)?.reasoning ?? 0,
348 chat: modelsBy.get(p.id)?.chat ?? 0,
349 deprecated: modelsBy.get(p.id)?.deprecated ?? 0,
350 endpoints: endpointsBy.get(p.id) ?? 0,
351 schemas: schemasBy.get(p.id) ?? 0,
352 },
353 })
354 }
355 // Registered but not yet in the DB (fresh adapter, seed not run).
356 for (const config of providerRegistry) {
357 if (byId.has(config.id)) continue
358 byId.set(config.id, {
359 id: config.id,
360 displayName: config.displayName,
361 status: 'pending',
362 lastPolledAt: null,
363 lastSyncedAt: null,
364 completeness: completeness(config.id),
365 counts: {
366 models: 0,
367 priced: 0,
368 reasoning: 0,
369 chat: 0,
370 deprecated: 0,
371 endpoints: 0,
372 schemas: 0,
373 },
374 })
375 }
376​
377 return {
378 service: 'modelschemas',
379 time: now,
380 completenessComputedAt: computedAt,
381 providers: [...byId.values()].sort((a, b) => a.id.localeCompare(b.id)),
382 }