Open History to see the last commit
1/**
2 * Sync engine (PLAN.md task 2.3): per provider — fetch spec → classify
3 * endpoints → bundle schemas → diff content hashes against
4 * `schema_versions` → insert new versions (reviving superseded rows when
5 * upstream reverts to previously seen content), mark superseded, upsert
6 * `endpoints`, write `changes` rows, warm KV with new blobs. Idempotent.
7 */
8import { and, eq, inArray, isNull, notInArray } from 'drizzle-orm'
9
10import type { Db } from '#/db/index.ts'
11import {
12 changes,
13 endpoints,
14 models,
15 providers,
16 providerModelNamespaces,
17 schemaVersions,
18} from '#/db/schema.ts'
19import type { Activity } from '#/db/schema.ts'
20import { seedForProvider } from '#/db/seed-providers.ts'
21import { errorMessage } from '#/server/errors.ts'
22import { contentHash, putJson, schemaKey } from '#/server/kv.ts'
23import { PROVENANCE_MARKER } from '#/server/providers/types.ts'
24import type {
25 BundledEndpoint,
26 Derivation,
27 EndpointProvenance,
28 OpenApiOperation,
29 ProviderConfig,
30 ProviderEnvironment,
31 SpecFetchResult,
32 SpecSource,
33} from '#/server/providers/types.ts'
34import { hasAsyncApiFlag, withAsyncApiFlag } from './asyncapi.ts'
35import {
36 captureIngestEvents,
37 ingestFailedEvent,
38 noteIngest,
39 runIngestScope,
40} from './ingest-signals.ts'
41import { providerRegistry } from '#/server/providers/index.ts'
42import {
43 EXTRACTOR_VERSION,
44 endpointIdFromPath,
45 extractEndpointSchemas,
46 findDanglingRefs,
47} from './bundle.ts'
48import type { JsonValue } from './bundle.ts'
49import { dropModelsDevSchemaVersions } from './retire-models-dev.ts'
50
51export interface SyncDeps {
52 db: Db
53 kv: KVNamespace
54 secrets: ProviderEnvironment
55 /** Injectable clock (unix epoch seconds) for deterministic tests. */
56 now?: () => number
57}
58
59export interface SyncOutcome {
60 providerId: string
61 endpointsSeen: number
62 versionsAdded: number
63 changesWritten: number
64 skipped?: string
65 error?: string
66 warnings: Array<string>
67}
68
69export interface ClassifiedEndpoint {
70 /** Globally unique db id: `${providerId}/${pathId}`. */
71 dbId: string
72 path: string
73 /** HTTP method. GET is the BytePlus video-task retrieve call. */
74 method: 'POST' | 'GET'
75 activity: Activity
76 description: string | null
77 /** Provenance of the spec document this endpoint came from. */
78 source: SpecSource | null
79 /** How this endpoint's schemas were arrived at (the trust ladder). */
80 derivation: Derivation
81 /** `YYYY-MM-DD` a `probe-verified` claim was last confirmed. */
82 verifiedAt: string | null
83 input?: Record<string, JsonValue>
84 output?: Record<string, JsonValue>
85 /** WMA AsyncAPI — catalog flag; skip HTTP OpenAPI assembly. */
86 asyncapi?: boolean
87}
88
89function newChangeId(): string {
90 return crypto.randomUUID()
91}
92
93const DERIVATIONS: ReadonlyArray<Derivation> = [
94 'upstream-spec',
95 'generated',
96 'probe-verified',
97 'docs-derived',
98]
99
100/**
101 * Per-operation provenance annotation, falling back to the provider's
102 * default. A malformed marker falls back too rather than throwing — a bad
103 * annotation should not sink a sync.
104 */
105function readProvenance(
106 op: OpenApiOperation,
107 provider: ProviderConfig,
108): { derivation: Derivation; verifiedAt: string | null } {
109 const marker = op[PROVENANCE_MARKER]
110 if (marker !== null && typeof marker === 'object') {
111 const { derivation, verifiedAt } = marker as Partial<EndpointProvenance>
112 if (typeof derivation === 'string' && DERIVATIONS.includes(derivation)) {
113 return {
114 derivation: derivation,
115 verifiedAt: typeof verifiedAt === 'string' ? verifiedAt : null,
116 }
117 }
118 }
119 return { derivation: provider.defaultDerivation, verifiedAt: null }
120}
121
122/**
123 * Classify + bundle every generation endpoint across a provider's fetched
124 * specs. Pure (no storage) — shared by the sync engine and the offline
125 * re-derivation script (`scripts/rederive.ts`), so both always derive
126 * identical content hashes from the same upstream documents.
127 */
128export function classifyAndBundle(
129 provider: ProviderConfig,
130 fetched: SpecFetchResult,
131): { endpoints: Array<ClassifiedEndpoint>; warnings: Array<string> } {
132 const classified: Array<ClassifiedEndpoint> = []
133 const warnings: Array<string> = []
134 for (const [specIndex, spec] of fetched.specs.entries()) {
135 const source = fetched.sources[specIndex] ?? null
136 for (const [pathKey, operations] of Object.entries(spec.paths ?? {})) {
137 const methods = ['post', 'get'] as const
138 for (const method of methods) {
139 const operation = operations[method]
140 if (!operation) continue
141 if (method === 'get' && provider.classifyGets !== true) continue
142 const activity = provider.classify(pathKey, operation)
143 if (activity === null) continue
144
145 const extracted = extractEndpointSchemas(
146 spec,
147 pathKey,
148 fetched.outputStrategy,
149 method,
150 )
151 warnings.push(...extracted.warnings)
152 for (const [kind, schema] of [
153 ['input', extracted.input],
154 ['output', extracted.output],
155 ] as const) {
156 if (schema && findDanglingRefs(schema).length > 0) {
157 warnings.push(
158 `${provider.id}${pathKey}: bundled ${kind} schema has dangling refs`,
159 )
160 }
161 }
162 if (
163 !extracted.input &&
164 !extracted.output &&
165 operation['x-modelschemas-route-only'] !== true
166 )
167 continue
168
169 const description =
170 typeof operation.summary === 'string'
171 ? operation.summary
172 : typeof operation.description === 'string'
173 ? operation.description
174 : null
175 const { derivation, verifiedAt } = readProvenance(operation, provider)
176 classified.push({
177 dbId: `${provider.id}/${endpointIdFromPath(pathKey)}`,
178 path: pathKey,
179 method: method === 'get' ? 'GET' : 'POST',
180 activity,
181 description,
182 source,
183 derivation,
184 verifiedAt,
185 input: extracted.input,
186 output: extracted.output,
187 })
188 }
189 }
190 }
191 for (const extra of fetched.bundledEndpoints ?? []) {
192 if (!extra.input && !extra.output) continue
193 classified.push(classifiedFromBundled(provider.id, extra, warnings))
194 }
195 return { endpoints: classified, warnings }
196}
197
198function asBundledSchema(
199 schema: Record<string, unknown> | undefined,
200): Record<string, JsonValue> | undefined {
201 return schema as Record<string, JsonValue> | undefined
202}
203
204function classifiedFromBundled(
205 providerId: string,
206 extra: BundledEndpoint,
207 warnings: Array<string>,
208): ClassifiedEndpoint {
209 const input = asBundledSchema(extra.input)
210 const output = asBundledSchema(extra.output)
211 const pathKey = extra.path
212 for (const [kind, schema] of [
213 ['input', input],
214 ['output', output],
215 ] as const) {
216 if (schema && findDanglingRefs(schema).length > 0) {
217 warnings.push(
218 `${providerId}${pathKey}: bundled ${kind} schema has dangling refs`,
219 )
220 }
221 }
222 return {
223 dbId: `${providerId}/${extra.publicId ?? endpointIdFromPath(pathKey)}`,
224 path: pathKey,
225 method: 'POST',
226 activity: extra.activity,
227 description: extra.description,
228 source: extra.source,
229 derivation: extra.derivation,
230 verifiedAt: extra.verifiedAt ?? null,
231 input,
232 output,
233 asyncapi: extra.asyncapi,
234 }
235}
236
237async function applyAsyncApiCatalogFlags(
238 db: Db,
239 providerId: string,
240 asyncApiRawIds: Array<string>,
241): Promise<void> {
242 const flagged = new Set(asyncApiRawIds)
243 const rows = await db
244 .select()
245 .from(models)
246 .where(eq(models.providerId, providerId))
247 for (const row of rows) {
248 const want = flagged.has(row.rawId)
249 const has = hasAsyncApiFlag(row.capabilities)
250 if (want === has) continue
251 await db
252 .update(models)
253 .set({ capabilities: withAsyncApiFlag(row.capabilities, want) })
254 .where(eq(models.id, row.id))
255 }
256}
257
258/** Sync one provider. Throws only on programmer error — upstream/storage
259 * failures surface in the outcome's `error` via syncAllProviders. */
260/**
261 * Upsert the provider's row from its registry config: refreshes the seeded
262 * config columns, preserves runtime state (lastPolledAt/lastSyncedAt/status).
263 * Same provider row as `scripts/seed.ts`, plus the provider's namespaces
264 * (which only this writes), run automatically so a newly
265 * registered provider self-heals into the DB on its first sync/poll instead
266 * of waiting on a manual seed against prod.
267 */
268export async function ensureProviderRow(
269 db: Db,
270 provider: ProviderConfig,
271): Promise<void> {
272 const seed = seedForProvider(provider)
273 if (seed === null) return
274 const namespaces = provider.modelNamespaces ?? []
275 // One batch: the namespace table mirrors config (a dropped name stops
276 // resolving) without a second round trip on every poll.
277 await db.batch([
278 db
279 .insert(providers)
280 .values(seed)
281 .onConflictDoUpdate({
282 target: providers.id,
283 set: {
284 displayName: seed.displayName,
285 specSourceUrl: seed.specSourceUrl,
286 modelsEndpoint: seed.modelsEndpoint ?? null,
287 authEnvVar: seed.authEnvVar ?? null,
288 },
289 }),
290 db
291 .delete(providerModelNamespaces)
292 .where(
293 and(
294 eq(providerModelNamespaces.providerId, seed.id),
295 notInArray(providerModelNamespaces.namespace, namespaces),
296 ),
297 ),
298 ...(namespaces.length
299 ? [
300 db
301 .insert(providerModelNamespaces)
302 .values(
303 namespaces.map((namespace) => ({
304 namespace,
305 providerId: seed.id,
306 })),
307 )
308 .onConflictDoNothing(),
309 ]
310 : []),
311 ])
312}
313
314/** Composite map key for an endpoint's current version of one kind. */
315function versionKey(endpointId: string, kind: 'input' | 'output'): string {
316 return `${endpointId}\u0000${kind}`
317}
318
319export async function syncProvider(
320 deps: SyncDeps,
321 provider: ProviderConfig,
322): Promise<SyncOutcome> {
323 const { db, kv, secrets } = deps
324 const now = deps.now?.() ?? Math.floor(Date.now() / 1000)
325 const outcome: SyncOutcome = {
326 providerId: provider.id,
327 endpointsSeen: 0,
328 versionsAdded: 0,
329 changesWritten: 0,
330 warnings: [],
331 }
332 await ensureProviderRow(db, provider)
333 await dropModelsDevSchemaVersions(db, provider.id)
334
335 const fetched = await provider.fetchSpec(secrets)
336 if (fetched.withdrawnSchemaSources?.length) {
337 const withdrawn = await db
338 .select({ id: schemaVersions.id })
339 .from(schemaVersions)
340 .innerJoin(endpoints, eq(schemaVersions.endpointId, endpoints.id))
341 .where(
342 and(
343 eq(endpoints.providerId, provider.id),
344 isNull(schemaVersions.supersededAt),
345 inArray(schemaVersions.sourceUrl, fetched.withdrawnSchemaSources),
346 ),
347 )
348 for (let i = 0; i < withdrawn.length; i += 90) {
349 await db
350 .update(schemaVersions)
351 .set({ supersededAt: now })
352 .where(
353 inArray(
354 schemaVersions.id,
355 withdrawn.slice(i, i + 90).map((row) => row.id),
356 ),
357 )
358 }
359 if (withdrawn.length)
360 outcome.warnings.push(
361 `Withdrew ${withdrawn.length} schema versions from rejected former sources`,
362 )
363 }
364 if (fetched.warnings) outcome.warnings.push(...fetched.warnings)
365 if (fetched.skipped) {
366 outcome.skipped = fetched.skipped
367 outcome.warnings.push(fetched.skipped)
368 return outcome
369 }
370
371 const { endpoints: classified, warnings } = classifyAndBundle(
372 provider,
373 fetched,
374 )
375 outcome.warnings.push(...warnings)
376 outcome.endpointsSeen = classified.length
377
378 const existingEndpoints = await db
379 .select()
380 .from(endpoints)
381 .where(eq(endpoints.providerId, provider.id))
382 const existingById = new Map(existingEndpoints.map((e) => [e.id, e]))
383
384 // Current (non-superseded) versions for the whole provider in one query.
385 // Per-endpoint lookups here are 2 D1 round trips per endpoint — FAL alone
386 // has ~1,400 endpoints, which blows the invocation's 1,000-subrequest
387 // budget (and its wall clock) before the sync can complete, leaving the
388 // provider stuck degraded. Same batching treatment as poll-models.
389 const currentVersionRows = await db
390 .select({
391 id: schemaVersions.id,
392 endpointId: schemaVersions.endpointId,
393 kind: schemaVersions.kind,
394 contentHash: schemaVersions.contentHash,
395 })
396 .from(schemaVersions)
397 .innerJoin(endpoints, eq(schemaVersions.endpointId, endpoints.id))
398 .where(
399 and(
400 eq(endpoints.providerId, provider.id),
401 isNull(schemaVersions.supersededAt),
402 ),
403 )
404 const currentByEndpointKind = new Map(
405 currentVersionRows.map((v) => [versionKey(v.endpointId, v.kind), v]),
406 )
407
408 for (const endpoint of classified) {
409 const existing = existingById.get(endpoint.dbId)
410 if (!existing) {
411 await db.insert(endpoints).values({
412 id: endpoint.dbId,
413 providerId: provider.id,
414 activity: endpoint.activity,
415 method: endpoint.method,
416 path: endpoint.path,
417 description: endpoint.description,
418 })
419 await db.insert(changes).values({
420 id: newChangeId(),
421 type: 'endpoint.added',
422 providerId: provider.id,
423 subjectId: endpoint.dbId,
424 summary: `Endpoint ${endpoint.path} added (${endpoint.activity})`,
425 createdAt: now,
426 })
427 outcome.changesWritten++
428 } else if (
429 existing.activity !== endpoint.activity ||
430 existing.description !== endpoint.description
431 ) {
432 await db
433 .update(endpoints)
434 .set({
435 activity: endpoint.activity,
436 description: endpoint.description,
437 })
438 .where(eq(endpoints.id, endpoint.dbId))
439 }
440
441 for (const [kind, schema] of [
442 ['input', endpoint.input],
443 ['output', endpoint.output],
444 ] as const) {
445 if (!schema) continue
446 const hash = await contentHash(schema)
447 const current = currentByEndpointKind.get(versionKey(endpoint.dbId, kind))
448 if (current?.contentHash === hash) continue
449
450 // The id is deterministic on content, so when upstream reverts to a
451 // previously served schema (A → B → A) the row already exists as a
452 // superseded version — revive it instead of colliding on the insert.
453 await db
454 .insert(schemaVersions)
455 .values({
456 id: `${endpoint.dbId}:${kind}:${hash.slice(0, 16)}`,
457 endpointId: endpoint.dbId,
458 kind,
459 contentHash: hash,
460 schema: JSON.stringify(schema),
461 specRevision: fetched.specRevision ?? null,
462 sourceUrl: endpoint.source?.url ?? null,
463 sourceHash: endpoint.source?.hash ?? null,
464 extractorVersion: EXTRACTOR_VERSION,
465 derivation: endpoint.derivation,
466 verifiedAt: endpoint.verifiedAt,
467 createdAt: now,
468 })
469 .onConflictDoUpdate({
470 target: schemaVersions.id,
471 set: {
472 supersededAt: null,
473 specRevision: fetched.specRevision ?? null,
474 sourceUrl: endpoint.source?.url ?? null,
475 sourceHash: endpoint.source?.hash ?? null,
476 extractorVersion: EXTRACTOR_VERSION,
477 derivation: endpoint.derivation,
478 verifiedAt: endpoint.verifiedAt,
479 createdAt: now,
480 },
481 })
482 outcome.versionsAdded++
483 await putJson(kv, schemaKey(hash), schema)
484
485 if (current) {
486 await db
487 .update(schemaVersions)
488 .set({ supersededAt: now })
489 .where(eq(schemaVersions.id, current.id))
490 }
491 await db.insert(changes).values({
492 id: newChangeId(),
493 type: current ? 'schema.updated' : 'schema.added',
494 providerId: provider.id,
495 subjectId: endpoint.dbId,
496 summary: `${current ? 'Updated' : 'Added'} ${kind} schema for ${endpoint.path}`,
497 payload: {
498 kind,
499 contentHash: hash,
500 previousHash: current?.contentHash ?? null,
501 },
502 createdAt: now,
503 })
504 outcome.changesWritten++
505 }
506 }
507
508 // Endpoints in D1 that vanished from the spec: record the removal but keep
509 // the rows — schema version history stays queryable. The already-recorded
510 // set loads in one query: vanished endpoints accumulate for good (rows are
511 // kept), so per-endpoint lookups would cost one subrequest each, forever.
512 const seenIds = new Set(classified.map((e) => e.dbId))
513 const removalRows = await db
514 .select({ subjectId: changes.subjectId })
515 .from(changes)
516 .where(
517 and(
518 eq(changes.type, 'endpoint.removed'),
519 eq(changes.providerId, provider.id),
520 ),
521 )
522 const alreadyRemovedIds = new Set(removalRows.map((r) => r.subjectId))
523 for (const existing of existingEndpoints) {
524 if (seenIds.has(existing.id)) continue
525 if (alreadyRemovedIds.has(existing.id)) continue
526 await db.insert(changes).values({
527 id: newChangeId(),
528 type: 'endpoint.removed',
529 providerId: provider.id,
530 subjectId: existing.id,
531 summary: `Endpoint ${existing.path} removed from the ${provider.id} spec`,
532 createdAt: now,
533 })
534 outcome.changesWritten++
535 }
536
537 if (fetched.asyncApiRawIds !== undefined) {
538 await applyAsyncApiCatalogFlags(db, provider.id, fetched.asyncApiRawIds)
539 }
540
541 await db
542 .update(providers)
543 .set({ lastSyncedAt: now, status: 'active' })
544 .where(eq(providers.id, provider.id))
545
546 return outcome
547}
548
549/**
550 * Spec-sync shard schedule: each cron fires its own Worker invocation with
551 * a fresh subrequest/CPU/memory budget, so one provider's blowup can't
552 * starve the providers behind it (production data: FAL's oversized sync
553 * killed the shared 05:00 run mid-flight daily, leaving every provider
554 * after it in the registry permanently unsynced). Shard 0 is FAL alone —
555 * by far the largest (~1,400 per-model specs); the rest of the registry
556 * round-robins across the remaining shards, which also spreads the other
557 * big-spec providers (they're adjacent in the registry). Keep this array
558 * in lockstep with `triggers.crons` in wrangler.jsonc.
559 */
560export const SPEC_SYNC_SHARD_CRONS = [
561 '0 5 * * *',
562 '10 5 * * *',
563 '20 5 * * *',
564 '30 5 * * *',
565] as const
566
567export function specSyncShard(shardIndex: number): Array<ProviderConfig> {
568 const rest = providerRegistry.filter((p) => p.id !== 'fal')
569 if (shardIndex === 0) return providerRegistry.filter((p) => p.id === 'fal')
570 const restShards = SPEC_SYNC_SHARD_CRONS.length - 1
571 return rest.filter((_, i) => i % restShards === shardIndex - 1)
572}
573
574/**
575 * Sync the given providers (default: the full registry), sequentially
576 * (subrequest limits), with per-provider isolation: one provider's outage
577 * doesn't sink the run.
578 */
579export async function syncAllProviders(
580 deps: SyncDeps,
581 providersToSync: Array<ProviderConfig> = providerRegistry,
582): Promise<Array<SyncOutcome>> {
583 const outcomes: Array<SyncOutcome> = []
584 for (const provider of providersToSync) {
585 await runIngestScope(async () => {
586 try {
587 outcomes.push(await syncProvider(deps, provider))
588 } catch (error) {
589 const message = errorMessage(error)
590 // Own log line per failure: the aggregate outcomes blob can exceed
591 // what Workers Logs stores, which silently loses these errors.
592 console.error(
593 JSON.stringify({
594 job: 'spec-sync',
595 providerId: provider.id,
596 error: message,
597 }),
598 )
599 noteIngest(ingestFailedEvent('spec-sync', provider.id, message))
600 outcomes.push({
601 providerId: provider.id,
602 endpointsSeen: 0,
603 versionsAdded: 0,
604 changesWritten: 0,
605 error: message,
606 warnings: [],
607 })
608 try {
609 await deps.db
610 .update(providers)
611 .set({ status: 'degraded' })
612 .where(eq(providers.id, provider.id))
613 } catch (statusError) {
614 // Best-effort: if the failure was resource exhaustion (subrequest
615 // budget), this write fails too — don't let it sink the run.
616 console.error(
617 JSON.stringify({
618 job: 'spec-sync',
619 providerId: provider.id,
620 error: `degraded-status write failed: ${errorMessage(statusError)}`,
621 }),
622 )
623 noteIngest(
624 ingestFailedEvent(
625 'spec-sync',
626 provider.id,
627 `degraded-status write failed: ${errorMessage(statusError)}`,
628 ),
629 )
630 }
631 } finally {
632 await captureIngestEvents(deps.secrets.POSTHOG_PROJECT_KEY)
633 }
634 })
635 }
636 return outcomes
637}