Skip to content
Source
Sign inSign up
TanStack/schemas
Open History to see the last commit

export.ts

230 lines7.6 KB

1/**
2 * Catalog export (issue #343): the whole catalog in one response, for the
3 * public git mirror. Fields that change on every poll are left out and
4 * object keys are sorted, so a mirror commit holds only real changes.
5 *
6 * The full catalog is tens of MB; built as one string, plus its encoded
7 * copy, it risks the 128 MB isolate limit. The body is streamed one
8 * provider at a time; a failure part way errors the stream, so the client
9 * gets no valid JSON and commits nothing.
10 */
11import { and, desc, eq, isNull } from 'drizzle-orm'
12​
13import type { Db } from '#/db/index.ts'
14import {
15 changes,
16 endpoints,
17 providerModelNamespaces,
18 providers,
19 schemaVersions,
20} from '#/db/schema.ts'
21import { listModelsCatalog } from '#/server/catalog.ts'
22import type { ApiModel } from '#/server/catalog.ts'
23import { errorMessage } from '#/server/errors.ts'
24import { publicEndpointId, schemaProvenance } from '#/server/schemas-api.ts'
25import type { SchemaProvenance } from '#/server/schemas-api.ts'
26​
27/** Stored JSON text, written to the body as it is. */
28class RawJson {
29 constructor(readonly text: string) {}
31​
32/**
33 * JSON with every object's keys sorted. A schema is the exception: it is
34 * written as its stored text. That text is fixed for a content hash (the
35 * hash is over sorted keys), so its key order is the one first stored.
36 */
37export function stableJson(value: unknown): string {
38 if (value instanceof RawJson) return value.text
39 if (Array.isArray(value)) {
40 return `[${value.map((item) => stableJson(item ?? null)).join(',')}]`
41 }
42 if (value !== null && typeof value === 'object') {
43 const record = value as Record<string, unknown>
44 const members = Object.keys(record)
45 .sort()
46 .filter((key) => record[key] !== undefined)
47 .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
48 return `{${members.join(',')}}`
49 }
50 const text = JSON.stringify(value) as string | undefined
51 if (text === undefined) throw new Error(`export: ${typeof value} is not JSON`)
52 return text
54​
55/** `lastSeenAt` moves on every poll; `_links` are API navigation. */
56function exportModel({ lastSeenAt: _lastSeenAt, _links, ...model }: ApiModel) {
57 return model
59​
60interface SchemaVersionEntry {
61 contentHash: string
62 specRevision: string | null
63 provenance: SchemaProvenance
64 schema: RawJson
66​
67interface SchemaEntry {
68 provider: string
69 endpointId: string
70 activity: string
71 method: string
72 path: string
73 description: string | null
74 input: SchemaVersionEntry | null
75 output: SchemaVersionEntry | null
76 /** SSE event schema when `output` is the JSON success body. */
77 stream?: SchemaVersionEntry
79​
80/** Every endpoint of one provider with its current input and output. */
81async function exportProviderSchemas(db: Db, providerId: string) {
82 const rows = await db
83 .select({ endpoint: endpoints, version: schemaVersions })
84 .from(endpoints)
85 .leftJoin(
86 schemaVersions,
87 and(
88 eq(schemaVersions.endpointId, endpoints.id),
89 isNull(schemaVersions.supersededAt),
90 ),
91 )
92 .where(eq(endpoints.providerId, providerId))
93 .orderBy(endpoints.id, schemaVersions.kind, schemaVersions.id)
94​
95 const byEndpoint = new Map<string, SchemaEntry>()
96 for (const { endpoint, version } of rows) {
97 let entry = byEndpoint.get(endpoint.id)
98 if (!entry) {
99 entry = {
100 provider: providerId,
101 endpointId: publicEndpointId(endpoint.id, providerId),
102 activity: endpoint.activity,
103 method: endpoint.method,
104 path: endpoint.path,
105 description: endpoint.description,
106 input: null,
107 output: null,
108 }
109 byEndpoint.set(endpoint.id, entry)
110 }
111 if (version) {
112 // An interrupted sync can leave two current rows; picking one would
113 // make the mirror flip between them, so the export refuses.
114 if (version.kind === 'stream' ? entry.stream : entry[version.kind]) {
115 throw new Error(
116 `${endpoint.id} has more than one current ${version.kind} version`,
117 )
118 }
119 // Written raw, so a broken text would break the body with no clue.
120 if (!/^\s*\{[\s\S]*\}\s*$/.test(version.schema)) {
121 throw new Error(`schema version ${version.id} is not a JSON object`)
122 }
123 const recorded = {
124 contentHash: version.contentHash,
125 specRevision: version.specRevision,
126 provenance: schemaProvenance(version),
127 schema: new RawJson(version.schema),
128 }
129 if (version.kind === 'stream') entry.stream = recorded
130 else entry[version.kind] = recorded
131 }
132 }
133 return [...byEndpoint.values()]
135​
136/** Each provider's rows as JSON, comma-joined across providers. */
137async function* perProvider(
138 section: string,
139 providerIds: Array<string>,
140 load: (providerId: string) => Promise<Array<unknown>>,
141): AsyncGenerator<string> {
142 let separator = ''
143 for (const providerId of providerIds) {
144 const rows = await load(providerId).catch((error: unknown) => {
145 throw new Error(`${section} of ${providerId}: ${errorMessage(error)}`, {
146 cause: error,
147 })
148 })
149 if (rows.length === 0) continue
150 yield separator + rows.map(stableJson).join(',')
151 separator = ','
152 }
154​
155/**
156 * The body in order: `latestChange`, then `models`, `providers`, and
157 * `schemas`. `providers` is ordered by id; `models` and `schemas` by
158 * provider id, then their own id.
159 */
160async function* exportChunks(db: Db): AsyncGenerator<string> {
161 // Read first: every change it covers is already in the rows below. The
162 // export is not one snapshot, so those rows may also hold later changes.
163 const [latestChange] = await db
164 .select({ id: changes.id, createdAt: changes.createdAt })
165 .from(changes)
166 .orderBy(desc(changes.createdAt), desc(changes.id))
167 .limit(1)
168 const providerRows = await db.select().from(providers).orderBy(providers.id)
169 const namespaceRows = await db
170 .select()
171 .from(providerModelNamespaces)
172 .orderBy(providerModelNamespaces.namespace)
173​
174 const providerIds = providerRows.map((provider) => provider.id)
175​
176 yield `{"latestChange":${stableJson(latestChange ?? null)},"models":[`
177 yield* perProvider('models', providerIds, async (provider) => {
178 const catalog = await listModelsCatalog(db, {
179 provider,
180 includeDeprecated: true,
181 pricing: true,
182 })
183 return catalog.models.map(exportModel)
184 })
185​
186 const exportedProviders = providerRows.map((provider) => ({
187 id: provider.id,
188 displayName: provider.displayName,
189 specSourceUrl: provider.specSourceUrl,
190 modelsEndpoint: provider.modelsEndpoint,
191 authEnvVar: provider.authEnvVar,
192 status: provider.status,
193 modelNamespaces: namespaceRows
194 .filter((row) => row.providerId === provider.id)
195 .map((row) => row.namespace),
196 }))
197 yield `],"providers":${stableJson(exportedProviders)},"schemas":[`
198​
199 yield* perProvider('schemas', providerIds, (provider) =>
200 exportProviderSchemas(db, provider),
201 )
202 yield ']}'
204​
205/** GET /v1/admin/export — the response body. */
206export function exportCatalogStream(db: Db): ReadableStream<Uint8Array> {
207 const encoder = new TextEncoder()
208 const chunks = exportChunks(db)
209 return new ReadableStream({
210 async pull(controller) {
211 try {
212 const next = await chunks.next()
213 if (next.done) controller.close()
214 else controller.enqueue(encoder.encode(next.value))
215 } catch (error) {
216 console.error(
217 JSON.stringify({
218 job: 'catalog-export',
219 error: errorMessage(error),
220 stack: error instanceof Error ? error.stack : undefined,
221 }),
222 )
223 controller.error(error)
224 }
225 },
226 async cancel() {
227 await chunks.return(undefined)
228 },
229 })