Skip to content
Source
Sign inSign up
TanStack/schemas
Open History to see the last commit

export.ts

226 lines7.3 KB

1/**
2 * Catalog export (issue #343): the whole catalog in one response, for the
3 * public git mirror. Fields that change on every poll are left out and
4 * object keys are sorted, so a mirror commit holds only real changes.
5 *
6 * The full catalog is tens of MB; built as one string, plus its encoded
7 * copy, it risks the 128 MB isolate limit. The body is streamed one
8 * provider at a time; a failure part way errors the stream, so the client
9 * gets no valid JSON and commits nothing.
10 */
11import { and, desc, eq, isNull } from 'drizzle-orm'
12​
13import type { Db } from '#/db/index.ts'
14import {
15 changes,
16 endpoints,
17 providerModelNamespaces,
18 providers,
19 schemaVersions,
20} from '#/db/schema.ts'
21import { listModelsCatalog } from '#/server/catalog.ts'
22import type { ApiModel } from '#/server/catalog.ts'
23import { errorMessage } from '#/server/errors.ts'
24import { publicEndpointId, schemaProvenance } from '#/server/schemas-api.ts'
25import type { SchemaProvenance } from '#/server/schemas-api.ts'
26​
27/** Stored JSON text, written to the body as it is. */
28class RawJson {
29 constructor(readonly text: string) {}
31​
32/**
33 * JSON with every object's keys sorted. A schema is the exception: it is
34 * written as its stored text. That text is fixed for a content hash (the
35 * hash is over sorted keys), so its key order is the one first stored.
36 */
37export function stableJson(value: unknown): string {
38 if (value instanceof RawJson) return value.text
39 if (Array.isArray(value)) {
40 return `[${value.map((item) => stableJson(item ?? null)).join(',')}]`
41 }
42 if (value !== null && typeof value === 'object') {
43 const record = value as Record<string, unknown>
44 const members = Object.keys(record)
45 .sort()
46 .filter((key) => record[key] !== undefined)
47 .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
48 return `{${members.join(',')}}`
49 }
50 const text = JSON.stringify(value) as string | undefined
51 if (text === undefined) throw new Error(`export: ${typeof value} is not JSON`)
52 return text
54​
55/** `lastSeenAt` moves on every poll; `_links` are API navigation. */
56function exportModel({ lastSeenAt, _links, ...model }: ApiModel) {
57 return model
59​
60interface SchemaVersionEntry {
61 contentHash: string
62 specRevision: string | null
63 provenance: SchemaProvenance
64 schema: RawJson
66​
67interface SchemaEntry {
68 provider: string
69 endpointId: string
70 activity: string
71 method: string
72 path: string
73 description: string | null
74 input: SchemaVersionEntry | null
75 output: SchemaVersionEntry | null
77​
78/** Every endpoint of one provider with its current input and output. */
79async function exportProviderSchemas(db: Db, providerId: string) {
80 const rows = await db
81 .select({ endpoint: endpoints, version: schemaVersions })
82 .from(endpoints)
83 .leftJoin(
84 schemaVersions,
85 and(
86 eq(schemaVersions.endpointId, endpoints.id),
87 isNull(schemaVersions.supersededAt),
88 ),
89 )
90 .where(eq(endpoints.providerId, providerId))
91 .orderBy(endpoints.id, schemaVersions.kind, schemaVersions.id)
92​
93 const byEndpoint = new Map<string, SchemaEntry>()
94 for (const { endpoint, version } of rows) {
95 let entry = byEndpoint.get(endpoint.id)
96 if (!entry) {
97 entry = {
98 provider: providerId,
99 endpointId: publicEndpointId(endpoint.id, providerId),
100 activity: endpoint.activity,
101 method: endpoint.method,
102 path: endpoint.path,
103 description: endpoint.description,
104 input: null,
105 output: null,
106 }
107 byEndpoint.set(endpoint.id, entry)
108 }
109 if (version) {
110 // An interrupted sync can leave two current rows; picking one would
111 // make the mirror flip between them, so the export refuses.
112 if (entry[version.kind]) {
113 throw new Error(
114 `${endpoint.id} has more than one current ${version.kind} version`,
115 )
116 }
117 // Written raw, so a broken text would break the body with no clue.
118 if (!/^\s*\{[\s\S]*\}\s*$/.test(version.schema)) {
119 throw new Error(`schema version ${version.id} is not a JSON object`)
120 }
121 entry[version.kind] = {
122 contentHash: version.contentHash,
123 specRevision: version.specRevision,
124 provenance: schemaProvenance(version),
125 schema: new RawJson(version.schema),
126 }
127 }
128 }
129 return [...byEndpoint.values()]
131​
132/** Each provider's rows as JSON, comma-joined across providers. */
133async function* perProvider(
134 section: string,
135 providerIds: Array<string>,
136 load: (providerId: string) => Promise<Array<unknown>>,
137): AsyncGenerator<string> {
138 let separator = ''
139 for (const providerId of providerIds) {
140 const rows = await load(providerId).catch((error: unknown) => {
141 throw new Error(`${section} of ${providerId}: ${errorMessage(error)}`, {
142 cause: error,
143 })
144 })
145 if (rows.length === 0) continue
146 yield separator + rows.map(stableJson).join(',')
147 separator = ','
148 }
150​
151/**
152 * The body in order: `latestChange`, then `models`, `providers`, and
153 * `schemas`. `providers` is ordered by id; `models` and `schemas` by
154 * provider id, then their own id.
155 */
156async function* exportChunks(db: Db): AsyncGenerator<string> {
157 // Read first: every change it covers is already in the rows below. The
158 // export is not one snapshot, so those rows may also hold later changes.
159 const [latestChange] = await db
160 .select({ id: changes.id, createdAt: changes.createdAt })
161 .from(changes)
162 .orderBy(desc(changes.createdAt), desc(changes.id))
163 .limit(1)
164 const providerRows = await db.select().from(providers).orderBy(providers.id)
165 const namespaceRows = await db
166 .select()
167 .from(providerModelNamespaces)
168 .orderBy(providerModelNamespaces.namespace)
169​
170 const providerIds = providerRows.map((provider) => provider.id)
171​
172 yield `{"latestChange":${stableJson(latestChange ?? null)},"models":[`
173 yield* perProvider('models', providerIds, async (provider) => {
174 const catalog = await listModelsCatalog(db, {
175 provider,
176 includeDeprecated: true,
177 pricing: true,
178 })
179 return catalog.models.map(exportModel)
180 })
181​
182 const exportedProviders = providerRows.map((provider) => ({
183 id: provider.id,
184 displayName: provider.displayName,
185 specSourceUrl: provider.specSourceUrl,
186 modelsEndpoint: provider.modelsEndpoint,
187 authEnvVar: provider.authEnvVar,
188 status: provider.status,
189 modelNamespaces: namespaceRows
190 .filter((row) => row.providerId === provider.id)
191 .map((row) => row.namespace),
192 }))
193 yield `],"providers":${stableJson(exportedProviders)},"schemas":[`
194​
195 yield* perProvider('schemas', providerIds, (provider) =>
196 exportProviderSchemas(db, provider),
197 )
198 yield ']}'
200​
201/** GET /v1/admin/export — the response body. */
202export function exportCatalogStream(db: Db): ReadableStream<Uint8Array> {
203 const encoder = new TextEncoder()
204 const chunks = exportChunks(db)
205 return new ReadableStream({
206 async pull(controller) {
207 try {
208 const next = await chunks.next()
209 if (next.done) controller.close()
210 else controller.enqueue(encoder.encode(next.value))
211 } catch (error) {
212 console.error(
213 JSON.stringify({
214 job: 'catalog-export',
215 error: errorMessage(error),
216 stack: error instanceof Error ? error.stack : undefined,
217 }),
218 )
219 controller.error(error)
220 }
221 },
222 async cancel() {
223 await chunks.return(undefined)
224 },
225 })