Open History to see the last commit
1/**
2 * Pull core (PLAN.md task 12.3). `pull()` expands selections against the
3 * live catalog and writes generated modules + the manifest; `verify()` is
4 * the zero-network production-build check; `checkUpdates()` is the one
5 * cheap dev-boot call that reports upstream drift without rewriting
6 * anything.
7 */
8import { access } from 'node:fs/promises'
9import { join } from 'node:path'
10
11import { KIND_LABEL, matchTargets, parseSelection } from './selection.ts'
12import type { ResolvedTarget } from './selection.ts'
13import {
14 atomicWrite,
15 fileRelPath,
16 readManifest,
17 targetKey,
18 writeManifest,
19} from './manifest.ts'
20import type { Manifest, ManifestEntry } from './manifest.ts'
21
22export type OptionalStyle = 'exact' | 'undefined'
23
24export interface PullConfig {
25 /** Service origin; defaults to the production deployment. */
26 baseUrl?: string
27 /** Optional API key — lifts anonymous rate limits. */
28 apiKey?: string
29 /** Directory the generated files + manifest live in (committed). */
30 outDir: string
31 /** Selection strings, e.g. `anthropic/v1/messages#request`, `gemini/*`. */
32 selections: Array<string>
33 /** Pull TypeScript modules (default) or schema-const-only modules. */
34 types?: boolean
35 optionalStyle?: OptionalStyle
36 /** Injectable for tests. */
37 fetchImpl?: typeof fetch
38 log?: (message: string) => void
39}
40
41export const DEFAULT_BASE_URL = 'https://modelschemas.com'
42
43interface Resolved {
44 baseUrl: string
45 apiKey: string | undefined
46 outDir: string
47 selections: Array<string>
48 types: boolean
49 optionalStyle: OptionalStyle
50 fetchImpl: typeof fetch
51 log: (message: string) => void
52}
53
54function resolve(config: PullConfig): Resolved {
55 return {
56 baseUrl: (config.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, ''),
57 apiKey: config.apiKey,
58 outDir: config.outDir,
59 selections: config.selections,
60 types: config.types ?? true,
61 optionalStyle: config.optionalStyle ?? 'exact',
62 fetchImpl: config.fetchImpl ?? fetch,
63 log: config.log ?? (() => undefined),
64 }
65}
66
67function authHeaders(cfg: Resolved): Record<string, string> {
68 return cfg.apiKey !== undefined
69 ? { Authorization: `Bearer ${cfg.apiKey}` }
70 : {}
71}
72
73async function fetchJson<T>(cfg: Resolved, path: string): Promise<T> {
74 const response = await cfg.fetchImpl(`${cfg.baseUrl}${path}`, {
75 headers: authHeaders(cfg),
76 })
77 if (!response.ok) {
78 throw new Error(`GET ${path} → ${String(response.status)}`)
79 }
80 return (await response.json()) as T
81}
82
83/** Expand every selection into concrete targets via the provider indexes. */
84export async function expandSelections(
85 config: PullConfig,
86): Promise<Array<ResolvedTarget>> {
87 const cfg = resolve(config)
88 const parsed = config.selections.map(parseSelection)
89 const providers = [...new Set(parsed.map((s) => s.provider))]
90 const indexes = new Map<
91 string,
92 { provider: string; activities: Record<string, Array<string>> }
93 >()
94 for (const provider of providers) {
95 indexes.set(
96 provider,
97 await fetchJson(cfg, `/v1/schemas/${encodeURIComponent(provider)}`),
98 )
99 }
100 const seen = new Set<string>()
101 const targets: Array<ResolvedTarget> = []
102 for (const selection of parsed) {
103 const index = indexes.get(selection.provider)
104 if (index === undefined) continue
105 for (const target of matchTargets(selection, index)) {
106 const key = targetKey(target)
107 if (seen.has(key)) continue
108 seen.add(key)
109 targets.push(target)
110 }
111 }
112 return targets
113}
114
115function schemaUrlPath(cfg: Resolved, target: ResolvedTarget): string {
116 const base = `/v1/schemas/${encodeURIComponent(target.provider)}/${encodeURIComponent(target.activity)}/${target.endpointId.split('/').map(encodeURIComponent).join('/')}`
117 const params = new URLSearchParams({ kind: target.kind })
118 if (cfg.types) {
119 params.set('format', 'types')
120 params.set('optional', cfg.optionalStyle)
121 }
122 return `${base}?${params.toString()}`
123}
124
125/** Schema content hash from a types ETag (`<hash>-types-v<N>-<style>`). */
126function hashFromTypesEtag(etag: string): string {
127 const bare = etag.replace(/^W\//, '').replace(/^"|"$/g, '')
128 const cut = bare.indexOf('-types-v')
129 return cut === -1 ? bare : bare.slice(0, cut)
130}
131
132function pascal(text: string): string {
133 const name = text
134 .split(/[^A-Za-z0-9]+/)
135 .filter(Boolean)
136 .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
137 .join('')
138 if (name === '') return 'Schema'
139 return /^[0-9]/.test(name) ? `_${name}` : name
140}
141
142/** Schema-const-only module for `types: false` pulls — same banner and
143 * export-name conventions as the server's `?format=types` output. */
144function schemaOnlyModule(
145 target: ResolvedTarget,
146 contentHash: string,
147 schema: unknown,
148 sourceUrl: string,
149): string {
150 const kindLabel = target.kind === 'input' ? 'Request' : 'Response'
151 const root = pascal(target.provider) + pascal(target.endpointId) + kindLabel
152 const constName = `${root.charAt(0).toLowerCase()}${root.slice(1)}Schema`
153 return [
154 `// @generated by modelschemas — do not edit; refresh with \`modelschemas update\`.`,
155 `// provider: ${target.provider} · endpoint: ${target.endpointId} · kind: ${target.kind}`,
156 `// source: ${sourceUrl}`,
157 `// schema version: ${contentHash}`,
158 ``,
159 `export type JsonSchema = { readonly [key: string]: unknown }`,
160 ``,
161 `export const ${constName}: JsonSchema = ${JSON.stringify(schema, null, 2)}`,
162 ``,
163 ].join('\n')
164}
165
166/** Format with the *user's* prettier (their config) when it's installed;
167 * otherwise return the canonical text untouched. */
168async function maybePrettier(path: string, text: string): Promise<string> {
169 try {
170 const prettier = await import('prettier')
171 const config = await prettier.resolveConfig(path)
172 return await prettier.format(text, {
173 ...(config ?? {}),
174 parser: 'typescript',
175 })
176 } catch {
177 return text
178 }
179}
180
181export interface PullSummary {
182 written: Array<string>
183 unchanged: Array<string>
184 /** Implicit-kind targets the server has no schema for. */
185 skipped: Array<string>
186 failed: Array<{ key: string; reason: string }>
187}
188
189export interface PullOptions {
190 /** Only fetch targets whose file or manifest entry is missing (dev boot). */
191 missingOnly?: boolean
192 /** Injectable clock (unix epoch seconds). */
193 now?: () => number
194}
195
196export async function pull(
197 config: PullConfig,
198 options: PullOptions = {},
199): Promise<PullSummary> {
200 const cfg = resolve(config)
201 const previous = await readManifest(cfg.outDir)
202 const entries: Record<string, ManifestEntry> = {}
203 const summary: PullSummary = {
204 written: [],
205 unchanged: [],
206 skipped: [],
207 failed: [],
208 }
209
210 const targets = await expandSelections(config)
211 let changed = false
212 for (const target of targets) {
213 const key = targetKey(target)
214 const relPath = fileRelPath(target)
215 const absPath = join(cfg.outDir, relPath)
216 const prior = previous?.entries[key]
217 const fileExists = await access(absPath).then(
218 () => true,
219 () => false,
220 )
221
222 if (options.missingOnly === true && prior !== undefined && fileExists) {
223 entries[key] = prior
224 summary.unchanged.push(key)
225 continue
226 }
227
228 const path = schemaUrlPath(cfg, target)
229 let response: Response
230 try {
231 response = await cfg.fetchImpl(`${cfg.baseUrl}${path}`, {
232 headers: {
233 ...authHeaders(cfg),
234 ...(prior !== undefined && fileExists
235 ? { 'If-None-Match': prior.etag }
236 : {}),
237 },
238 })
239 } catch (error) {
240 summary.failed.push({
241 key,
242 reason: error instanceof Error ? error.message : String(error),
243 })
244 continue
245 }
246
247 if (response.status === 304 && prior !== undefined) {
248 entries[key] = prior
249 summary.unchanged.push(key)
250 continue
251 }
252 if (response.status === 404) {
253 if (target.explicitKind) {
254 summary.failed.push({
255 key,
256 reason: `404 — no ${KIND_LABEL[target.kind]} schema for '${target.selector}'.`,
257 })
258 } else {
259 summary.skipped.push(key)
260 }
261 continue
262 }
263 if (!response.ok) {
264 summary.failed.push({
265 key,
266 reason: `GET ${path} → ${String(response.status)}`,
267 })
268 continue
269 }
270
271 const etag = response.headers.get('etag') ?? ''
272 const sourceUrl = `${cfg.baseUrl}${path}`
273 let text: string
274 let contentHash: string
275 if (cfg.types) {
276 text = await response.text()
277 contentHash = hashFromTypesEtag(etag)
278 } else {
279 const body = (await response.json()) as {
280 contentHash: string
281 schema: unknown
282 }
283 contentHash = body.contentHash
284 text = schemaOnlyModule(target, contentHash, body.schema, sourceUrl)
285 }
286 await atomicWrite(absPath, await maybePrettier(absPath, text))
287 entries[key] = { path: relPath, contentHash, etag }
288 summary.written.push(key)
289 changed = true
290 cfg.log(`pulled ${key} → ${relPath}`)
291 }
292
293 const now = options.now?.() ?? Math.floor(Date.now() / 1000)
294 const manifest: Manifest = {
295 version: 1,
296 baseUrl: cfg.baseUrl,
297 selections: cfg.selections,
298 types: cfg.types,
299 optionalStyle: cfg.optionalStyle,
300 fetchedAt: changed ? now : (previous?.fetchedAt ?? now),
301 entries,
302 }
303 if (
304 changed ||
305 previous === null ||
306 JSON.stringify(previous) !== JSON.stringify(manifest)
307 ) {
308 await writeManifest(cfg.outDir, manifest)
309 }
310 return summary
311}
312
313export interface VerifyResult {
314 ok: boolean
315 problems: Array<string>
316}
317
318/** Offline check for production builds: the manifest exists, covers the
319 * configured selections, and every generated file is present. */
320export async function verify(config: PullConfig): Promise<VerifyResult> {
321 const cfg = resolve(config)
322 const problems: Array<string> = []
323 const manifest = await readManifest(cfg.outDir)
324 if (manifest === null) {
325 return {
326 ok: false,
327 problems: [
328 `No ${cfg.outDir}/.manifest.json — run \`modelschemas pull\` (or boot the dev server) and commit the generated files.`,
329 ],
330 }
331 }
332 for (const selection of cfg.selections) {
333 if (!manifest.selections.includes(selection)) {
334 problems.push(
335 `Selection '${selection}' is not in the manifest — run \`modelschemas pull\` and commit the result.`,
336 )
337 }
338 }
339 for (const [key, entry] of Object.entries(manifest.entries)) {
340 const exists = await access(join(cfg.outDir, entry.path)).then(
341 () => true,
342 () => false,
343 )
344 if (!exists) {
345 problems.push(
346 `Missing generated file ${entry.path} (${key}) — run \`modelschemas pull\`.`,
347 )
348 }
349 }
350 return { ok: problems.length === 0, problems }
351}
352
353export interface UpstreamUpdate {
354 key: string
355 contentHash: string
356 summary: string
357}
358
359/** One cheap call per provider against `/v1/changes` — reports entries whose
360 * upstream schema hash moved past the manifest. Never rewrites files. */
361export async function checkUpdates(
362 config: PullConfig,
363): Promise<Array<UpstreamUpdate>> {
364 const cfg = resolve(config)
365 const manifest = await readManifest(cfg.outDir)
366 if (manifest === null) return []
367
368 // entry lookup by `subjectId#label` (subjectId is `provider/endpointId`).
369 const bySubject = new Map<string, { key: string; entry: ManifestEntry }>()
370 for (const [key, entry] of Object.entries(manifest.entries)) {
371 const hashIndex = key.indexOf('#')
372 if (hashIndex === -1) continue
373 const label = key.slice(hashIndex + 1)
374 const [provider, , ...endpoint] = key.slice(0, hashIndex).split('/')
375 if (provider === undefined) continue
376 bySubject.set(`${provider}/${endpoint.join('/')}#${label}`, { key, entry })
377 }
378
379 const providers = [
380 ...new Set(
381 Object.keys(manifest.entries).flatMap((key) => {
382 const provider = key.split('/')[0]
383 return provider !== undefined ? [provider] : []
384 }),
385 ),
386 ]
387 const updates: Array<UpstreamUpdate> = []
388 const reported = new Set<string>()
389 for (const provider of providers) {
390 const result = await fetchJson<{
391 changes: Array<{
392 type: string
393 subjectId: string | null
394 summary: string
395 payload?: { kind?: string; contentHash?: string } | null
396 }>
397 nextCursor: string | null
398 }>(
399 cfg,
400 `/v1/changes?provider=${encodeURIComponent(String(provider))}&type=schema.updated&since=${String(manifest.fetchedAt)}&limit=200`,
401 )
402 if (result.nextCursor !== null) {
403 cfg.log(
404 `more than one page of upstream changes for ${String(provider)} — run \`modelschemas update\`.`,
405 )
406 }
407 for (const change of result.changes) {
408 const kind = change.payload?.kind
409 const hash = change.payload?.contentHash
410 if (change.subjectId === null || kind === undefined || hash === undefined)
411 continue
412 const label = kind === 'input' ? 'request' : 'response'
413 const match = bySubject.get(`${change.subjectId}#${label}`)
414 if (
415 match !== undefined &&
416 match.entry.contentHash !== hash &&
417 !reported.has(match.key)
418 ) {
419 // Newest first — the first hit per key is the current upstream hash.
420 reported.add(match.key)
421 updates.push({
422 key: match.key,
423 contentHash: hash,
424 summary: change.summary,
425 })
426 }
427 }
428 }
429 return updates
430}