Open History to see the last commit
1// Run one provider's live probes. Spends a key and calls the provider.
2// Never prints the key.
3//
4// pnpm run probe -- <provider> [--write] [--skip-missing-key]
5//
6// Route records (opencode-go): `--write` sets `verifiedAt` to today on
7// records that still match; a mismatch exits 1 and leaves the file alone.
8//
9// Chat probes (`chatProbes`, issue #717): every model gets each check in
10// `probeChecks(probe)`. A response that states a fact is printed; `--write`
11// stores it in records/<provider>.ts. An undecided check prints why and
12// keeps the stored record as it was (printed as not reconfirmed). More than MAX_PROBE_REQUESTS planned
13// requests aborts before the first. `--skip-missing-key` (the weekly CI
14// job) reports a missing key and exits 0.
15import { spawnSync } from 'node:child_process'
16import { readFileSync, writeFileSync } from 'node:fs'
17import { userInfo } from 'node:os'
18
19import {
20 MAX_PROBE_REQUESTS,
21 classifyProbeResponse,
22 mergeProbedRecords,
23 probeBody,
24 probeChecks,
25} from '../src/server/providers/probes/chat-completions.ts'
26import type { ChatCompletionsProbe } from '../src/server/providers/probes/chat-completions.ts'
27import {
28 bumpVerifiedAt,
29 observationMatches,
30} from '../src/server/providers/probes/compare.ts'
31import {
32 chatProbes,
33 probedFactFile,
34 probedFactRecords,
35 probeRecords,
36 probesFor,
37} from '../src/server/providers/probes/index.ts'
38import { OPENCODE_GO_PROBE_SOURCE } from '../src/server/providers/probes/opencode-go.ts'
39import type {
40 ProbedFactRecord,
41 ProbeRecord,
42} from '../src/server/providers/probes/types.ts'
43
44const ROUTE_PROBES: Record<
45 string,
46 { file: string; origin: string; keyEnv: string }
47> = {
48 'opencode-go': {
49 file: OPENCODE_GO_PROBE_SOURCE,
50 origin: 'https://opencode.ai',
51 keyEnv: 'OPENCODE_GO_API_KEY',
52 },
53}
54
55// pnpm 11 passes the `--` through.
56const providerId = process.argv.slice(2).find((arg) => arg !== '--')
57const write = process.argv.includes('--write')
58const skipMissingKey = process.argv.includes('--skip-missing-key')
59const chat = chatProbes.find((probe) => probe.provider === providerId)
60const known = [
61 ...new Set([
62 ...probeRecords.map((record) => record.provider),
63 ...chatProbes.map((probe) => probe.provider),
64 ]),
65]
66if (!providerId || !known.includes(providerId)) {
67 console.error(
68 `usage: pnpm run probe -- <provider> [--write] [--skip-missing-key]\nproviders: ${known.join(', ')}`,
69 )
70 process.exit(1)
71}
72const today = new Date().toISOString().slice(0, 10)
73
74if (chat) await runChatProbe(chat)
75else await runRouteProbes()
76
77async function runRouteProbes(): Promise<void> {
78 const config = ROUTE_PROBES[providerId!]
79 if (!config) throw new Error(`probe: ${providerId} has no route config`)
80 const records = probesFor(providerId!)
81 const apiKey = readKey(config.keyEnv)
82 const mismatches: Array<string> = []
83 for (const record of records) {
84 const mismatch = await runRecord(record, config, apiKey)
85 if (mismatch) mismatches.push(`${record.id}: ${mismatch}`)
86 else console.log(`${record.id}: match`)
87 }
88 if (mismatches.length > 0) {
89 for (const line of mismatches) console.error(line)
90 process.exit(1)
91 }
92 if (write) {
93 const next = records.reduce(
94 (source, record) => bumpVerifiedAt(source, record.id, today),
95 readFileSync(config.file, 'utf8'),
96 )
97 writeFileSync(config.file, next)
98 console.log(`wrote verifiedAt ${today} in ${config.file}`)
99 }
100}
101
102async function runChatProbe(probe: ChatCompletionsProbe): Promise<void> {
103 const checks = probeChecks(probe)
104 const planned = probe.models.length * checks.length
105 if (planned > MAX_PROBE_REQUESTS) {
106 console.error(
107 `probe: ${probe.provider} plans ${planned} requests, over the per-run cap of ${MAX_PROBE_REQUESTS}. Shorten its model list.`,
108 )
109 process.exit(1)
110 }
111 const apiKey = readKey(probe.keyEnv)
112 const ranBy =
113 process.env.TANSTACK_SOURCE_JOB_ID || process.env.CI
114 ? 'weekly-probe'
115 : userInfo().username
116 const found: Array<ProbedFactRecord> = []
117 let sent = 0
118 for (const model of probe.models) {
119 for (const check of checks) {
120 // Belt and braces: the plan above already bounds this.
121 if (++sent > MAX_PROBE_REQUESTS) throw new Error('probe: request cap hit')
122 const body = probeBody(check, model)
123 const { status, text } = await send(probe.url, body, apiKey)
124 const outcome = classifyProbeResponse(
125 check,
126 status,
127 text,
128 probe.mandatoryRule,
129 )
130 const label = `${model} ${check}`
131 if ('undecided' in outcome) {
132 console.log(
133 `${label}: undecided (${redact(outcome.undecided, apiKey)})`,
134 )
135 continue
136 }
137 console.log(`${label}: ${outcome.fact} = ${String(outcome.value)}`)
138 found.push({
139 id: `${model}:${outcome.fact}`,
140 provider: probe.provider,
141 model,
142 fact: outcome.fact,
143 value: outcome.value,
144 route: probe.route,
145 verifiedAt: today,
146 ranBy,
147 request: { url: probe.url, body },
148 evidence: { status, message: redact(outcome.message, apiKey) },
149 })
150 }
151 }
152 const { records, unconfirmed } = mergeProbedRecords(
153 probe.provider,
154 probedFactRecords,
155 found,
156 new Date(),
157 )
158 // Kept, never overwritten by an undecided check, but visible.
159 for (const record of unconfirmed) {
160 console.log(`stored record not reconfirmed: ${record.model} ${record.fact}`)
161 }
162 if (!write) return
163 const file = probedFactFile(probe.provider)
164 writeFileSync(
165 file,
166 `// Written by \`pnpm run probe -- ${probe.provider} --write\`. Do not edit by hand.\nimport type { ProbedFactRecord } from '../types.ts'\n\nexport const records: Array<ProbedFactRecord> = ${JSON.stringify(records, null, 2)}\n`,
167 )
168 const format = spawnSync('pnpm', ['exec', 'oxfmt', file], {
169 stdio: 'inherit',
170 })
171 if (format.status !== 0) throw new Error(`probe: oxfmt failed on ${file}`)
172 console.log(`wrote ${records.length} records to ${file}`)
173}
174
175/** A 2xx body is cancelled unread: it states nothing, and it bills. */
176async function send(
177 url: string,
178 body: Record<string, unknown>,
179 secret: string,
180): Promise<{ status: number; text: string }> {
181 try {
182 const response = await fetch(url, {
183 method: 'POST',
184 headers: {
185 authorization: `Bearer ${secret}`,
186 'content-type': 'application/json',
187 'user-agent': 'modelschemas-probe/1',
188 },
189 body: JSON.stringify(body),
190 signal: AbortSignal.timeout(30_000),
191 })
192 if (response.ok) {
193 await response.body?.cancel()
194 return { status: response.status, text: '' }
195 }
196 return { status: response.status, text: await response.text() }
197 } catch {
198 // The error can quote the request; report only that it failed.
199 return { status: 0, text: '' }
200 }
201}
202
203function redact(text: string, secret: string): string {
204 return secret ? text.replaceAll(secret, '[key]') : text
205}
206
207function readKey(keyEnv: string): string {
208 const fromEnv = process.env[keyEnv]?.trim()
209 if (fromEnv) return fromEnv
210 if (skipMissingKey) {
211 console.log(`skip ${providerId}: ${keyEnv} is not set`)
212 process.exit(0)
213 }
214 const result = spawnSync(
215 'doppler',
216 [
217 'secrets',
218 'get',
219 keyEnv,
220 '--project',
221 'modelschemas',
222 '--config',
223 'dev',
224 '--plain',
225 ],
226 { encoding: 'utf8' },
227 )
228 const value = result.error ? '' : result.stdout.trim()
229 if (result.status !== 0 || value.length === 0) {
230 console.error(
231 `${keyEnv} is missing from the environment and from Doppler dev.`,
232 )
233 const detail = result.error ? result.error.message : result.stderr.trim()
234 if (detail) console.error(detail)
235 process.exit(1)
236 }
237 return value
238}
239
240function headersFor(
241 record: ProbeRecord,
242 keyEnv: string,
243 secret: string,
244): Record<string, string> {
245 const headers: Record<string, string> = {}
246 for (const [name, value] of Object.entries(record.request.headers)) {
247 headers[name] = value.replaceAll(`$${keyEnv}`, secret)
248 }
249 return headers
250}
251
252async function runRecord(
253 record: ProbeRecord,
254 config: { origin: string; keyEnv: string },
255 secret: string,
256): Promise<string | null> {
257 const response = await fetch(`${config.origin}${record.request.path}`, {
258 method: record.request.method,
259 headers: headersFor(record, config.keyEnv, secret),
260 body: JSON.stringify(record.request.body),
261 })
262 const text = await response.text()
263 const payloads = record.expect.sse ? ssePayloads(text) : [parseJson(text)]
264 return observationMatches(record.expect, {
265 status: response.status,
266 payloads: payloads.filter((payload) => payload !== undefined),
267 })
268}
269
270function parseJson(text: string): unknown {
271 try {
272 return JSON.parse(text) as unknown
273 } catch {
274 return undefined
275 }
276}
277
278function ssePayloads(text: string): Array<unknown> {
279 return text
280 .split('\n')
281 .filter((line) => line.startsWith('data:'))
282 .map((line) => parseJson(line.slice('data:'.length).trim()))
283 .filter((payload) => payload !== undefined)
284}