Open History to see the last commit
1import { assert, describe, it } from "@effect/vitest";
2import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts";
3
4import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts";
5import {
6 capClaudeCatalogEffort,
7 type ClaudeModelCatalog,
8 formatClaudeVersionUpgradeMessage,
9 getClaudeCatalogModelCapabilities,
10 normalizeClaudeCatalogEffort,
11 resolveClaudeCatalogApiModelId,
12 resolveClaudeCatalogContextWindowTokens,
13 resolveClaudeCatalogEffort,
14 resolveClaudeModelCatalog,
15 resolveClaudeModelsForVersion,
16 resolveClaudeModelSlug,
17 resolveClaudeUpdateRequiredModels,
18 scopeClaudeModelCatalog,
19} from "./ClaudeModelCatalog.ts";
20
21const manifest = (): ModelManifestData => ({
22 version: 1,
23 currentModels: {},
24 providers: {
25 claudeAgent: {
26 profiles: {
27 synthetic: {
28 capabilities: {
29 optionDescriptors: [
30 {
31 id: "effort",
32 label: "Reasoning",
33 type: "select",
34 options: [{ id: "extreme", label: "Extreme", isDefault: true }],
35 },
36 {
37 id: "contextWindow",
38 label: "Context Window",
39 type: "select",
40 options: [
41 { id: "large", label: "Large", isDefault: true },
42 { id: "small", label: "Small" },
43 ],
44 },
45 ],
46 },
47 adapter: {
48 claudeCode: {
49 effortMap: { extreme: "high" },
50 modelSuffixes: { contextWindow: { large: "[large]" } },
51 },
52 },
53 },
54 },
55 models: [
56 {
57 slug: "claude-synthetic-next",
58 name: "Claude Synthetic Next",
59 aliases: ["synthetic"],
60 status: "current",
61 profile: "synthetic",
62 adapter: { claudeCode: { minVersion: "3.2.0" } },
63 },
64 ],
65 },
66 },
67});
68
69const CLAUDE = ProviderDriverKind.make("claudeAgent");
70
71const catalogFromManifest = (data: ModelManifestData) =>
72 resolveClaudeModelCatalog(resolveProviderCatalog(data, CLAUDE) ?? undefined);
73
74describe("Claude model catalog", () => {
75 it("resolves capacity from selected options and fixed catalog windows without guessing custom models", () => {
76 const source = manifest();
77 const profile = source.providers!.claudeAgent!.profiles.synthetic!;
78 const catalog = catalogFromManifest({
79 ...source,
80 providers: {
81 claudeAgent: {
82 ...source.providers!.claudeAgent!,
83 profiles: {
84 fixed: {
85 capabilities: { optionDescriptors: [] },
86 adapter: { claudeCode: { fixedContextWindowTokens: 64_000 } },
87 },
88 synthetic: {
89 ...profile,
90 adapter: { claudeCode: { contextWindowTokens: { large: 1_000_000, small: 32_000 } } },
91 },
92 },
93 models: [
94 ...source.providers!.claudeAgent!.models,
95 {
96 slug: "fixed",
97 name: "Fixed",
98 status: "current",
99 profile: "fixed",
100 },
101 ],
102 },
103 },
104 });
105 const selection = { instanceId: ProviderInstanceId.make("claudeAgent"), model: "synthetic" };
106 assert.equal(resolveClaudeCatalogContextWindowTokens(catalog, selection), 1_000_000);
107 assert.equal(
108 resolveClaudeCatalogContextWindowTokens(catalog, {
109 ...selection,
110 options: [{ id: "contextWindow", value: "small" }],
111 }),
112 32_000,
113 );
114 assert.equal(
115 resolveClaudeCatalogContextWindowTokens(catalog, { ...selection, model: "fixed" }),
116 64_000,
117 );
118 assert.isUndefined(
119 resolveClaudeCatalogContextWindowTokens(catalog, { ...selection, model: "custom" }),
120 );
121 });
122
123 it("filters models at runtime-version boundaries and derives the upgrade message", () => {
124 const catalog = catalogFromManifest(manifest());
125 assert.deepStrictEqual(resolveClaudeModelsForVersion(catalog, "3.1.9"), []);
126 assert.deepStrictEqual(
127 resolveClaudeModelsForVersion(catalog, "3.2.0").map((model) => model.slug),
128 ["claude-synthetic-next"],
129 );
130 assert.deepStrictEqual(resolveClaudeUpdateRequiredModels(catalog, "3.1.9"), [
131 { slug: "claude-synthetic-next", name: "Claude Synthetic Next", minVersion: "3.2.0" },
132 ]);
133 assert.deepStrictEqual(resolveClaudeUpdateRequiredModels(catalog, "3.2.0"), []);
134 assert.strictEqual(
135 formatClaudeVersionUpgradeMessage(catalog, "3.1.9"),
136 "Claude Code v3.1.9 is too old for Claude Synthetic Next. Upgrade to v3.2.0 or newer to access it.",
137 );
138 });
139
140 it("resolves aliases and declarative adapter mappings", () => {
141 const base = manifest();
142 const input: ModelManifestData = {
143 ...base,
144 providers: {
145 ...base.providers,
146 claudeAgent: {
147 ...base.providers!.claudeAgent!,
148 models: [
149 {
150 slug: "claude-synthetic-collision",
151 name: "Claude Synthetic Collision",
152 aliases: ["claude-synthetic-next"],
153 status: "current",
154 },
155 ...base.providers!.claudeAgent!.models,
156 ],
157 },
158 },
159 };
160 const catalog = catalogFromManifest(input);
161 assert.strictEqual(resolveClaudeModelSlug(catalog, "synthetic"), "claude-synthetic-next");
162 assert.strictEqual(
163 resolveClaudeModelSlug(catalog, "claude-synthetic-next"),
164 "claude-synthetic-next",
165 );
166 assert.strictEqual(normalizeClaudeCatalogEffort(catalog, "extreme", "synthetic"), "high");
167 assert.strictEqual(
168 resolveClaudeCatalogApiModelId(catalog, {
169 instanceId: ProviderInstanceId.make("claudeAgent"),
170 model: "synthetic",
171 }),
172 "claude-synthetic-next[large]",
173 );
174 });
175
176 it("appends custom models with their own descriptors and keeps bare slugs opaque", () => {
177 const catalog = scopeClaudeModelCatalog(catalogFromManifest(manifest()), [
178 "synthetic",
179 {
180 slug: "claude-custom-tuned",
181 name: "Tuned",
182 capabilities: {
183 optionDescriptors: [
184 {
185 id: "effort",
186 label: "Reasoning",
187 type: "select",
188 options: [
189 { id: "gentle", label: "Gentle", isDefault: true },
190 { id: "brutal", label: "Brutal" },
191 { id: "ultracode", label: "Ultracode" },
192 ],
193 },
194 ],
195 },
196 },
197 ]);
198
199 assert.strictEqual(resolveClaudeModelSlug(catalog, "synthetic"), "synthetic");
200 assert.isUndefined(resolveClaudeCatalogEffort(catalog, "synthetic", "extreme"));
201 assert.strictEqual(resolveClaudeCatalogEffort(catalog, "synthetic", "max"), "max");
202 assert.strictEqual(normalizeClaudeCatalogEffort(catalog, "max", "synthetic"), "max");
203
204 assert.strictEqual(
205 resolveClaudeCatalogEffort(catalog, "claude-custom-tuned", "brutal"),
206 "brutal",
207 );
208 assert.strictEqual(
209 resolveClaudeCatalogEffort(catalog, "claude-custom-tuned", "bogus"),
210 "gentle",
211 );
212 assert.deepStrictEqual(
213 getClaudeCatalogModelCapabilities(catalog, "claude-custom-tuned").optionDescriptors?.map(
214 ({ id, type }) => `${id}:${type}`,
215 ),
216 ["effort:select", "ultracode:boolean"],
217 );
218 assert.strictEqual(
219 normalizeClaudeCatalogEffort(catalog, "brutal", "claude-custom-tuned"),
220 "brutal",
221 );
222 assert.strictEqual(
223 resolveClaudeCatalogApiModelId(catalog, {
224 instanceId: ProviderInstanceId.make("claudeAgent"),
225 model: "claude-custom-tuned",
226 options: [{ id: "effort", value: "brutal" }],
227 }),
228 "claude-custom-tuned",
229 );
230 assert.deepStrictEqual(
231 resolveClaudeModelsForVersion(catalog, "3.2.0").map((model) => model.slug),
232 ["claude-synthetic-next", "claude-custom-tuned"],
233 );
234 });
235
236 it("marks effort options that would run above the cap without changing ids or defaults", () => {
237 const effortModel = (slug: string) => ({
238 slug,
239 name: slug,
240 isCustom: false,
241 capabilities: {
242 optionDescriptors: [
243 {
244 id: "effort",
245 label: "Reasoning",
246 type: "select" as const,
247 options: [
248 { id: "medium", label: "Medium" },
249 { id: "high", label: "High", isDefault: true },
250 { id: "xhigh", label: "Extra High" },
251 { id: "max", label: "Max" },
252 { id: "ultracode", label: "Ultracode" },
253 { id: "ultrathink", label: "Ultrathink" },
254 ],
255 },
256 ],
257 },
258 });
259 const models = [effortModel("claude-plain"), effortModel("claude-remapped")];
260 const catalog: ClaudeModelCatalog = {
261 models: [
262 {
263 model: models[0]!,
264 runtime: { effortMap: { ultracode: "xhigh", ultrathink: null } },
265 compatibility: {},
266 },
267 {
268 model: models[1]!,
269 runtime: { effortMap: { max: "high", xhigh: "max", ultrathink: null } },
270 compatibility: {},
271 },
272 ],
273 };
274 const options = (cap: "high" | "xhigh") =>
275 capClaudeCatalogEffort(catalog, models, [{ maxEffortLevel: cap }]).map((model) => {
276 const effort = model.capabilities?.optionDescriptors?.[0];
277 return effort?.type === "select" ? effort.options : undefined;
278 });
279
280 assert.deepStrictEqual(options("high"), [
281 [
282 { id: "medium", label: "Medium" },
283 { id: "high", label: "High", isDefault: true },
284 { id: "xhigh", label: "Extra High (capped to High)" },
285 { id: "max", label: "Max (capped to High)" },
286 { id: "ultracode", label: "Ultracode (capped to High)" },
287 { id: "ultrathink", label: "Ultrathink" },
288 ],
289 [
290 { id: "medium", label: "Medium" },
291 { id: "high", label: "High", isDefault: true },
292 { id: "xhigh", label: "Extra High (capped to High)" },
293 { id: "max", label: "Max" },
294 { id: "ultracode", label: "Ultracode" },
295 { id: "ultrathink", label: "Ultrathink" },
296 ],
297 ]);
298 // An effortMap that sends Extra High as max stays unmarked under an xhigh cap: the CLI runs
299 // it at xhigh, which is what the label says.
300 assert.deepStrictEqual(
301 options("xhigh").map((model) => model?.map((option) => option.label)),
302 [
303 ["Medium", "High", "Extra High", "Max (capped to Extra High)", "Ultracode", "Ultrathink"],
304 ["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"],
305 ],
306 );
307
308 // A per-model cap applies without a top-level one and leaves other models uncapped.
309 assert.deepStrictEqual(
310 capClaudeCatalogEffort(catalog, models, [
311 { modelMaxEffortLevels: new Map([["claude-plain", "high"]]) },
312 ]).map((model) => {
313 const effort = model.capabilities?.optionDescriptors?.[0];
314 return effort?.type === "select" ? effort.options.map((option) => option.label) : [];
315 }),
316 [
317 [
318 "Medium",
319 "High",
320 "Extra High (capped to High)",
321 "Max (capped to High)",
322 "Ultracode (capped to High)",
323 "Ultrathink",
324 ],
325 ["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"],
326 ],
327 );
328
329 // Without an Extra High option (Opus 4.6), the CLI clamps an xhigh cap down to High.
330 const noXhigh = {
331 slug: "claude-no-xhigh",
332 name: "claude-no-xhigh",
333 isCustom: false,
334 capabilities: {
335 optionDescriptors: [
336 {
337 id: "effort",
338 label: "Reasoning",
339 type: "select" as const,
340 options: [
341 { id: "low", label: "Low" },
342 { id: "medium", label: "Medium" },
343 { id: "high", label: "High", isDefault: true },
344 { id: "max", label: "Max" },
345 ],
346 },
347 ],
348 },
349 };
350 const [capped] = capClaudeCatalogEffort(
351 { models: [] },
352 [noXhigh],
353 [{ maxEffortLevel: "xhigh" }],
354 );
355 const cappedEffort = capped?.capabilities?.optionDescriptors?.[0];
356 assert.deepStrictEqual(
357 cappedEffort?.type === "select" ? cappedEffort.options.map((option) => option.label) : [],
358 ["Low", "Medium", "High", "Max (capped to High)"],
359 );
360 });
361
362 it("leaves custom model effort labels alone under a cap", () => {
363 const custom = {
364 slug: "claude-custom-copy",
365 name: "Copy",
366 isCustom: true,
367 capabilities: {
368 optionDescriptors: [
369 {
370 id: "effort",
371 label: "Reasoning",
372 type: "select" as const,
373 options: [
374 { id: "high", label: "High", isDefault: true },
375 { id: "max", label: "Max (capped to High)" },
376 ],
377 },
378 ],
379 },
380 };
381
382 assert.deepStrictEqual(
383 capClaudeCatalogEffort({ models: [] }, [custom], [{ maxEffortLevel: "high" }]),
384 [custom],
385 );
386 });
387
388 it("resolves maxEffortLevel per settings source and per model key like Claude Code", () => {
389 const opus = {
390 slug: "claude-opus-5-5",
391 name: "Opus 5.5",
392 isCustom: false,
393 aliases: ["opus", "opus-5.5", "claude-opus-5.5"],
394 capabilities: {
395 optionDescriptors: [
396 {
397 id: "effort",
398 label: "Reasoning",
399 type: "select" as const,
400 options: [
401 { id: "low", label: "Low" },
402 { id: "medium", label: "Medium" },
403 { id: "high", label: "High", isDefault: true },
404 { id: "max", label: "Max" },
405 ],
406 },
407 ],
408 },
409 };
410 const catalog: ClaudeModelCatalog = {
411 models: [
412 { model: opus, runtime: {}, compatibility: {} },
413 {
414 model: {
415 slug: "claude-opus-5",
416 name: "Opus 5",
417 isCustom: false,
418 capabilities: { optionDescriptors: [] },
419 },
420 runtime: {},
421 compatibility: {},
422 },
423 ],
424 };
425 const maxLabel = (sources: Parameters<typeof capClaudeCatalogEffort>[2]) => {
426 const effort = capClaudeCatalogEffort(catalog, [opus], sources)[0]?.capabilities
427 ?.optionDescriptors?.[0];
428 return effort?.type === "select" ? effort.options.at(-1)?.label : undefined;
429 };
430 const perModel = (entries: Record<string, "low" | "medium" | "high" | "max">) => ({
431 modelMaxEffortLevels: new Map(Object.entries(entries)),
432 });
433
434 // Claude Code matches family aliases and [1m] spellings case-insensitively.
435 assert.strictEqual(maxLabel([perModel({ Opus: "medium" })]), "Max (capped to Medium)");
436 assert.strictEqual(
437 maxLabel([perModel({ "claude-opus-5-5[1M]": "medium" })]),
438 "Max (capped to Medium)",
439 );
440 assert.strictEqual(maxLabel([perModel({ " OPUS[1m] ": "medium" })]), "Max (capped to Medium)");
441 // Other keys name the most specific model id they contain, so dated and provider-prefixed ids
442 // match while the catalog's dotted aliases and a less specific model id do not.
443 for (const key of [
444 "claude-opus-5-5-20260101",
445 "CLAUDE-OPUS-5-5-20260101[1m]",
446 "us.anthropic.claude-opus-5-5-v1:0",
447 "claude-opus-5-5-latest",
448 ]) {
449 assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max (capped to Medium)", key);
450 }
451 for (const key of ["claude-opus-5.5", "opus-5.5", "opus-5-5", "opus [1m]", "claude-opus-5"]) {
452 assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max", key);
453 }
454 // Keys naming the same model keep the lowest cap, and a `max` key does not exempt it.
455 assert.strictEqual(
456 maxLabel([perModel({ opus: "max", "claude-opus-5-5": "low" })]),
457 "Max (capped to Low)",
458 );
459 // A per-model `max` only exempts the model from its own source's top-level cap.
460 assert.strictEqual(
461 maxLabel([{ maxEffortLevel: "medium", ...perModel({ opus: "max" }) }]),
462 "Max",
463 );
464 assert.strictEqual(
465 maxLabel([perModel({ opus: "max" }), { maxEffortLevel: "high" }]),
466 "Max (capped to High)",
467 );
468 assert.strictEqual(
469 maxLabel([{ maxEffortLevel: "medium" }, perModel({ opus: "high" })]),
470 "Max (capped to Medium)",
471 );
472 });
473});