Open History to see the last commit
1import { assert, describe, it } from "@effect/vitest";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { ProviderDriverKind } from "@t3tools/contracts";
4import * as ModelCatalog from "@t3tools/provider-core/server/ModelCatalog";
5import * as Effect from "effect/Effect";
6import * as FileSystem from "effect/FileSystem";
7import * as Layer from "effect/Layer";
8import * as Path from "effect/Path";
9import * as TestClock from "effect/testing/TestClock";
10import { HttpClient, HttpClientResponse } from "effect/http";
11
12import * as ServerConfig from "../config.ts";
13import * as ServerSettings from "../serverSettings.ts";
14import * as ModelManifest from "./ModelManifest.ts";
15
16describe("resolveProviderCatalog", () => {
17 it.each([
18 { releasedAt: "2026-01-01", now: "2026-01-01T00:00:00Z", badge: "new" },
19 { releasedAt: "2026-01-01", now: "2026-01-07T23:59:59.999Z", badge: "new" },
20 { releasedAt: "2026-01-01", now: "2026-01-08T00:00:00Z", badge: undefined },
21 { releasedAt: "2026-01-01", now: "2026-02-01T00:00:00Z", badge: undefined },
22 { releasedAt: "2026-01-02T00:00:00Z", now: "2026-01-01T00:00:00Z", badge: undefined },
23 { releasedAt: undefined, now: "2026-01-01T00:00:00Z", badge: undefined },
24 { releasedAt: "invalid", now: "2026-01-01T00:00:00Z", badge: undefined },
25 ])("derives the new badge from releasedAt=$releasedAt at $now", ({ releasedAt, now, badge }) => {
26 const manifest: ModelManifest.ModelManifestData = {
27 version: 1,
28 currentModels: {},
29 providers: {
30 synthetic: {
31 profiles: {},
32 models: [
33 {
34 slug: "model-next",
35 name: "Model Next",
36 status: "current",
37 ...(releasedAt === undefined ? {} : { releasedAt }),
38 },
39 {
40 slug: "model-legacy",
41 name: "Model Legacy",
42 status: "legacy",
43 releasedAt: now,
44 },
45 ],
46 },
47 },
48 };
49 const catalog = ModelManifest.resolveProviderCatalog(
50 manifest,
51 ProviderDriverKind.make("synthetic"),
52 Date.parse(now),
53 );
54 assert.strictEqual(catalog?.models[0]?.badge, badge);
55 assert.strictEqual(catalog?.models[1]?.badge, undefined);
56 });
57
58 it.effect("expires badges when rereading a cached manifest without a refresh", () => {
59 const manifest: ModelManifest.ModelManifestData = {
60 version: 1,
61 currentModels: {},
62 providers: {
63 synthetic: {
64 profiles: {},
65 models: [
66 {
67 slug: "model-next",
68 name: "Model Next",
69 status: "current",
70 releasedAt: "1970-01-01T00:00:00Z",
71 },
72 ],
73 },
74 },
75 };
76 return Effect.gen(function* () {
77 const service = yield* ModelCatalog.ModelCatalog;
78 const readCatalog = service.current(ProviderDriverKind.make("synthetic"));
79 assert.strictEqual((yield* readCatalog)?.models[0]?.badge, "new");
80 yield* TestClock.adjust("7 days");
81 assert.strictEqual((yield* readCatalog)?.models[0]?.badge, undefined);
82 }).pipe(
83 Effect.provide(
84 ModelManifest.layerModelCatalog.pipe(
85 Layer.provide(
86 Layer.succeed(ModelManifest.ModelManifest, {
87 current: Effect.succeed(manifest),
88 refresh: Effect.die("Catalog reads must not refresh the manifest"),
89 forceRefresh: Effect.die("Catalog reads must not refresh the manifest"),
90 refreshOnDemand: Effect.die("Catalog reads must not refresh the manifest"),
91 refreshInBackground: Effect.die("Catalog reads must not refresh the manifest"),
92 }),
93 ),
94 ),
95 ),
96 );
97 });
98
99 it("rejects invalid catalog references", () => {
100 const invalidCatalog = (input: {
101 readonly models: NonNullable<ModelManifest.ModelManifestData["providers"]>[string]["models"];
102 readonly defaultChat?: string;
103 }): ModelManifest.ModelManifestData => ({
104 version: 1,
105 currentModels: {},
106 providers: {
107 synthetic: {
108 ...(input.defaultChat ? { defaults: { chat: input.defaultChat } } : {}),
109 profiles: {},
110 models: input.models,
111 },
112 },
113 });
114
115 for (const invalid of [
116 invalidCatalog({
117 models: [
118 { slug: "duplicate", name: "First", status: "current" },
119 { slug: "duplicate", name: "Second", status: "current" },
120 ],
121 }),
122 invalidCatalog({
123 models: [
124 {
125 slug: "missing-profile",
126 name: "Missing Profile",
127 status: "current",
128 profile: "missing",
129 },
130 ],
131 }),
132 invalidCatalog({
133 models: [{ slug: "present", name: "Present", status: "current" }],
134 defaultChat: "absent",
135 }),
136 ]) {
137 assert.isNull(
138 ModelManifest.resolveProviderCatalog(invalid, ProviderDriverKind.make("synthetic")),
139 );
140 }
141 });
142});
143
144const REMOTE_UPDATED_AT = "2099-01-01T00:00:00Z";
145
146const REMOTE_MANIFEST: ModelManifest.ModelManifestData = {
147 version: 1,
148 updatedAt: REMOTE_UPDATED_AT,
149 currentModels: {
150 codex: ["remote-model"],
151 claudeAgent: ["remote-agent-model"],
152 },
153};
154
155const REMOTE_CLAUDE_MANIFEST: ModelManifest.ModelManifestData = {
156 version: 1,
157 updatedAt: REMOTE_UPDATED_AT,
158 currentModels: {},
159 providers: {
160 claudeAgent: {
161 profiles: {
162 synthetic: {
163 adapter: { claudeCode: { effortMap: { extreme: "high" } } },
164 },
165 },
166 models: [
167 {
168 slug: "remote-only-model",
169 name: "Remote Only Model",
170 status: "current",
171 profile: "synthetic",
172 },
173 ],
174 },
175 },
176};
177
178const remoteClaudeManifestWithCompatibility = (
179 compatibility: unknown,
180): ModelManifest.ModelManifestData => ({
181 ...REMOTE_CLAUDE_MANIFEST,
182 providers: {
183 claudeAgent: {
184 profiles: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.profiles,
185 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models.map((model) => ({
186 ...model,
187 adapter: { claudeCode: compatibility },
188 })),
189 },
190 },
191});
192
193const INVALID_REMOTE_MANIFESTS: ReadonlyArray<ModelManifest.ModelManifestData> = [
194 {
195 ...REMOTE_CLAUDE_MANIFEST,
196 providers: {
197 claudeAgent: {
198 profiles: {
199 synthetic: {
200 adapter: { claudeCode: { effortMap: { extreme: 123 } } },
201 },
202 },
203 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
204 },
205 },
206 },
207 {
208 ...REMOTE_CLAUDE_MANIFEST,
209 providers: {
210 claudeAgent: {
211 profiles: {},
212 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
213 },
214 },
215 },
216 {
217 ...REMOTE_CLAUDE_MANIFEST,
218 providers: {
219 claudeAgent: {
220 profiles: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.profiles,
221 models: [
222 ...REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
223 {
224 slug: "remote-only-model",
225 name: "Duplicate Remote Model",
226 status: "current",
227 profile: "synthetic",
228 },
229 ],
230 },
231 },
232 },
233 {
234 ...REMOTE_CLAUDE_MANIFEST,
235 providers: {
236 claudeAgent: {
237 defaults: { chat: "absent-model" },
238 profiles: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.profiles,
239 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
240 },
241 },
242 },
243 remoteClaudeManifestWithCompatibility({ minVersion: "2.x" }),
244 remoteClaudeManifestWithCompatibility({ maxVersionExclusive: "2.x" }),
245 remoteClaudeManifestWithCompatibility({
246 minVersion: "2.2",
247 maxVersionExclusive: "2.1",
248 }),
249];
250
251const layerHttpClient = (handler: () => Response) =>
252 Layer.succeed(
253 HttpClient.HttpClient,
254 HttpClient.make((request) => Effect.succeed(HttpClientResponse.fromWeb(request, handler()))),
255 );
256
257const layerService = (input: {
258 readonly prefix: string;
259 readonly response: () => Response;
260 readonly settings?: Parameters<typeof ServerSettings.layerTest>[0];
261}) =>
262 ServerConfig.layerTest(process.cwd(), { prefix: input.prefix }).pipe(
263 Layer.provideMerge(NodeServices.layer),
264 Layer.provideMerge(ServerSettings.layerTest(input.settings ?? {})),
265 Layer.provideMerge(layerHttpClient(input.response)),
266 );
267
268describe("ModelManifest service", () => {
269 it.live("prefers a fetched manifest over the bundle and caches it to disk", () =>
270 Effect.gen(function* () {
271 const service = yield* ModelManifest.make;
272 const refreshed = yield* service.refresh;
273 assert.deepStrictEqual(refreshed, REMOTE_MANIFEST);
274
275 const rebooted = yield* ModelManifest.make;
276 assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST);
277 }).pipe(
278 Effect.scoped,
279 Effect.provide(
280 layerService({
281 prefix: "model-manifest-fetch-test",
282 response: () => Response.json(REMOTE_MANIFEST),
283 }),
284 ),
285 ),
286 );
287
288 it.live("ignores older remote edits without replacing the current manifest or disk cache", () => {
289 let remote = { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" };
290 return Effect.gen(function* () {
291 const service = yield* ModelManifest.make;
292 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
293 assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST);
294
295 remote = { ...REMOTE_MANIFEST, updatedAt: REMOTE_UPDATED_AT };
296 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
297 remote = { ...REMOTE_MANIFEST, updatedAt: ModelManifest.BUNDLED_MODEL_MANIFEST.updatedAt! };
298 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
299 const rebooted = yield* ModelManifest.make;
300 assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST);
301 }).pipe(
302 Effect.scoped,
303 Effect.provide(
304 layerService({
305 prefix: "model-manifest-stale-fetch-test",
306 response: () => Response.json(remote),
307 }),
308 ),
309 );
310 });
311
312 it.effect("preserves the last-good remote cache when later payloads are invalid", () => {
313 let responseIndex = 0;
314 const responses = [REMOTE_CLAUDE_MANIFEST, ...INVALID_REMOTE_MANIFESTS];
315
316 return Effect.gen(function* () {
317 const service = yield* ModelManifest.make;
318 assert.deepStrictEqual(yield* service.refresh, REMOTE_CLAUDE_MANIFEST);
319
320 for (const _invalid of INVALID_REMOTE_MANIFESTS) {
321 yield* TestClock.adjust("1 hour");
322 responseIndex += 1;
323 assert.deepStrictEqual(yield* service.refresh, REMOTE_CLAUDE_MANIFEST);
324 }
325
326 const rebooted = yield* ModelManifest.make;
327 assert.deepStrictEqual(yield* rebooted.current, REMOTE_CLAUDE_MANIFEST);
328 }).pipe(
329 Effect.scoped,
330 Effect.provide(
331 layerService({
332 prefix: "model-manifest-last-good-test",
333 response: () => Response.json(responses[responseIndex]),
334 }),
335 ),
336 );
337 });
338
339 it.effect("refetches for an opened model picker after five minutes, not an hour", () => {
340 let fetches = 0;
341 return Effect.gen(function* () {
342 const service = yield* ModelManifest.make;
343 yield* service.refresh;
344 yield* TestClock.adjust("4 minutes");
345 yield* service.refreshOnDemand;
346 assert.strictEqual(fetches, 1);
347
348 yield* TestClock.adjust("1 minute");
349 yield* service.refresh;
350 assert.strictEqual(fetches, 1);
351 yield* service.refreshOnDemand;
352 assert.strictEqual(fetches, 2);
353 }).pipe(
354 Effect.scoped,
355 Effect.provide(
356 layerService({
357 prefix: "model-manifest-on-demand-test",
358 response: () => {
359 fetches += 1;
360 return Response.json(REMOTE_MANIFEST);
361 },
362 }),
363 ),
364 );
365 });
366
367 it.live("drops a disk cache of a manifest older than the bundled one", () =>
368 Effect.gen(function* () {
369 const fs = yield* FileSystem.FileSystem;
370 const path = yield* Path.Path;
371 const config = yield* ServerConfig.ServerConfig;
372 const cachePath = path.join(config.stateDir, "model-manifest.json");
373 const { updatedAt: _undated, ...undatedManifest } = REMOTE_MANIFEST;
374 for (const stale of [
375 undatedManifest,
376 { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" },
377 ]) {
378 yield* fs.writeFileString(
379 cachePath,
380 yield* ModelManifest.encodeManifestCache({ fetchedAtMs: 0, manifest: stale }),
381 );
382 const service = yield* ModelManifest.make;
383 assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST);
384 }
385
386 yield* fs.writeFileString(
387 cachePath,
388 yield* ModelManifest.encodeManifestCache({ fetchedAtMs: 0, manifest: REMOTE_MANIFEST }),
389 );
390 const later = yield* ModelManifest.make;
391 assert.deepStrictEqual(yield* later.current, REMOTE_MANIFEST);
392 }).pipe(
393 Effect.scoped,
394 Effect.provide(
395 layerService({
396 prefix: "model-manifest-newer-bundle-test",
397 response: () => Response.json(REMOTE_MANIFEST),
398 }),
399 ),
400 ),
401 );
402
403 it.live("does not fetch when provider update checks are disabled", () =>
404 Effect.gen(function* () {
405 let fetchCount = 0;
406 const service = yield* ModelManifest.make.pipe(
407 Effect.provide(
408 layerHttpClient(() => {
409 fetchCount += 1;
410 return Response.json(REMOTE_MANIFEST);
411 }),
412 ),
413 );
414 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
415 assert.deepStrictEqual(yield* service.forceRefresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
416 assert.strictEqual(fetchCount, 0);
417 }).pipe(
418 Effect.scoped,
419 Effect.provide(
420 layerService({
421 prefix: "model-manifest-optout-test",
422 response: () => Response.json(REMOTE_MANIFEST),
423 settings: { enableProviderUpdateChecks: false },
424 }),
425 ),
426 ),
427 );
428});