Open History to see the last commit
1import { describe, expect, it } from "@effect/vitest";
2
3import {
4 decodeScanCache,
5 makeScanCacheWriter,
6 type CachedFile,
7 type ScanCache,
8} from "./usageScanCache.ts";
9import type { UsageRecord } from "@t3tools/provider-core/server/usage";
10
11import { TEST_FORMAT_MAP } from "./usageTestFormats.ts";
12import { initialCodexScanState, parseCodexLine } from "../provider/Drivers/codexUsage.ts";
13
14const parse = (parts: Iterable<string>) => JSON.parse([...parts].join(""));
15
16function record(overrides: Partial<UsageRecord> = {}): UsageRecord {
17 return {
18 provider: "claude",
19 timestampMs: 1_786_000_000_000,
20 model: "claude-fable-5",
21 sessionId: "session-a",
22 totals: {
23 uncachedInputTokens: 2,
24 cachedInputTokens: 1000,
25 cacheCreationTokens: 10,
26 outputTokens: 50,
27 reasoningTokens: 0,
28 },
29 reportedCostUsd: null,
30 speed: "standard",
31 dedupeKey: "msg_1:",
32 ...overrides,
33 };
34}
35
36function position(overrides: Partial<CachedFile["position"]> = {}): CachedFile["position"] {
37 return {
38 resumeOffset: 120,
39 guardLength: 64,
40 guardHash: 0xdeadbeef,
41 state: null,
42 ...overrides,
43 };
44}
45
46function cacheWith(entries: readonly [string, number, readonly UsageRecord[]][]): ScanCache {
47 const cache: ScanCache = new Map();
48 for (const [path, mtimeMs, records] of entries) {
49 cache.set(path, {
50 size: records.length * 10,
51 mtimeMs,
52 provider: "claude",
53 records,
54 tailRecords: [],
55 malformedRecords: 0,
56 position: position(),
57 });
58 }
59 return cache;
60}
61
62describe("scan cache round trip", () => {
63 it("restores cached history unchanged except for Claude local placeholders", () => {
64 const original = cacheWith([
65 [
66 "/a.jsonl",
67 100,
68 [
69 record(),
70 record({
71 dedupeKey: "msg_2:",
72 model: "claude-opus-5-5",
73 speed: "fast",
74 cacheCreation1hTokens: 8,
75 }),
76 ],
77 ],
78 [
79 "/b.jsonl",
80 200,
81 [
82 record({
83 sessionId: "session-b",
84 reportedCostUsd: 1.5,
85 totals: {
86 uncachedInputTokens: 0,
87 cachedInputTokens: 0,
88 cacheCreationTokens: 0,
89 outputTokens: 0,
90 reasoningTokens: 0,
91 },
92 }),
93 ],
94 ],
95 ]);
96 original.set("/grok.jsonl", {
97 size: 40,
98 malformedRecords: 2,
99 mtimeMs: 300,
100 provider: "grok",
101 records: [
102 record({ provider: "grok", model: "grok-4.5-build", dedupeKey: "s:p:grok-4.5-build" }),
103 record({ provider: "grok", model: "<synthetic>", dedupeKey: "s:p:<synthetic>" }),
104 ],
105 tailRecords: [record({ provider: "grok", model: "<synthetic>", dedupeKey: null })],
106 position: position({ resumeOffset: 30, guardLength: 30, guardHash: 123 }),
107 });
108 const state = initialCodexScanState();
109 for (const line of [
110 { type: "session_meta", payload: { id: "session-c", model_provider: "openai" } },
111 { type: "turn_context", payload: { model: "gpt-6-astra", turn_id: "native-turn" } },
112 {
113 type: "event_msg",
114 timestamp: "2026-08-01T10:00:00Z",
115 payload: {
116 type: "token_count",
117 info: { last_token_usage: { input_tokens: 1, output_tokens: 2 } },
118 },
119 },
120 ])
121 parseCodexLine(JSON.stringify(line), state);
122 const pending = {
123 ...record({
124 provider: "codex",
125 model: "gpt-6-astra",
126 sessionId: "session-c",
127 dedupeKey: null,
128 }),
129 unresolvedServiceTierTurnId: "native-turn",
130 };
131 const pendingTail = { ...pending, unresolvedServiceTierTurnId: "native-tail" };
132 original.set("/codex.jsonl", {
133 size: 80,
134 malformedRecords: 3,
135 mtimeMs: 400,
136 provider: "codex",
137 records: [
138 record({
139 provider: "codex",
140 model: "gpt-6-astra",
141 rateModel: "gpt-6",
142 dedupeKey: null,
143 speed: "ultrafast",
144 }),
145 pending,
146 ],
147 tailRecords: [pendingTail],
148 position: position({ state }),
149 });
150
151 original.set("/a.jsonl", {
152 ...original.get("/a.jsonl")!,
153 tailRecords: [record({ dedupeKey: "msg_tail:" })],
154 });
155 const claude = original.get("/a.jsonl")!;
156 const cached = new Map(original);
157 cached.set("/a.jsonl", {
158 ...claude,
159 records: [...claude.records, record({ model: "<synthetic>" })],
160 tailRecords: [
161 ...claude.tailRecords,
162 record({ model: "<synthetic>", dedupeKey: "msg_tail:" }),
163 ],
164 });
165 const restored = decodeScanCache(parse(makeScanCacheWriter()(cached, {})), TEST_FORMAT_MAP);
166
167 expect(restored.size).toBe(4);
168 expect(restored).toEqual(original);
169 });
170
171 it.each([5, 6, 7, 8, 10, 11, 12, 13])(
172 "retains v%s history and compatible positions but invalidates obsolete state",
173 (version) => {
174 const original = cacheWith([["/gone.jsonl", 100, [record({ speed: "fast" })]]]);
175 original.set("/gone-codex.jsonl", {
176 ...original.get("/gone.jsonl")!,
177 provider: "codex",
178 malformedRecords: 1,
179 records: [record({ provider: "codex", speed: "fast", dedupeKey: null })],
180 position: position({
181 state: {
182 ...(version >= 6 ? initialCodexScanState() : {}),
183 model: "gpt-6-astra",
184 speed: "fast",
185 sessionId: "session-a",
186 lastUsageSignature: null,
187 sawSessionMeta: true,
188 suppressingForkCopies: false,
189 forkCopyAnchorMs: 0,
190 },
191 }),
192 });
193 original.set("/gone-pi.jsonl", {
194 ...original.get("/gone.jsonl")!,
195 size: 256,
196 provider: "pi",
197 records: [record({ provider: "pi", model: "anthropic/claude-fable-5" })],
198 malformedRecords: 3,
199 position: position({ state: { sessionId: "session-a" } }),
200 });
201 const cleanCodex = {
202 ...original.get("/gone-codex.jsonl")!,
203 malformedRecords: 0,
204 };
205 original.set("/clean-codex.jsonl", cleanCodex);
206 const encoded = parse(makeScanCacheWriter()(original, {}));
207 const entry = encoded.files["/gone.jsonl"]!;
208 const legacy = {
209 ...encoded,
210 version,
211 files: {
212 ...encoded.files,
213 "/gone.jsonl": {
214 s: entry.s,
215 m: entry.m,
216 p: entry.p,
217 r: entry.r,
218 t: entry.t,
219 o: entry.o,
220 gl: entry.gl,
221 gh: entry.gh,
222 cs: entry.cs,
223 },
224 },
225 };
226
227 const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP);
228 expect(restored.get("/gone.jsonl")).toEqual(
229 version >= 13
230 ? original.get("/gone.jsonl")
231 : {
232 ...original.get("/gone.jsonl"),
233 size: -1,
234 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
235 },
236 );
237 expect(restored.get("/gone-codex.jsonl")).toEqual({
238 ...original.get("/gone-codex.jsonl"),
239 size: -1,
240 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
241 });
242 expect(restored.get("/clean-codex.jsonl")).toEqual({
243 ...cleanCodex,
244 size: -1,
245 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
246 });
247 expect(restored.get("/gone-pi.jsonl")).toEqual(
248 version >= 6
249 ? original.get("/gone-pi.jsonl")
250 : {
251 ...original.get("/gone-pi.jsonl"),
252 size: -1,
253 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
254 },
255 );
256 },
257 );
258
259 it("re-parses a v14 Codex entry that skipped records and restarts a clean one when it grows", () => {
260 const codex = (malformedRecords: number): CachedFile => ({
261 size: 10,
262 mtimeMs: 100,
263 provider: "codex",
264 records: [record({ provider: "codex", model: "gpt-6-astra", dedupeKey: null })],
265 tailRecords: [],
266 malformedRecords,
267 position: position({
268 state: { ...initialCodexScanState(), model: "gpt-6-astra", sessionId: "session-a" },
269 }),
270 });
271 const original: ScanCache = new Map([
272 ["/skipped-codex.jsonl", codex(1)],
273 ["/clean-codex.jsonl", codex(0)],
274 ]);
275 const v14 = { ...parse(makeScanCacheWriter()(original, {})), version: 14 };
276
277 const restored = decodeScanCache(JSON.parse(JSON.stringify(v14)), TEST_FORMAT_MAP);
278 expect(restored.get("/skipped-codex.jsonl")).toEqual({
279 ...codex(1),
280 size: -1,
281 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
282 });
283 // Served while unchanged, but its state predates the latest snapshot time, so it cannot resume.
284 expect(restored.get("/clean-codex.jsonl")).toEqual({
285 ...codex(0),
286 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
287 });
288 });
289
290 it("re-parses a Claude transcript cached before 1-hour cache writes were counted", () => {
291 const original = cacheWith([["/claude.jsonl", 100, [record()]]]);
292 const v12 = { ...parse(makeScanCacheWriter()(original, {})), version: 12 };
293
294 const restored = decodeScanCache(JSON.parse(JSON.stringify(v12)), TEST_FORMAT_MAP);
295 expect(restored.get("/claude.jsonl")).toEqual({
296 ...original.get("/claude.jsonl"),
297 size: -1,
298 position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
299 });
300 });
301
302 it("rewrites only changed entries and still restores the whole cache", () => {
303 const write = makeScanCacheWriter();
304 const cache = cacheWith([
305 ["/a.jsonl", 100, [record()]],
306 ["/b.jsonl", 200, [record({ sessionId: "session-b" })]],
307 ]);
308 cache.set("/b.jsonl", { ...cache.get("/b.jsonl")!, malformedRecords: 2 });
309 const sources = { "claude\u0000/projects": { dir: "/projects", volumeId: "1:2" } };
310 expect(decodeScanCache(parse(write(cache, { sources })), TEST_FORMAT_MAP)).toEqual(cache);
311
312 cache.set("/b.jsonl", {
313 ...cache.get("/b.jsonl")!,
314 size: 30,
315 malformedRecords: 4,
316 records: [record({ sessionId: "session-c", model: "claude-opus-5-5", dedupeKey: "msg_3:" })],
317 });
318 const document = parse(write(cache, { sources }));
319 expect(decodeScanCache(document, TEST_FORMAT_MAP)).toEqual(cache);
320 expect(document.sources).toEqual(sources);
321 });
322
323 it("treats a corrupt or foreign document as an empty cache", () => {
324 expect(decodeScanCache(null, TEST_FORMAT_MAP).size).toBe(0);
325 expect(decodeScanCache("nonsense", TEST_FORMAT_MAP).size).toBe(0);
326 expect(
327 decodeScanCache({ version: 999, models: [], sessions: [], files: {} }, TEST_FORMAT_MAP).size,
328 ).toBe(0);
329 });
330
331 it("skips malformed file entries but keeps good ones", () => {
332 const encoded = parse(makeScanCacheWriter()(cacheWith([["/good.jsonl", 100, [record()]]]), {}));
333 const withJunk = {
334 ...encoded,
335 files: { ...encoded.files, "/bad.jsonl": { s: "nope", m: 1, p: "claude", r: [] } },
336 };
337
338 const restored = decodeScanCache(JSON.parse(JSON.stringify(withJunk)), TEST_FORMAT_MAP);
339 expect([...restored.keys()]).toEqual(["/good.jsonl"]);
340 });
341
342 it("drops the whole entry when any row is corrupt, forcing a cold re-parse", () => {
343 const encoded = parse(
344 makeScanCacheWriter()(
345 cacheWith([["/a.jsonl", 100, [record(), record({ dedupeKey: "msg_2:" })]]]),
346 {},
347 ),
348 );
349 const rows = encoded.files["/a.jsonl"]!.r;
350 const poisoned = {
351 ...encoded,
352 files: {
353 "/a.jsonl": {
354 ...encoded.files["/a.jsonl"]!,
355 r: [rows[0]!, [...rows[1]!.slice(0, 3), "not-a-number", ...rows[1]!.slice(4)]],
356 },
357 },
358 };
359
360 const restored = decodeScanCache(JSON.parse(JSON.stringify(poisoned)), TEST_FORMAT_MAP);
361 expect(restored.has("/a.jsonl")).toBe(false);
362 });
363});