Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off
2// Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files.
3import * as NodeFSP from "node:fs/promises";
4import * as NodePath from "node:path";
5import * as NodeStringDecoder from "node:string_decoder";
6import * as NodeZlib from "node:zlib";
7
8import type {
9 SelectedFields,
10 TranscriptUsageFormat,
11 UsageRecord,
12} from "@t3tools/provider-core/server/usage";
13import * as Option from "effect/Option";
14import * as Schema from "effect/Schema";
15
16import { createTranscriptJsonReader } from "../project/AgentSessionJson.ts";
17
18export interface TranscriptFile {
19 readonly path: string;
20 readonly size: number;
21 readonly mtimeMs: number;
22}
23
24/** Resume only if bytes before resumeOffset match the guard hash; rewritten files would corrupt totals. */
25export interface TranscriptParsePosition {
26 readonly resumeOffset: number;
27 readonly guardLength: number;
28 readonly guardHash: number;
29 readonly state: unknown;
30}
31
32export interface TranscriptParseResult {
33 readonly records: readonly UsageRecord[];
34 /** Unterminated tail records stay outside records/position so the next scan rereads them. */
35 readonly tailRecords: readonly UsageRecord[];
36 /** Includes tentative tail diagnostics; position.state only includes committed lines. */
37 readonly malformedRecords: number;
38 readonly position: TranscriptParsePosition;
39 readonly resumed: boolean;
40}
41
42export const GUARD_LENGTH = 64;
43const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024;
44const NEWLINE = 0x0a;
45const STAT_CONCURRENCY = 32;
46const CARRIAGE_RETURN = 0x0d;
47
48function selectUsageFields(fields: SelectedFields) {
49 return (path: ReadonlyArray<string | number | null>): boolean => {
50 let selected: true | SelectedFields = fields;
51 for (const key of path) {
52 if (selected === true) return true;
53 if (typeof key !== "string" || !Object.hasOwn(selected, key)) return false;
54 selected = selected[key]!;
55 }
56 return true;
57 };
58}
59
60function fnv1a(buffer: Buffer): number {
61 let hash = 0x811c9dc5;
62 for (let index = 0; index < buffer.length; index += 1) {
63 hash ^= buffer[index]!;
64 hash = Math.imul(hash, 0x01000193);
65 }
66 return hash >>> 0;
67}
68
69/** Skips Grok logs without usage; preserves readdir order for first-seen deduplication. */
70export async function listTranscriptFiles(
71 root: string,
72 sinceMs: number,
73 options?: { readonly fileName?: string },
74): Promise<{ readonly files: readonly TranscriptFile[]; readonly failedPaths: number }> {
75 const fileName = options?.fileName;
76 const candidates: string[] = [];
77 let failedPaths = 0;
78 const walk = async (dir: string): Promise<void> => {
79 let entries;
80 try {
81 entries = await NodeFSP.readdir(dir, { withFileTypes: true });
82 } catch {
83 failedPaths += 1;
84 return;
85 }
86 for (const entry of entries) {
87 const child = NodePath.join(dir, entry.name);
88 if (entry.isDirectory()) await walk(child);
89 else if (
90 fileName !== undefined
91 ? entry.name === fileName
92 : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst")
93 ) {
94 candidates.push(child);
95 }
96 }
97 };
98 await walk(root);
99
100 const found: Array<TranscriptFile | undefined> = Array.from({ length: candidates.length });
101 const queue = candidates.entries();
102 const statQueued = async (): Promise<void> => {
103 for (const [index, path] of queue) {
104 try {
105 const stats = await NodeFSP.stat(path);
106 if (stats.mtimeMs >= sinceMs) {
107 found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs };
108 }
109 } catch (error) {
110 if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
111 failedPaths += 1;
112 }
113 }
114 }
115 };
116 await Promise.all(
117 Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued),
118 );
119 return { files: found.filter((file) => file !== undefined), failedPaths };
120}
121
122export async function readDirectoryVolumeId(path: string): Promise<string> {
123 try {
124 const stats = await NodeFSP.stat(path);
125 return `${stats.dev}:${stats.ino}`;
126 } catch {
127 return "";
128 }
129}
130
131async function guardMatches(
132 handle: NodeFSP.FileHandle,
133 position: TranscriptParsePosition,
134): Promise<boolean> {
135 if (position.guardLength <= 0 || position.guardLength > GUARD_LENGTH) return false;
136 try {
137 const window = Buffer.alloc(position.guardLength);
138 const { bytesRead } = await handle.read(
139 window,
140 0,
141 position.guardLength,
142 position.resumeOffset - position.guardLength,
143 );
144 return bytesRead === position.guardLength && fnv1a(window) === position.guardHash;
145 } catch {
146 return false;
147 }
148}
149
150// Match markers in one latin1 alternation; one character per byte preserves exact ASCII offsets.
151export function transcriptLineGate(markers: readonly [string, ...string[]]) {
152 if (markers.length === 1) {
153 const marker = Buffer.from(markers[0]);
154 return (line: Buffer) => line.includes(marker);
155 }
156 const pattern = new RegExp(
157 markers.map((marker) => marker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"),
158 );
159 return (line: Buffer) => pattern.test(line.toString("latin1"));
160}
161
162/** Returns null on read failure; caching it as empty would hide usage until size/mtime changes. */
163export async function readTranscriptRecords<State>(
164 filePath: string,
165 format: TranscriptUsageFormat<State>,
166 resumeFrom?: TranscriptParsePosition,
167 options?: { readonly streamingThresholdBytes?: number },
168): Promise<TranscriptParseResult | null> {
169 const streamingThresholdBytes = options?.streamingThresholdBytes ?? STREAMING_THRESHOLD_BYTES;
170 const compressed = filePath.endsWith(".jsonl.zst");
171 let handle: NodeFSP.FileHandle;
172 try {
173 handle = await NodeFSP.open(filePath, "r");
174 } catch {
175 return null;
176 }
177
178 try {
179 const codec = format.state;
180 const decodeState = codec === undefined ? undefined : Schema.decodeUnknownOption(codec.schema);
181 const encodeState = codec === undefined ? undefined : Schema.encodeSync(codec.schema);
182 let state = codec === undefined ? (undefined as State) : codec.initial();
183 let resumed = false;
184 let start = 0;
185 const savedState =
186 resumeFrom === undefined || decodeState === undefined || resumeFrom.state === null
187 ? Option.none()
188 : decodeState(resumeFrom.state);
189 if (
190 !compressed &&
191 resumeFrom !== undefined &&
192 resumeFrom.resumeOffset > 0 &&
193 (codec === undefined || Option.isSome(savedState)) &&
194 (await guardMatches(handle, resumeFrom))
195 ) {
196 if (Option.isSome(savedState)) state = savedState.value;
197 start = resumeFrom.resumeOffset;
198 resumed = true;
199 }
200
201 const toLineString = (lineBuffer: Buffer): string => {
202 const content =
203 lineBuffer.length > 0 && lineBuffer[lineBuffer.length - 1] === CARRIAGE_RETURN
204 ? lineBuffer.subarray(0, -1)
205 : lineBuffer;
206 return content.toString("utf8");
207 };
208
209 const mightCarryUsage = transcriptLineGate(format.usageMarkers);
210 const parseLine = (lineBuffer: Buffer, lineState: State, out: UsageRecord[]): void => {
211 if (!mightCarryUsage(lineBuffer)) return;
212 for (const record of format.parseLine(toLineString(lineBuffer), lineState)) out.push(record);
213 };
214
215 const records: UsageRecord[] = [];
216 let resumeOffset = start;
217 let scanOffset = start;
218 let pendingChunks: Buffer[] = [];
219 let pendingBytes = 0;
220 let streaming: ReturnType<typeof createTranscriptJsonReader> | undefined;
221 let decoder: NodeStringDecoder.StringDecoder | undefined;
222 const selectPath = selectUsageFields(format.selectFields);
223
224 const append = (segment: Buffer) => {
225 if (!streaming && pendingBytes + segment.length <= streamingThresholdBytes) {
226 if (segment.length > 0) pendingChunks.push(segment);
227 pendingBytes += segment.length;
228 return;
229 }
230 if (!streaming) {
231 // Usage has no import-history budget; retain selected fields regardless of surrounding tool payload size.
232 streaming = createTranscriptJsonReader(() => {}, selectPath, { maxDepth: Infinity });
233 decoder = new NodeStringDecoder.StringDecoder("utf8");
234 for (const pending of pendingChunks) streaming.write(decoder.write(pending));
235 pendingChunks = [];
236 pendingBytes = 0;
237 }
238 streaming.write(decoder!.write(segment));
239 };
240 const finish = (lineState: State, out: UsageRecord[]) => {
241 if (streaming) {
242 streaming.write(decoder!.end());
243 const projected = streaming.finish();
244 for (const record of format.parseProjected(projected, lineState)) out.push(record);
245 } else if (pendingBytes > 0) {
246 const line =
247 pendingChunks.length === 1
248 ? pendingChunks[0]!
249 : Buffer.concat(pendingChunks, pendingBytes);
250 parseLine(line, lineState, out);
251 }
252 pendingChunks = [];
253 pendingBytes = 0;
254 streaming = undefined;
255 decoder = undefined;
256 };
257 const input = handle.createReadStream({
258 start,
259 autoClose: compressed,
260 highWaterMark: 256 * 1024,
261 });
262 // Compressed snapshots cache a full parse; decoded byte offsets cannot
263 // resume the compressed input.
264 const stream = compressed ? input.compose(NodeZlib.createZstdDecompress()) : input;
265 for await (const chunk of stream) {
266 let lineStart = 0;
267 while (lineStart < chunk.length) {
268 const newlineIndex = chunk.indexOf(NEWLINE, lineStart);
269 if (newlineIndex === -1) {
270 append(chunk.subarray(lineStart));
271 break;
272 }
273 if (!streaming && pendingBytes === 0) {
274 parseLine(chunk.subarray(lineStart, newlineIndex), state, records);
275 } else {
276 append(chunk.subarray(lineStart, newlineIndex));
277 finish(state, records);
278 }
279 lineStart = newlineIndex + 1;
280 resumeOffset = scanOffset + lineStart;
281 }
282 scanOffset += chunk.length;
283 }
284
285 // Parse unfinished tails against copied state so the next scan can replay from resumeOffset.
286 const encodedState = encodeState === undefined ? null : encodeState(state);
287 const tailRecords: UsageRecord[] = [];
288 const tailState =
289 decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState));
290 finish(tailState, tailRecords);
291
292 const guardLength = compressed ? 0 : Math.min(GUARD_LENGTH, resumeOffset);
293 let guardHash = 0;
294 if (guardLength > 0) {
295 const window = Buffer.alloc(guardLength);
296 await handle.read(window, 0, guardLength, resumeOffset - guardLength);
297 guardHash = fnv1a(window);
298 }
299
300 return {
301 records,
302 tailRecords,
303 malformedRecords: format.readMalformedRecords?.(tailState) ?? 0,
304 position: {
305 resumeOffset: compressed ? 0 : resumeOffset,
306 guardLength,
307 guardHash,
308 state: encodedState,
309 },
310 resumed,
311 };
312 } catch {
313 return null;
314 } finally {
315 await handle.close().catch(() => undefined);
316 }
317}