Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

usageAggregation.test.ts

283 lines8.8 KB

1import { describe, expect, it } from "@effect/vitest";
2​
3import { resolveModelAliases, UsageAggregator } from "./usageAggregation.ts";
4import type { RateTable } from "./usagePricing.ts";
5import type { UsageRecord } from "@t3tools/provider-core/server/usage";
6​
7const rates: RateTable = new Map([
8 [
9 "claude-fable-5",
10 {
11 inputCostPerToken: 1e-5,
12 outputCostPerToken: 5e-5,
13 cacheReadCostPerToken: 1e-6,
14 cacheCreationCostPerToken: 1.25e-5,
15 fast: {
16 inputCostPerToken: 2e-5,
17 outputCostPerToken: 1e-4,
18 cacheReadCostPerToken: 2e-6,
19 cacheCreationCostPerToken: 2.5e-5,
20 },
21 ultrafast: null,
22 contextTiers: [],
23 },
24 ],
25]);
26​
27function record(overrides: Partial<UsageRecord> = {}): UsageRecord {
28 return {
29 provider: "claude",
30 timestampMs: Date.parse("2026-08-07T04:05:13.944Z"),
31 model: "claude-fable-5",
32 sessionId: "session-a",
33 totals: {
34 uncachedInputTokens: 100,
35 cachedInputTokens: 1000,
36 cacheCreationTokens: 10,
37 outputTokens: 50,
38 reasoningTokens: 0,
39 },
40 reportedCostUsd: null,
41 speed: "standard",
42 dedupeKey: null,
43 ...overrides,
44 };
46​
47function aggregate(
48 records: readonly UsageRecord[],
49 timeZone = "UTC",
50 resolution: "day" | "hour" = "day",
51) {
52 const hourlyBounds =
53 resolution === "hour"
54 ? {
55 sinceTimeMs: Date.parse("2026-08-06T04:37:00.000Z"),
56 untilTimeMs: Date.parse("2026-08-07T04:37:00.000Z"),
57 }
58 : {};
59 const aggregator = new UsageAggregator({
60 timeZone,
61 sinceDay: "2026-08-01",
62 untilDay: "2026-08-31",
63 resolution,
64 ...hourlyBounds,
65 rates,
66 });
67 for (const item of records) aggregator.add(item);
68 return aggregator.finish();
70​
71describe("UsageAggregator", () => {
72 it("splits a bucket's cost by category and speed", () => {
73 const [bucket] = aggregate([record(), record({ speed: "fast" })]).buckets;
74​
75 expect(bucket).toMatchObject({
76 costUsd: expect.closeTo(0.016875),
77 categoryCostUsd: {
78 input: expect.closeTo(0.003),
79 cacheRead: expect.closeTo(0.003),
80 cacheWrite: expect.closeTo(0.000375),
81 output: expect.closeTo(0.0075),
82 },
83 fastCostUsd: expect.closeTo(0.01125),
84 speedPremiumUsd: expect.closeTo(0.005625),
85 });
86 expect(bucket).not.toHaveProperty("ultrafastCostUsd");
87 });
88​
89 it("merges partial copies of a key by channel maximum in the first copy's bucket", () => {
90 const zeroTotals = {
91 uncachedInputTokens: 0,
92 cachedInputTokens: 0,
93 cacheCreationTokens: 0,
94 outputTokens: 0,
95 reasoningTokens: 0,
96 };
97 const partial = record({
98 dedupeKey: "msg_1:",
99 speed: "fast",
100 reportedCostUsd: 1,
101 totals: { ...record().totals, cachedInputTokens: 1200, outputTokens: 3, reasoningTokens: 5 },
102 });
103 const complete = record({
104 dedupeKey: "msg_1:",
105 model: "claude-fable-5[1m]",
106 timestampMs: Date.parse("2026-08-08T12:00:00Z"),
107 sessionId: "session-fork",
108 reportedCostUsd: 2,
109 totals: {
110 uncachedInputTokens: 2,
111 cachedInputTokens: 1000,
112 cacheCreationTokens: 40,
113 outputTokens: 264,
114 reasoningTokens: 135,
115 },
116 });
117 const zero = { ...partial, totals: zeroTotals, reportedCostUsd: null };
118 const merged = {
119 reportedCostUsd: 2,
120 totals: { ...complete.totals, uncachedInputTokens: 100, cachedInputTokens: 1200 },
121 };
122​
123 for (const copies of [
124 [zero, partial, complete, complete],
125 [complete, partial, zero, zero],
126 ]) {
127 const aggregator = new UsageAggregator({
128 timeZone: "UTC",
129 sinceDay: "2026-08-01",
130 untilDay: "2026-08-31",
131 rates,
132 });
133 copies.forEach((copy, index) => aggregator.add(copy, `${index}.jsonl`));
134 const result = aggregator.finish();
135 const expected = aggregate([{ ...copies[0]!, ...merged }]).buckets[0];
136​
137 expect(result.duplicatesDropped).toBe(3);
138 expect(result.buckets).toEqual([{ ...expected, sourcePath: "0.jsonl" }]);
139 }
140 });
141​
142 it("does not count a key whose first copy fell outside the window", () => {
143 const result = aggregate([
144 record({ dedupeKey: "msg_1:", timestampMs: Date.parse("2026-07-01T12:00:00Z") }),
145 record({ dedupeKey: "msg_1:", totals: { ...record().totals, outputTokens: 500 } }),
146 ]);
147​
148 expect(result.duplicatesDropped).toBe(1);
149 expect(result.outOfWindow).toBe(1);
150 expect(result.buckets).toHaveLength(0);
151 });
152​
153 it("buckets by the day in the requested time zone", () => {
154 const utc = aggregate([record()], "UTC");
155 const losAngeles = aggregate([record()], "America/Los_Angeles");
156​
157 expect(utc.buckets[0]?.day).toBe("2026-08-07");
158 expect(losAngeles.buckets[0]?.day).toBe("2026-08-06");
159 });
160​
161 it("finds the day at a quarter-hour zone's midnight across interleaved buckets", () => {
162 const result = aggregate(
163 [
164 record({ timestampMs: Date.parse("2026-08-01T18:14:59.999Z") }),
165 record({ timestampMs: Date.parse("2026-08-01T18:15:00.000Z") }),
166 record({ timestampMs: Date.parse("2026-08-01T18:15:00.000Z"), model: "claude-opus-5" }),
167 record({ timestampMs: Date.parse("2026-08-01T18:16:00.000Z") }),
168 ],
169 "Asia/Kathmandu",
170 );
171​
172 expect(result.buckets.map((bucket) => [bucket.day, bucket.model, bucket.records])).toEqual([
173 ["2026-08-01", "claude-fable-5", 1],
174 ["2026-08-02", "claude-fable-5", 2],
175 ["2026-08-02", "claude-opus-5", 1],
176 ]);
177 });
178​
179 it("finds the day at a fixed offset's midnight between quarter hours", () => {
180 const result = aggregate(
181 [
182 record({ timestampMs: Date.parse("2026-08-01T23:58:59.999Z") }),
183 record({ timestampMs: Date.parse("2026-08-01T23:59:00.000Z") }),
184 ],
185 "+00:01",
186 );
187​
188 expect(result.buckets.map((bucket) => [bucket.day, bucket.records])).toEqual([
189 ["2026-08-01", 1],
190 ["2026-08-02", 1],
191 ]);
192 });
193​
194 it("splits an hourly request into fixed buckets anchored to its exact start", () => {
195 const result = aggregate(
196 [
197 record({ timestampMs: Date.parse("2026-08-07T02:40:13.944Z") }),
198 record({ timestampMs: Date.parse("2026-08-07T03:40:13.944Z") }),
199 ],
200 "America/Los_Angeles",
201 "hour",
202 );
203​
204 expect(result.buckets.map((bucket) => [bucket.day, bucket.hourStart])).toEqual([
205 ["2026-08-06", "2026-08-07T02:37:00.000Z"],
206 ["2026-08-06", "2026-08-07T03:37:00.000Z"],
207 ]);
208 });
209​
210 it("uses an inclusive start and exclusive end for rolling windows", () => {
211 const result = aggregate(
212 [
213 record({ timestampMs: Date.parse("2026-08-06T04:36:59.999Z") }),
214 record({ timestampMs: Date.parse("2026-08-06T04:37:00.000Z") }),
215 record({ timestampMs: Date.parse("2026-08-07T04:36:59.999Z") }),
216 record({ timestampMs: Date.parse("2026-08-07T04:37:00.000Z") }),
217 ],
218 "UTC",
219 "hour",
220 );
221​
222 expect(result.outOfWindow).toBe(2);
223 expect(result.buckets.map((bucket) => bucket.hourStart)).toEqual([
224 "2026-08-06T04:37:00.000Z",
225 "2026-08-07T03:37:00.000Z",
226 ]);
227 });
228​
229 it("counts tokens but not cost for a model with no rate", () => {
230 const result = aggregate([record({ model: "kimi-k3" })]);
231​
232 expect(result.buckets[0]?.costUsd).toBe(0);
233 expect(result.buckets[0]?.costSource).toBe("unpriced");
234 expect(result.buckets[0]?.unpricedRecords).toBe(1);
235 expect(result.buckets[0]?.totals.outputTokens).toBe(50);
236 });
237​
238 it("prefers a reported cost over the rate table", () => {
239 const result = aggregate([record({ reportedCostUsd: 1.25 })]);
240​
241 expect(result.buckets[0]?.costUsd).toBe(1.25);
242 expect(result.buckets[0]?.costSource).toBe("providerReported");
243 });
244​
245 it("folds a mapped model into its target and prices it there", () => {
246 const aggregator = new UsageAggregator({
247 timeZone: "UTC",
248 sinceDay: "2026-08-01",
249 untilDay: "2026-08-31",
250 rates,
251 modelAliases: resolveModelAliases({ "example-preview": "claude-fable-5" }),
252 });
253 aggregator.add(record());
254 aggregator.add(record({ model: "example-preview", rateModel: "example-preview-high" }));
255 const result = aggregator.finish();
256​
257 expect(result.buckets).toHaveLength(1);
258 expect(result.buckets[0]?.model).toBe("claude-fable-5");
259 expect(result.buckets[0]?.records).toBe(2);
260 expect(result.buckets[0]?.costUsd).toBeCloseTo(0.00925, 9);
261 expect(result.buckets[0]?.unpricedRecords).toBe(0);
262 });
263});
264​
265describe("resolveModelAliases", () => {
266 it("follows chains to the final model and drops chains that enter a loop", () => {
267 expect(
268 resolveModelAliases({
269 "preview[1m]": "preview",
270 preview: "example-model",
271 loop: "back",
272 back: "loop",
273 intoLoop: "loop",
274 self: "self",
275 }),
276 ).toEqual(
277 new Map([
278 ["preview[1m]", "example-model"],
279 ["preview", "example-model"],
280 ]),
281 );
282 });
283});