Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

usageMerge.ts

589 lines19.4 KB

1import {
2 USAGE_MERGE_COMPATIBLE_SINCE,
3 type EnvironmentId,
4 type UsageBucket,
5 type UsageProviderKind,
6 type UsageSource,
7 type UsageSourceFingerprint,
8 type UsageSummary,
9 type UsageTokenTotals,
10} from "@t3tools/contracts";
11​
12export interface EnvironmentUsage {
13 readonly environmentId: EnvironmentId;
14 readonly label: string;
15 readonly summary: UsageSummary;
17​
18export interface ProviderTotals {
19 readonly provider: UsageProviderKind;
20 readonly costUsd: number;
21 readonly totalTokens: number;
22 readonly records: number;
23 readonly sessions: number;
24 readonly costShare: number;
25 readonly tokenShare: number;
27​
28export interface ModelTotals {
29 readonly model: string;
30 readonly provider: UsageProviderKind;
31 readonly costUsd: number;
32 readonly totalTokens: number;
33 readonly tokens: UsageTokenTotals;
34 readonly records: number;
35 readonly unpricedRecords: number;
36 readonly unpricedTokens: number;
37 readonly costShare: number;
38 readonly tokenShare: number;
40​
41/** When every record lacks rates, costUsd is unknown and must not be displayed as a dollar figure. */
42export function isModelCostUnknown(model: ModelTotals): boolean {
43 return model.records > 0 && model.unpricedRecords >= model.records;
45​
46export interface DailyTotals {
47 readonly day: string;
48 readonly costUsd: number;
49 readonly totalTokens: number;
50 readonly byProvider: ReadonlyMap<UsageProviderKind, { costUsd: number; totalTokens: number }>;
52​
53export interface HourlyTotals {
54 readonly day: string;
55 readonly hourStart: string;
56 readonly costUsd: number;
57 readonly totalTokens: number;
58 readonly byProvider: ReadonlyMap<UsageProviderKind, { costUsd: number; totalTokens: number }>;
60​
61export interface CostQuality {
62 readonly providerReportedShare: number;
63 readonly modelPricedShare: number;
64 readonly unpricedShare: number;
65 readonly cacheSavingsUsd: number;
67​
68export interface CategoryCost {
69 readonly input: number;
70 readonly cacheRead: number;
71 readonly cacheWrite: number;
72 readonly output: number;
73 readonly unsplit: number;
75​
76export interface SpeedCost {
77 readonly standard: number;
78 readonly fast: number;
79 readonly ultrafast: number;
80 readonly premium: number;
82​
83export interface UsageContractMismatch {
84 readonly environmentId: EnvironmentId;
85 readonly direction: "serverBehind" | "clientBehind";
86 readonly contractVersion: number;
88​
89export interface MergedUsage {
90 readonly costUsd: number;
91 readonly uncachedInputTokens: number;
92 readonly cachedInputTokens: number;
93 readonly cacheCreationTokens: number;
94 readonly outputTokens: number;
95 readonly reasoningTokens: number;
96 readonly totalTokens: number;
97 readonly records: number;
98 readonly sessions: number;
99 readonly providers: readonly ProviderTotals[];
100 readonly models: readonly ModelTotals[];
101 readonly daily: readonly DailyTotals[];
102 readonly hourly: readonly HourlyTotals[];
103 readonly costQuality: CostQuality;
104 readonly categoryCost: CategoryCost;
105 readonly speedCost: SpeedCost;
106 readonly duplicateSources: readonly string[];
107 readonly contributingEnvironments: readonly EnvironmentId[];
108 readonly contractMismatches: readonly UsageContractMismatch[];
110​
111// volumeId distinguishes machines sharing a hostname and home path.
112function fingerprintKey(fingerprint: UsageSourceFingerprint): string {
113 return [
114 fingerprint.hostId,
115 fingerprint.provider,
116 fingerprint.resolvedHomePath,
117 fingerprint.volumeId,
118 ].join(" ");
120​
121function bucketsForSource(summary: UsageSummary, source: UsageSource): readonly UsageBucket[] {
122 const providerSources = summary.sources.filter(
123 (entry) => entry.fingerprint.provider === source.fingerprint.provider,
124 );
125 return summary.buckets.filter(
126 (bucket) =>
127 bucket.provider === source.fingerprint.provider &&
128 (bucket.sourcePath === source.fingerprint.resolvedHomePath ||
129 (bucket.sourcePath === undefined && providerSources.length === 1)),
130 );
132​
133function bucketKey(bucket: UsageBucket): string {
134 return JSON.stringify([bucket.day, bucket.hourStart ?? null, bucket.provider, bucket.model]);
136​
137function claimSources(environments: readonly EnvironmentUsage[]): {
138 readonly ownerByFingerprint: ReadonlyMap<string, EnvironmentId>;
139 readonly supplementalBucketsByEnvironment: ReadonlyMap<EnvironmentId, ReadonlySet<UsageBucket>>;
140 readonly sessionsByFingerprint: ReadonlyMap<string, number>;
141 readonly duplicates: readonly string[];
142} {
143 const ownerByFingerprint = new Map<string, EnvironmentId>();
144 const ownerScanByFingerprint = new Map<
145 string,
146 { environment: EnvironmentUsage; source: UsageSource }
147 >();
148 const seenBucketKeysByFingerprint = new Map<string, Set<string>>();
149 const supplementalBucketsByEnvironment = new Map<EnvironmentId, Set<UsageBucket>>();
150 const sessionsByFingerprint = new Map<string, number>();
151 const duplicates: string[] = [];
152​
153 const ordered = [...environments].sort(
154 (a, b) =>
155 (Date.parse(b.summary.readAt) || 0) - (Date.parse(a.summary.readAt) || 0) ||
156 a.environmentId.localeCompare(b.environmentId),
157 );
158​
159 for (const status of ["ok", "partial", "failed"] as const) {
160 for (const environment of ordered) {
161 for (const source of environment.summary.sources) {
162 if (source.status !== status) continue;
163 const key = fingerprintKey(source.fingerprint);
164 if (ownerByFingerprint.has(key)) {
165 duplicates.push(`${environment.label}: ${source.fingerprint.resolvedHomePath}`);
166 continue;
167 }
168 ownerByFingerprint.set(key, environment.environmentId);
169 ownerScanByFingerprint.set(key, { environment, source });
170 sessionsByFingerprint.set(key, source.distinctSessions);
171 }
172 }
173 }
174​
175 // Keep cells unique to newer partial scans; aggregates cannot reconcile overlapping records without double-counting.
176 for (const environment of ordered) {
177 for (const source of environment.summary.sources) {
178 if (source.status !== "partial") continue;
179 const key = fingerprintKey(source.fingerprint);
180 const owner = ownerScanByFingerprint.get(key);
181 // A scan under another model filter counts different records, so its cells are not missing ones.
182 if (
183 owner?.source.status !== "ok" ||
184 Date.parse(environment.summary.readAt) <= Date.parse(owner.environment.summary.readAt) ||
185 JSON.stringify(environment.summary.modelFilter) !==
186 JSON.stringify(owner.environment.summary.modelFilter)
187 ) {
188 continue;
189 }
190 let seen = seenBucketKeysByFingerprint.get(key);
191 if (seen === undefined) {
192 seen = new Set(bucketsForSource(owner.environment.summary, owner.source).map(bucketKey));
193 seenBucketKeysByFingerprint.set(key, seen);
194 }
195 const supplemental =
196 supplementalBucketsByEnvironment.get(environment.environmentId) ?? new Set<UsageBucket>();
197 let added = false;
198 for (const bucket of bucketsForSource(environment.summary, source)) {
199 const cell = bucketKey(bucket);
200 if (seen.has(cell)) continue;
201 seen.add(cell);
202 supplemental.add(bucket);
203 added = true;
204 }
205 if (!added) continue;
206 supplementalBucketsByEnvironment.set(environment.environmentId, supplemental);
207 sessionsByFingerprint.set(
208 key,
209 Math.max(sessionsByFingerprint.get(key) ?? 0, source.distinctSessions),
210 );
211 }
212 }
213​
214 return {
215 ownerByFingerprint,
216 supplementalBucketsByEnvironment,
217 sessionsByFingerprint,
218 duplicates,
219 };
221​
222function ownedContribution(
223 environment: EnvironmentUsage,
224 ownerByFingerprint: ReadonlyMap<string, EnvironmentId>,
225 supplementalBuckets: ReadonlySet<UsageBucket>,
226 sessionsByFingerprint: ReadonlyMap<string, number>,
227): {
228 readonly buckets: readonly UsageBucket[];
229 readonly sessionsByProvider: ReadonlyMap<UsageProviderKind, number>;
230} {
231 const ownedProviders = new Set<UsageProviderKind>();
232 const ownedSources = new Set<string>();
233 const sessionsByProvider = new Map<UsageProviderKind, number>();
234 for (const source of environment.summary.sources) {
235 if (source.status === "missing") continue;
236 const key = fingerprintKey(source.fingerprint);
237 if (ownerByFingerprint.get(key) === environment.environmentId) {
238 const provider = source.fingerprint.provider;
239 ownedProviders.add(provider);
240 ownedSources.add(`${provider}\u0000${source.fingerprint.resolvedHomePath}`);
241 // Count distinct sessions per directory; bucket sums overcount sessions spanning days/models.
242 sessionsByProvider.set(
243 provider,
244 (sessionsByProvider.get(provider) ?? 0) +
245 (sessionsByFingerprint.get(key) ?? source.distinctSessions),
246 );
247 }
248 }
249 return {
250 buckets: environment.summary.buckets.filter(
251 (bucket) =>
252 supplementalBuckets.has(bucket) ||
253 (bucket.sourcePath === undefined
254 ? ownedProviders.has(bucket.provider)
255 : ownedSources.has(`${bucket.provider}\u0000${bucket.sourcePath}`)),
256 ),
257 sessionsByProvider,
258 };
260​
261function bucketTokens(bucket: UsageBucket): number {
262 // reasoningTokens is a subset of outputTokens and must not be added again.
263 return (
264 bucket.totals.uncachedInputTokens +
265 bucket.totals.cachedInputTokens +
266 bucket.totals.cacheCreationTokens +
267 bucket.totals.outputTokens
268 );
270​
271export function isCompatibleUsageContractVersion(version: number, expected: number): boolean {
272 return version >= USAGE_MERGE_COMPATIBLE_SINCE && version <= expected;
274​
275const EMPTY_MERGED: MergedUsage = {
276 costUsd: 0,
277 uncachedInputTokens: 0,
278 cachedInputTokens: 0,
279 cacheCreationTokens: 0,
280 outputTokens: 0,
281 reasoningTokens: 0,
282 totalTokens: 0,
283 records: 0,
284 sessions: 0,
285 providers: [],
286 models: [],
287 daily: [],
288 hourly: [],
289 costQuality: {
290 providerReportedShare: 0,
291 modelPricedShare: 0,
292 unpricedShare: 0,
293 cacheSavingsUsd: 0,
294 },
295 categoryCost: { input: 0, cacheRead: 0, cacheWrite: 0, output: 0, unsplit: 0 },
296 speedCost: { standard: 0, fast: 0, ultrafast: 0, premium: 0 },
297 duplicateSources: [],
298 contributingEnvironments: [],
299 contractMismatches: [],
300};
301​
302export function mergeUsage(
303 environments: readonly EnvironmentUsage[],
304 expectedContractVersion: number,
305): MergedUsage {
306 if (environments.length === 0) return EMPTY_MERGED;
307​
308 const current: EnvironmentUsage[] = [];
309 const contractMismatches: UsageContractMismatch[] = [];
310 for (const environment of environments) {
311 if (
312 isCompatibleUsageContractVersion(environment.summary.contractVersion, expectedContractVersion)
313 ) {
314 current.push(environment);
315 } else {
316 contractMismatches.push({
317 environmentId: environment.environmentId,
318 direction:
319 environment.summary.contractVersion < expectedContractVersion
320 ? "serverBehind"
321 : "clientBehind",
322 contractVersion: environment.summary.contractVersion,
323 });
324 }
325 }
326​
327 const {
328 ownerByFingerprint,
329 supplementalBucketsByEnvironment,
330 sessionsByFingerprint,
331 duplicates,
332 } = claimSources(current);
333​
334 let costUsd = 0;
335 let uncachedInputTokens = 0;
336 let cachedInputTokens = 0;
337 let cacheCreationTokens = 0;
338 let outputTokens = 0;
339 let reasoningTokens = 0;
340 let records = 0;
341 let sessions = 0;
342 let cacheSavingsUsd = 0;
343 let providerReportedRecords = 0;
344 let unpricedRecords = 0;
345 const categoryCost = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
346 const speedCost = { fast: 0, ultrafast: 0, premium: 0 };
347​
348 const providerAccumulator = new Map<
349 UsageProviderKind,
350 { costUsd: number; totalTokens: number; records: number; sessions: number }
351 >();
352 const modelAccumulator = new Map<
353 string,
354 {
355 provider: UsageProviderKind;
356 costUsd: number;
357 totalTokens: number;
358 tokens: UsageTokenTotals;
359 records: number;
360 unpricedRecords: number;
361 unpricedTokens: number;
362 }
363 >();
364 const dailyAccumulator = new Map<
365 string,
366 {
367 costUsd: number;
368 totalTokens: number;
369 byProvider: Map<UsageProviderKind, { costUsd: number; totalTokens: number }>;
370 }
371 >();
372 const hourlyAccumulator = new Map<
373 string,
374 {
375 day: string;
376 hourStart: string;
377 costUsd: number;
378 totalTokens: number;
379 byProvider: Map<UsageProviderKind, { costUsd: number; totalTokens: number }>;
380 }
381 >();
382 const contributingEnvironments: EnvironmentId[] = [];
383​
384 for (const environment of current) {
385 const { buckets, sessionsByProvider } = ownedContribution(
386 environment,
387 ownerByFingerprint,
388 supplementalBucketsByEnvironment.get(environment.environmentId) ?? new Set(),
389 sessionsByFingerprint,
390 );
391 if (buckets.length > 0) contributingEnvironments.push(environment.environmentId);
392​
393 for (const [providerKind, providerSessions] of sessionsByProvider) {
394 sessions += providerSessions;
395 if (providerSessions === 0) continue;
396 const provider = providerAccumulator.get(providerKind) ?? {
397 costUsd: 0,
398 totalTokens: 0,
399 records: 0,
400 sessions: 0,
401 };
402 provider.sessions += providerSessions;
403 providerAccumulator.set(providerKind, provider);
404 }
405​
406 for (const bucket of buckets) {
407 const tokens = bucketTokens(bucket);
408​
409 costUsd += bucket.costUsd;
410 cacheSavingsUsd += bucket.cacheSavingsUsd;
411 uncachedInputTokens += bucket.totals.uncachedInputTokens;
412 cachedInputTokens += bucket.totals.cachedInputTokens;
413 cacheCreationTokens += bucket.totals.cacheCreationTokens;
414 outputTokens += bucket.totals.outputTokens;
415 reasoningTokens += bucket.totals.reasoningTokens;
416 records += bucket.records;
417 unpricedRecords += bucket.unpricedRecords;
418 if (bucket.costSource === "providerReported") providerReportedRecords += bucket.records;
419 if (bucket.categoryCostUsd !== undefined) {
420 categoryCost.input += bucket.categoryCostUsd.input;
421 categoryCost.cacheRead += bucket.categoryCostUsd.cacheRead;
422 categoryCost.cacheWrite += bucket.categoryCostUsd.cacheWrite;
423 categoryCost.output += bucket.categoryCostUsd.output;
424 }
425 speedCost.fast += bucket.fastCostUsd ?? 0;
426 speedCost.ultrafast += bucket.ultrafastCostUsd ?? 0;
427 speedCost.premium += bucket.speedPremiumUsd ?? 0;
428​
429 const provider = providerAccumulator.get(bucket.provider) ?? {
430 costUsd: 0,
431 totalTokens: 0,
432 records: 0,
433 sessions: 0,
434 };
435 provider.costUsd += bucket.costUsd;
436 provider.totalTokens += tokens;
437 provider.records += bucket.records;
438 providerAccumulator.set(bucket.provider, provider);
439​
440 const modelKey = `${bucket.provider} ${bucket.model}`;
441 const model = modelAccumulator.get(modelKey) ?? {
442 provider: bucket.provider,
443 costUsd: 0,
444 totalTokens: 0,
445 tokens: {
446 uncachedInputTokens: 0,
447 cachedInputTokens: 0,
448 cacheCreationTokens: 0,
449 outputTokens: 0,
450 reasoningTokens: 0,
451 },
452 records: 0,
453 unpricedRecords: 0,
454 unpricedTokens: 0,
455 };
456 model.costUsd += bucket.costUsd;
457 model.totalTokens += tokens;
458 model.tokens = {
459 uncachedInputTokens: model.tokens.uncachedInputTokens + bucket.totals.uncachedInputTokens,
460 cachedInputTokens: model.tokens.cachedInputTokens + bucket.totals.cachedInputTokens,
461 cacheCreationTokens: model.tokens.cacheCreationTokens + bucket.totals.cacheCreationTokens,
462 outputTokens: model.tokens.outputTokens + bucket.totals.outputTokens,
463 reasoningTokens: model.tokens.reasoningTokens + bucket.totals.reasoningTokens,
464 };
465 model.records += bucket.records;
466 model.unpricedRecords += bucket.unpricedRecords;
467 if (bucket.records > 0) {
468 model.unpricedTokens += (tokens * bucket.unpricedRecords) / bucket.records;
469 }
470 modelAccumulator.set(modelKey, model);
471​
472 const day = dailyAccumulator.get(bucket.day) ?? {
473 costUsd: 0,
474 totalTokens: 0,
475 byProvider: new Map<UsageProviderKind, { costUsd: number; totalTokens: number }>(),
476 };
477 day.costUsd += bucket.costUsd;
478 day.totalTokens += tokens;
479 const dayProvider = day.byProvider.get(bucket.provider) ?? { costUsd: 0, totalTokens: 0 };
480 dayProvider.costUsd += bucket.costUsd;
481 dayProvider.totalTokens += tokens;
482 day.byProvider.set(bucket.provider, dayProvider);
483 dailyAccumulator.set(bucket.day, day);
484​
485 if (bucket.hourStart !== undefined) {
486 const hour = hourlyAccumulator.get(bucket.hourStart) ?? {
487 day: bucket.day,
488 hourStart: bucket.hourStart,
489 costUsd: 0,
490 totalTokens: 0,
491 byProvider: new Map<UsageProviderKind, { costUsd: number; totalTokens: number }>(),
492 };
493 hour.costUsd += bucket.costUsd;
494 hour.totalTokens += tokens;
495 const hourProvider = hour.byProvider.get(bucket.provider) ?? {
496 costUsd: 0,
497 totalTokens: 0,
498 };
499 hourProvider.costUsd += bucket.costUsd;
500 hourProvider.totalTokens += tokens;
501 hour.byProvider.set(bucket.provider, hourProvider);
502 hourlyAccumulator.set(bucket.hourStart, hour);
503 }
504 }
505 }
506​
507 const totalTokens = uncachedInputTokens + cachedInputTokens + cacheCreationTokens + outputTokens;
508​
509 const providers: ProviderTotals[] = [...providerAccumulator.entries()]
510 .map(([provider, totals]) => ({
511 provider,
512 costUsd: totals.costUsd,
513 totalTokens: totals.totalTokens,
514 records: totals.records,
515 sessions: totals.sessions,
516 costShare: costUsd === 0 ? 0 : totals.costUsd / costUsd,
517 tokenShare: totalTokens === 0 ? 0 : totals.totalTokens / totalTokens,
518 }))
519 .sort((a, b) => b.costUsd - a.costUsd);
520​
521 const models: ModelTotals[] = [...modelAccumulator.entries()]
522 .map(([key, totals]) => ({
523 model: key.slice(key.indexOf(" ") + 1),
524 provider: totals.provider,
525 costUsd: totals.costUsd,
526 totalTokens: totals.totalTokens,
527 tokens: totals.tokens,
528 records: totals.records,
529 unpricedRecords: totals.unpricedRecords,
530 unpricedTokens: totals.unpricedTokens,
531 costShare: costUsd === 0 ? 0 : totals.costUsd / costUsd,
532 tokenShare: totalTokens === 0 ? 0 : totals.totalTokens / totalTokens,
533 }))
534 .sort((a, b) => b.costUsd - a.costUsd || b.totalTokens - a.totalTokens);
535​
536 const daily: DailyTotals[] = [...dailyAccumulator.entries()]
537 .map(([day, totals]) => ({
538 day,
539 costUsd: totals.costUsd,
540 totalTokens: totals.totalTokens,
541 byProvider: totals.byProvider,
542 }))
543 .sort((a, b) => a.day.localeCompare(b.day));
544​
545 const hourly: HourlyTotals[] = [...hourlyAccumulator.values()].sort((a, b) =>
546 a.hourStart.localeCompare(b.hourStart),
547 );
548​
549 return {
550 costUsd,
551 uncachedInputTokens,
552 cachedInputTokens,
553 cacheCreationTokens,
554 outputTokens,
555 reasoningTokens,
556 totalTokens,
557 records,
558 sessions,
559 providers,
560 models,
561 daily,
562 hourly,
563 costQuality: {
564 providerReportedShare: records === 0 ? 0 : providerReportedRecords / records,
565 unpricedShare: records === 0 ? 0 : unpricedRecords / records,
566 modelPricedShare:
567 records === 0 ? 0 : (records - providerReportedRecords - unpricedRecords) / records,
568 cacheSavingsUsd,
569 },
570 categoryCost: {
571 ...categoryCost,
572 unsplit: Math.max(
573 0,
574 costUsd -
575 categoryCost.input -
576 categoryCost.cacheRead -
577 categoryCost.cacheWrite -
578 categoryCost.output,
579 ),
580 },
581 speedCost: {
582 ...speedCost,
583 standard: Math.max(0, costUsd - speedCost.fast - speedCost.ultrafast),
584 },
585 duplicateSources: duplicates,
586 contributingEnvironments,
587 contractMismatches,
588 };