1/**
2 * Provider token reporting: one `provider.turn.completed` per provider turn and one
3 * `provider.request.completed` per run, built from the same turns so each counts once.
4 *
5 * @module ProviderUsageAnalytics
6 */
7import type {
8 ModelSelection,
9 OrchestrationV2Actor,
10 OrchestrationV2CreationSource,
11 OrchestrationV2ProviderTurn,
12 OrchestrationV2RunAttempt,
13 ProviderInteractionMode,
14 RunId,
15 RuntimeMode,
16 TurnTokenUsage,
17} from "@t3tools/contracts";
18import * as Context from "effect/Context";
19import * as DateTime from "effect/DateTime";
20import * as Effect from "effect/Effect";
21import * as Layer from "effect/Layer";
22
23import {
24 getModelSelectionBooleanOptionValue,
25 getModelSelectionStringOptionValue,
26} from "@t3tools/shared/model";
27import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
28import * as AnalyticsService from "./AnalyticsService.ts";
29import { catalogModelLabel } from "./ProviderDimensions.ts";
30
31// Bump when a field changes meaning, so reports can split old and new rows.
32const REPORT_VERSION = 2;
33
34/**
35 * Where the run's starting message came from. Most turns start without a
36 * person (provider continuations, agent sends, delegated subagents), so
37 * reports need this to tell user turns apart.
38 */
39export interface ProviderTurnOrigin {
40 readonly createdBy: OrchestrationV2Actor;
41 readonly creationSource: OrchestrationV2CreationSource;
42 readonly scheduled: boolean;
43 readonly subagent: boolean;
44}
45
46/** What started this provider turn inside its run. */
47export type ProviderTurnSegment =
48 | OrchestrationV2RunAttempt["reason"]
49 | "continuation"
50 | "native_child";
51
52export interface ProviderTurnAnalyticsContext {
53 readonly runId?: RunId;
54 readonly modelSelection: ModelSelection;
55 readonly runtimeMode?: RuntimeMode;
56 readonly interactionMode?: ProviderInteractionMode;
57 readonly origin?: ProviderTurnOrigin;
58 readonly segment?: ProviderTurnSegment;
59 readonly restartContinuation?: boolean;
60}
61
62export interface ProviderTurnReport {
63 readonly driver: string;
64 readonly providerTurn: OrchestrationV2ProviderTurn;
65 readonly context?: ProviderTurnAnalyticsContext;
66}
67
68export interface RunSettledReport {
69 readonly runId: RunId;
70 readonly status: OrchestrationV2ProviderTurn["status"];
71 readonly completedAt: DateTime.Utc;
72 // Run-owned children were still open when the run settled; their later turns are not added.
73 readonly childrenRunning: boolean;
74}
75
76type Properties = Readonly<Record<string, unknown>>;
77/** Maps a native model id to a vendor catalog label, or undefined to keep it out. */
78export type ModelLabeler = (model: string) => string | undefined;
79
80const TERMINAL_STATUSES: ReadonlySet<OrchestrationV2ProviderTurn["status"]> = new Set([
81 "completed",
82 "failed",
83 "interrupted",
84 "cancelled",
85]);
86
87const REPORTED_EFFORTS: ReadonlySet<string> = new Set([
88 "none",
89 "minimal",
90 "low",
91 "medium",
92 "high",
93 "xhigh",
94 "max",
95 "ultrathink",
96 "ultracode",
97 "brutal",
98]);
99
100const defined = (entries: Properties): Properties =>
101 Object.fromEntries(Object.entries(entries).filter(([, value]) => value !== undefined));
102
103// OpenCode names its effort option `variant`, and variants can be user-defined.
104function selectedEffort(selection: ModelSelection): string | undefined {
105 const effort =
106 getModelSelectionStringOptionValue(selection, "reasoningEffort") ??
107 getModelSelectionStringOptionValue(selection, "effort") ??
108 getModelSelectionStringOptionValue(selection, "variant");
109 return effort !== undefined && REPORTED_EFFORTS.has(effort) ? effort : undefined;
110}
111
112// Codex sends a tier id (its "fast" config shows up as "priority"); Claude has a fastMode toggle.
113const REPORTED_SERVICE_TIERS: ReadonlyMap<string, string> = new Map([
114 ["default", "standard"],
115 ["flex", "flex"],
116 ["fast", "fast"],
117 ["priority", "fast"],
118 ["ultrafast", "ultrafast"],
119]);
120
121function selectedServiceTier(selection: ModelSelection): string | undefined {
122 const tier = getModelSelectionStringOptionValue(selection, "serviceTier");
123 if (tier !== undefined) return REPORTED_SERVICE_TIERS.get(tier);
124 const fastMode = getModelSelectionBooleanOptionValue(selection, "fastMode");
125 return fastMode === undefined ? undefined : fastMode ? "fast" : "standard";
126}
127
128/**
129 * Confirmed models across counted work. Unknown when any piece of work has no
130 * reported model, unless two distinct models were already seen.
131 */
132function modelObservation(
133 models: ReadonlyArray<ReadonlyArray<string> | undefined>,
134 label: ModelLabeler,
135): Properties {
136 const distinct = new Set(models.flatMap((entry) => entry ?? []));
137 if (distinct.size > 1) return { mixedModels: true };
138 if (distinct.size === 0 || models.some((entry) => entry === undefined || entry.length === 0)) {
139 return {};
140 }
141 return defined({ mixedModels: false, observedModel: label([...distinct][0]!) });
142}
143
144function selectionProperties(
145 context: ProviderTurnAnalyticsContext | undefined,
146 label: ModelLabeler,
147) {
148 const selection = context?.modelSelection;
149 return defined({
150 model: selection === undefined ? undefined : label(selection.model),
151 effort: selection === undefined ? undefined : selectedEffort(selection),
152 serviceTier: selection === undefined ? undefined : selectedServiceTier(selection),
153 runtimeMode: context?.runtimeMode,
154 interactionMode: context?.interactionMode,
155 ...context?.origin,
156 });
157}
158
159function childKind(context: ProviderTurnAnalyticsContext | undefined) {
160 if (context?.segment === "native_child") return "native";
161 return context?.origin?.subagent === true ? "t3" : undefined;
162}
163
164function counters(usage: TurnTokenUsage | undefined): Properties {
165 return defined({
166 inputTokens: usage?.inputTokens,
167 cachedInputTokens: usage?.cachedInputTokens,
168 cacheCreationTokens: usage?.cacheCreationTokens,
169 outputTokens: usage?.outputTokens,
170 reasoningTokens: usage?.reasoningTokens,
171 });
172}
173
174export function providerTurnProperties(
175 report: ProviderTurnReport,
176 label: ModelLabeler,
177): Properties {
178 const { providerTurn, context } = report;
179 const usage = providerTurn.turnTokenUsage;
180 return defined({
181 reportVersion: REPORT_VERSION,
182 provider: report.driver,
183 terminalStatus: providerTurn.status,
184 usageStatus: usage?.usageStatus ?? "unavailable",
185 usageScope: usage?.usageScope ?? "main_agent",
186 usageReported: usage !== undefined,
187 hasSubagents: usage?.hasSubagents,
188 ...counters(usage),
189 localCommand: usage?.localCommand === true ? true : undefined,
190 otherInputTokens: usage?.otherUsage?.inputTokens,
191 otherOutputTokens: usage?.otherUsage?.outputTokens,
192 ...selectionProperties(context, label),
193 ...modelObservation([usage?.models], label),
194 segment: context?.segment,
195 childKind: childKind(context),
196 durationMs:
197 providerTurn.startedAt && providerTurn.completedAt
198 ? Math.max(
199 0,
200 DateTime.toEpochMillis(providerTurn.completedAt) -
201 DateTime.toEpochMillis(providerTurn.startedAt),
202 )
203 : undefined,
204 });
205}
206
207/** How the request reached the provider; only `user` counts as a person asking. */
208function requestKind(origin: ProviderTurnOrigin | undefined) {
209 if (origin === undefined) return "unknown";
210 if (origin.scheduled) return "scheduled";
211 if (origin.subagent) return "t3_child";
212 if (origin.createdBy === "user") return "user";
213 if (origin.creationSource === "provider") return "provider_continuation";
214 return origin.createdBy;
215}
216
217function sumOf(
218 usages: ReadonlyArray<TurnTokenUsage>,
219 field:
220 | "inputTokens"
221 | "cachedInputTokens"
222 | "cacheCreationTokens"
223 | "outputTokens"
224 | "reasoningTokens",
225 requireAll: boolean,
226): number | undefined {
227 const values = usages.map((usage) => usage[field]);
228 const known = values.filter((value) => value !== undefined);
229 if (known.length === 0 || (requireAll && known.length < values.length)) return undefined;
230 return known.reduce((sum, value) => sum + value, 0);
231}
232
233/** complete only when every turn is complete; partial when any turn has counts. */
234function combinedStatus(usages: ReadonlyArray<TurnTokenUsage | undefined>) {
235 if (usages.length > 0 && usages.every((usage) => usage?.usageStatus === "complete")) {
236 return "complete";
237 }
238 return usages.some(
239 (usage) => usage?.inputTokens !== undefined || usage?.outputTokens !== undefined,
240 )
241 ? "partial"
242 : "unavailable";
243}
244
245interface RequestTurn {
246 readonly child: boolean;
247 readonly segment: ProviderTurnSegment | undefined;
248 readonly providerTurn: OrchestrationV2ProviderTurn;
249}
250
251interface RequestEntry {
252 driver: string | undefined;
253 context: ProviderTurnAnalyticsContext | undefined;
254 readonly lifecycleObserved: boolean;
255 readonly turns: Map<string, RequestTurn>;
256}
257
258const MAX_TRACKED_RUNS = 2048;
259
260/**
261 * Collects terminal provider turns per run in this process. A replayed turn
262 * replaces its earlier copy, a settled run ignores late turns, and a run whose
263 * initial attempt started before this process reports `lifecycleObserved: false`.
264 */
265export class RequestUsageLedger {
266 private readonly open = new Map<RunId, RequestEntry>();
267 private readonly settled = new Set<RunId>();
268
269 start(runId: RunId, driver: string, context: ProviderTurnAnalyticsContext): void {
270 if (this.settled.has(runId) || this.open.has(runId)) return;
271 // A restart continuation carries on a run whose earlier turns this process never saw.
272 const lifecycleObserved = context.restartContinuation !== true;
273 this.open.set(runId, { driver, context, lifecycleObserved, turns: new Map() });
274 evictOldest(this.open);
275 }
276
277 add(report: ProviderTurnReport): void {
278 const runId = report.context?.runId;
279 if (runId === undefined || this.settled.has(runId)) return;
280 if (!TERMINAL_STATUSES.has(report.providerTurn.status)) return;
281 let entry = this.open.get(runId);
282 if (entry === undefined) {
283 entry = { driver: undefined, context: undefined, lifecycleObserved: false, turns: new Map() };
284 this.open.set(runId, entry);
285 evictOldest(this.open);
286 }
287 const child = report.context?.segment === "native_child";
288 if (!child) {
289 entry.driver ??= report.driver;
290 entry.context ??= report.context;
291 }
292 entry.turns.set(String(report.providerTurn.id), {
293 child,
294 segment: report.context?.segment,
295 providerTurn: report.providerTurn,
296 });
297 }
298
299 settle(report: RunSettledReport, label: ModelLabeler): Properties | undefined {
300 if (this.settled.has(report.runId)) return undefined;
301 this.settled.add(report.runId);
302 evictOldest(this.settled);
303 const entry = this.open.get(report.runId);
304 this.open.delete(report.runId);
305 if (entry === undefined) return undefined;
306
307 const turns = [...entry.turns.values()];
308 const root = turns.filter((turn) => !turn.child);
309 const children = turns.filter((turn) => turn.child);
310 const rootUsage = root.map((turn) => turn.providerTurn.turnTokenUsage);
311 const childUsage = children.map((turn) => turn.providerTurn.turnTokenUsage);
312 const { lifecycleObserved } = entry;
313 const measuredRoot = rootUsage.filter((usage) => usage !== undefined);
314 const status = combinedStatus(rootUsage);
315 const rootComplete = status === "complete";
316 const otherUsage = measuredRoot.map((usage) => usage.otherUsage);
317 const allOtherKnown = otherUsage.length > 0 && otherUsage.every((usage) => usage !== undefined);
318 const anyRootSubagents = measuredRoot.some((usage) => usage.hasSubagents);
319 const measuredChildren = childUsage.filter((usage) => usage !== undefined);
320 const childStatus =
321 children.length === 0 && !anyRootSubagents && !report.childrenRunning
322 ? "none"
323 : children.length === 0
324 ? "unavailable"
325 : report.childrenRunning && combinedStatus(childUsage) === "complete"
326 ? "partial"
327 : combinedStatus(childUsage);
328 const startedAt = turns
329 .map((turn) => turn.providerTurn.startedAt)
330 .filter((value) => value !== null)
331 .map(DateTime.toEpochMillis);
332
333 return defined({
334 reportVersion: REPORT_VERSION,
335 provider: entry.driver ?? "unknown",
336 requestKind: requestKind(entry.context?.origin),
337 restartContinuation: entry.context?.restartContinuation === true ? true : undefined,
338 terminalStatus: report.status,
339 lifecycleObserved,
340 ...selectionProperties(entry.context, label),
341 providerTurns: root.length,
342 continuations: root.filter((turn) => turn.segment === "continuation").length,
343 steeringRestarts: root.filter((turn) => turn.segment === "steering_restart").length,
344 usageStatus: lifecycleObserved || status === "unavailable" ? status : "partial",
345 inputTokens: sumOf(measuredRoot, "inputTokens", rootComplete),
346 cachedInputTokens: sumOf(measuredRoot, "cachedInputTokens", true),
347 cacheCreationTokens: sumOf(measuredRoot, "cacheCreationTokens", true),
348 outputTokens: sumOf(measuredRoot, "outputTokens", rootComplete),
349 reasoningTokens: sumOf(measuredRoot, "reasoningTokens", true),
350 localCommand: measuredRoot.some((usage) => usage.localCommand === true) ? true : undefined,
351 otherInputTokens: allOtherKnown
352 ? otherUsage.reduce((sum, usage) => sum + (usage?.inputTokens ?? 0), 0)
353 : undefined,
354 otherOutputTokens: allOtherKnown
355 ? otherUsage.reduce((sum, usage) => sum + (usage?.outputTokens ?? 0), 0)
356 : undefined,
357 nativeChildTurns: children.length,
358 childUsageStatus: childStatus,
359 childInputTokens: sumOf(measuredChildren, "inputTokens", false),
360 childOutputTokens: sumOf(measuredChildren, "outputTokens", false),
361 childrenRunning: report.childrenRunning ? true : undefined,
362 ...modelObservation(
363 [...rootUsage, ...childUsage].map((usage) => usage?.models),
364 label,
365 ),
366 durationMs:
367 startedAt.length === 0
368 ? undefined
369 : Math.max(0, DateTime.toEpochMillis(report.completedAt) - Math.min(...startedAt)),
370 });
371 }
372}
373
374function evictOldest(collection: Map<RunId, unknown> | Set<RunId>) {
375 if (collection.size <= MAX_TRACKED_RUNS) return;
376 const oldest = collection.keys().next().value;
377 if (oldest !== undefined) collection.delete(oldest);
378}
379
380export class ProviderTurnAnalytics extends Context.Reference<{
381 readonly runStarted: (input: {
382 readonly runId: RunId;
383 readonly driver: string;
384 readonly context: ProviderTurnAnalyticsContext;
385 }) => Effect.Effect<void>;
386 readonly turnCompleted: (report: ProviderTurnReport) => Effect.Effect<void>;
387 readonly runSettled: (report: RunSettledReport) => Effect.Effect<void>;
388}>("t3/telemetry/ProviderTurnAnalytics", {
389 defaultValue: () => ({
390 runStarted: () => Effect.void,
391 turnCompleted: () => Effect.void,
392 runSettled: () => Effect.void,
393 }),
394}) {}
395
396export const layer = Layer.effect(
397 ProviderTurnAnalytics,
398 Effect.gen(function* () {
399 const analytics = yield* AnalyticsService.AnalyticsService;
400 const registry = yield* ProviderRegistry.ProviderRegistry;
401 const ledger = new RequestUsageLedger();
402 const labeler = Effect.map(
403 registry.getProviders,
404 (providers): ModelLabeler =>
405 (model) =>
406 catalogModelLabel(providers, model),
407 );
408 return {
409 runStarted: Effect.fn("ProviderTurnAnalytics.runStarted")((input) =>
410 Effect.sync(() => ledger.start(input.runId, input.driver, input.context)),
411 ),
412 turnCompleted: Effect.fn("ProviderTurnAnalytics.turnCompleted")(function* (report) {
413 ledger.add(report);
414 yield* analytics.record(
415 "provider.turn.completed",
416 providerTurnProperties(report, yield* labeler),
417 );
418 }),
419 runSettled: Effect.fn("ProviderTurnAnalytics.runSettled")(function* (report) {
420 const properties = ledger.settle(report, yield* labeler);
421 if (properties !== undefined) {
422 yield* analytics.record("provider.request.completed", properties);
423 }
424 }),
425 };
426 }),
427);