1import {
2 NodeId,
3 ProviderInstanceId,
4 ProviderThreadId,
5 ProviderTurnId,
6 RunId,
7 type OrchestrationV2ProviderTurn,
8 type ServerProvider,
9 type TurnTokenUsage,
10} from "@t3tools/contracts";
11import { describe, expect, it } from "@effect/vitest";
12import * as DateTime from "effect/DateTime";
13import * as Effect from "effect/Effect";
14import * as Layer from "effect/Layer";
15
16import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
17import * as AnalyticsService from "./AnalyticsService.ts";
18import { catalogModelLabel } from "./ProviderDimensions.ts";
19import {
20 ProviderTurnAnalytics,
21 RequestUsageLedger,
22 layer,
23 providerTurnProperties,
24 type ProviderTurnAnalyticsContext,
25 type ProviderTurnSegment,
26} from "./ProviderUsageAnalytics.ts";
27
28const providers = [
29 // Codex lists whatever its configured model provider serves, including private deployments.
30 { driver: "codex", models: [{ slug: "gpt-6-sol" }, { slug: "acme-azure-deployment" }] },
31 { driver: "claudeAgent", models: [{ slug: "claude-opus-5-5" }, { slug: "claude-haiku-4-5" }] },
32 { driver: "opencode", models: [{ slug: "ollama/acme-internal" }] },
33].map(
34 ({ driver, models }) =>
35 ({
36 instanceId: ProviderInstanceId.make(driver),
37 driver,
38 models: models.map((model) => ({ ...model, isCustom: false })),
39 }) as unknown as ServerProvider,
40);
41const label = (model: string) => catalogModelLabel(providers, model);
42
43const at = (ms: number) => DateTime.makeUnsafe(ms);
44const complete = (input: number, output: number, extra: Partial<TurnTokenUsage> = {}) =>
45 ({
46 usageStatus: "complete",
47 usageScope: "main_agent",
48 hasSubagents: false,
49 inputTokens: input,
50 outputTokens: output,
51 ...extra,
52 }) as TurnTokenUsage;
53
54const turn = (
55 id: string,
56 usage: TurnTokenUsage | undefined,
57 status: OrchestrationV2ProviderTurn["status"] = "completed",
58): OrchestrationV2ProviderTurn => ({
59 id: ProviderTurnId.make(id),
60 providerThreadId: ProviderThreadId.make("provider-thread"),
61 nodeId: NodeId.make(`node:${id}`),
62 runAttemptId: null,
63 nativeTurnRef: null,
64 ordinal: 1,
65 status,
66 startedAt: at(1_000),
67 completedAt: at(4_000),
68 ...(usage === undefined ? {} : { turnTokenUsage: usage }),
69});
70
71const context = (
72 runId: string,
73 segment: ProviderTurnSegment,
74 model = "gpt-6-sol",
75): ProviderTurnAnalyticsContext => ({
76 runId: RunId.make(runId),
77 modelSelection: {
78 instanceId: ProviderInstanceId.make("codex"),
79 model,
80 options: [
81 { id: "reasoningEffort", value: "xhigh" },
82 { id: "serviceTier", value: "priority" },
83 ],
84 },
85 runtimeMode: "full-access",
86 interactionMode: "default",
87 origin: { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
88 segment,
89});
90
91const settled = (runId: string, childrenRunning = false) => ({
92 runId: RunId.make(runId),
93 status: "completed" as const,
94 completedAt: at(9_000),
95 childrenRunning,
96});
97
98describe("provider usage reporting", () => {
99 it.effect(
100 "records one request across a continuation, a child, a replay, and a late child",
101 () => {
102 const recorded: Array<readonly [string, unknown]> = [];
103 const layerTest = layer.pipe(
104 Layer.provide(
105 Layer.mergeAll(
106 Layer.mock(AnalyticsService.AnalyticsService)({
107 record: (event, properties) =>
108 Effect.sync(() => void recorded.push([event, properties])),
109 }),
110 Layer.mock(ProviderRegistry.ProviderRegistry)({
111 getProviders: Effect.succeed(providers),
112 }),
113 ),
114 ),
115 );
116 return Effect.gen(function* () {
117 const analytics = yield* ProviderTurnAnalytics;
118 const root = context("run-a", "initial");
119 yield* analytics.runStarted({ runId: RunId.make("run-a"), driver: "codex", context: root });
120 const first = { driver: "codex", context: root };
121 const firstTurn = turn("root", complete(100, 10, { models: ["gpt-6-sol"] }));
122 yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
123 yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
124 yield* analytics.turnCompleted({
125 driver: "codex",
126 context: context("run-a", "continuation"),
127 providerTurn: turn("goal", complete(50, 5, { models: ["gpt-6-sol"] })),
128 });
129 yield* analytics.turnCompleted({
130 driver: "codex",
131 context: context("run-a", "native_child"),
132 providerTurn: turn("child", complete(20, 2)),
133 });
134 yield* analytics.runSettled(settled("run-a"));
135 yield* analytics.turnCompleted({
136 driver: "codex",
137 context: context("run-a", "native_child"),
138 providerTurn: turn("late-child", complete(7, 1)),
139 });
140 yield* analytics.runSettled(settled("run-a"));
141 yield* analytics.runStarted({
142 runId: RunId.make("run-b"),
143 driver: "codex",
144 context: context("run-b", "initial"),
145 });
146 yield* analytics.runSettled(settled("run-b"));
147
148 const requests = recorded.filter(([event]) => event === "provider.request.completed");
149 expect(recorded.filter(([event]) => event === "provider.turn.completed")).toHaveLength(5);
150 expect(requests).toHaveLength(2);
151 // The whole payload: no run, turn, or thread ids, prompts, or raw model names.
152 expect(requests[0]?.[1]).toEqual({
153 reportVersion: 2,
154 provider: "codex",
155 requestKind: "user",
156 terminalStatus: "completed",
157 lifecycleObserved: true,
158 model: "gpt-6-sol",
159 effort: "xhigh",
160 serviceTier: "fast",
161 runtimeMode: "full-access",
162 interactionMode: "default",
163 createdBy: "user",
164 creationSource: "web",
165 scheduled: false,
166 subagent: false,
167 providerTurns: 2,
168 continuations: 1,
169 steeringRestarts: 0,
170 usageStatus: "complete",
171 inputTokens: 150,
172 outputTokens: 15,
173 nativeChildTurns: 1,
174 childUsageStatus: "complete",
175 childInputTokens: 20,
176 childOutputTokens: 2,
177 durationMs: 8_000,
178 });
179 expect(requests[1]?.[1]).toMatchObject({ providerTurns: 0, usageStatus: "unavailable" });
180 }).pipe(Effect.provide(layerTest));
181 },
182 );
183
184 it("keeps a request partial when its start, a steered segment, or a counter is missing", () => {
185 const ledger = new RequestUsageLedger();
186 ledger.start(RunId.make("restarted"), "codex", {
187 ...context("restarted", "initial"),
188 restartContinuation: true,
189 });
190 ledger.add({
191 driver: "codex",
192 context: context("restarted", "initial"),
193 providerTurn: turn("after-restart", complete(10, 1)),
194 });
195 expect(ledger.settle(settled("restarted"), label)).toMatchObject({
196 restartContinuation: true,
197 lifecycleObserved: false,
198 usageStatus: "partial",
199 inputTokens: 10,
200 });
201
202 ledger.start(RunId.make("steered"), "claudeAgent", context("steered", "initial"));
203 ledger.add({
204 driver: "claudeAgent",
205 context: context("steered", "initial"),
206 providerTurn: turn(
207 "steered-away",
208 { usageStatus: "partial", usageScope: "main_agent", hasSubagents: false, outputTokens: 4 },
209 "interrupted",
210 ),
211 });
212 ledger.add({
213 driver: "claudeAgent",
214 context: context("steered", "steering_restart"),
215 providerTurn: turn("steered-to", complete(30, 6)),
216 });
217 expect(ledger.settle(settled("steered"), label)).toMatchObject({
218 usageStatus: "partial",
219 steeringRestarts: 1,
220 inputTokens: 30,
221 outputTokens: 10,
222 });
223
224 ledger.start(RunId.make("silent"), "grok", context("silent", "initial"));
225 ledger.add({
226 driver: "grok",
227 context: context("silent", "initial"),
228 providerTurn: turn("silent", undefined),
229 });
230 const silent = ledger.settle(settled("silent", true), label);
231 expect(silent).toMatchObject({
232 usageStatus: "unavailable",
233 childrenRunning: true,
234 childUsageStatus: "unavailable",
235 });
236 expect(silent).not.toHaveProperty("outputTokens");
237 });
238
239 it("confirms only the models a provider reported", () => {
240 const opus = context("models", "initial", "claude-opus-5-5");
241 const properties = (usage: TurnTokenUsage | undefined) =>
242 providerTurnProperties(
243 { driver: "claudeAgent", context: opus, providerTurn: turn("t", usage) },
244 label,
245 );
246 expect(properties(complete(1, 1, { models: ["claude-haiku-4-5-20251001"] }))).toMatchObject({
247 model: "claude-opus-5-5",
248 observedModel: "claude-haiku-4-5",
249 mixedModels: false,
250 });
251 const unconfirmed = properties(complete(1, 1));
252 expect(unconfirmed).toMatchObject({ model: "claude-opus-5-5" });
253 expect(unconfirmed).not.toHaveProperty("mixedModels");
254 expect(properties(complete(1, 1, { models: ["a", "b"] }))).toMatchObject({ mixedModels: true });
255
256 const ledger = new RequestUsageLedger();
257 ledger.start(RunId.make("mixed"), "codex", context("mixed", "initial"));
258 ledger.add({
259 driver: "codex",
260 context: context("mixed", "initial"),
261 providerTurn: turn("root", complete(1, 1, { models: ["gpt-6-sol"] })),
262 });
263 ledger.add({
264 driver: "codex",
265 context: context("mixed", "native_child"),
266 providerTurn: turn("child", complete(1, 1, { models: ["gpt-6-luna"] })),
267 });
268 expect(ledger.settle(settled("mixed"), label)).toMatchObject({ mixedModels: true });
269 });
270
271 it("keeps zero, absent, local-command, and ledger usage distinct", () => {
272 const properties = (usage: TurnTokenUsage | undefined) =>
273 providerTurnProperties(
274 { driver: "claudeAgent", context: context("z", "initial"), providerTurn: turn("t", usage) },
275 label,
276 );
277 expect(properties(complete(0, 0))).toMatchObject({
278 usageStatus: "complete",
279 usageReported: true,
280 inputTokens: 0,
281 outputTokens: 0,
282 });
283 const absent = properties(undefined);
284 expect(absent).toMatchObject({ usageStatus: "unavailable", usageReported: false });
285 expect(absent).not.toHaveProperty("outputTokens");
286 expect(
287 properties(
288 complete(0, 0, { localCommand: true, otherUsage: { inputTokens: 900, outputTokens: 80 } }),
289 ),
290 ).toMatchObject({ localCommand: true, otherInputTokens: 900, otherOutputTokens: 80 });
291 expect(properties(complete(74_167, 236, { reasoningTokens: 30 }))).toMatchObject({
292 outputTokens: 236,
293 reasoningTokens: 30,
294 });
295 });
296
297 it("reports routed catalog models and closed efforts only", () => {
298 const routed = (model: string, variant: string) =>
299 providerTurnProperties(
300 {
301 driver: "opencode",
302 context: {
303 modelSelection: {
304 instanceId: ProviderInstanceId.make("opencode"),
305 model,
306 options: [{ id: "variant", value: variant }],
307 },
308 },
309 providerTurn: turn("t", complete(1, 1, { models: [model] })),
310 },
311 label,
312 );
313 expect(routed("my-proxy/claude-opus-5-5", "high")).toMatchObject({
314 model: "claude-opus-5-5",
315 observedModel: "claude-opus-5-5",
316 effort: "high",
317 });
318 for (const local of [
319 routed("ollama/acme-internal", "secret-variant"),
320 routed("acme-azure-deployment", "high"),
321 ]) {
322 expect(local).not.toHaveProperty("model");
323 expect(local).not.toHaveProperty("observedModel");
324 }
325 expect(routed("ollama/acme-internal", "secret-variant")).not.toHaveProperty("effort");
326 });
327
328 it("reports closed service tiers from a tier option or a fast mode toggle", () => {
329 const tier = (option: { id: string; value: string | boolean }) =>
330 providerTurnProperties(
331 {
332 driver: "codex",
333 context: {
334 modelSelection: {
335 instanceId: ProviderInstanceId.make("codex"),
336 model: "gpt-6-sol",
337 options: [option],
338 },
339 },
340 providerTurn: turn("t", complete(1, 1)),
341 },
342 label,
343 ).serviceTier;
344 expect(tier({ id: "serviceTier", value: "ultrafast" })).toBe("ultrafast");
345 expect(tier({ id: "serviceTier", value: "default" })).toBe("standard");
346 expect(tier({ id: "serviceTier", value: "/home/me/secret" })).toBeUndefined();
347 expect(tier({ id: "fastMode", value: true })).toBe("fast");
348 expect(tier({ id: "fastMode", value: false })).toBe("standard");
349 });
350});