Open History to see the last commit
1import * as Clock from "effect/Clock";
2import * as Duration from "effect/Duration";
3import * as Effect from "effect/Effect";
4import * as Exit from "effect/Exit";
5import * as Metric from "effect/Metric";
6import { dual } from "effect/Function";
7
8import { compactMetricAttributes, outcomeFromExit } from "./Attributes.ts";
9
10export const rpcRequestsTotal = Metric.counter("t3_rpc_requests_total", {
11 description: "Total RPC requests handled by the websocket RPC server.",
12});
13
14export const rpcRequestDuration = Metric.timer("t3_rpc_request_duration", {
15 description: "RPC request handling duration.",
16});
17
18export const orchestrationCommandsTotal = Metric.counter("t3_orchestration_commands_total", {
19 description: "Total orchestration commands dispatched, by outcome.",
20});
21
22export const orchestrationCommandDuration = Metric.timer("t3_orchestration_command_duration", {
23 description: "Orchestration command handling duration, excluding thread dispatch lock wait.",
24});
25
26export const orchestrationEffectClaimsTotal = Metric.counter(
27 "t3_orchestration_effect_claims_total",
28 {
29 description: "Total completed orchestration effect outbox claim attempts by result.",
30 },
31);
32
33export const orchestrationEffectQueueWait = Metric.timer("t3_orchestration_effect_queue_wait", {
34 description:
35 "Time from an orchestration effect's temporal availability until claim, including same-thread blocking.",
36});
37
38export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {
39 description: "Total provider session lifecycle operations.",
40});
41
42export const providerTurnsTotal = Metric.counter("t3_provider_turns_total", {
43 description: "Total provider turn lifecycle operations.",
44});
45
46export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
47 description: "Time for the provider adapter to start a turn, not how long the turn runs.",
48});
49
50// OTLP carries the unit, and Prometheus names the series by it: a unitless gauge gains `_ratio`.
51export const serverResidentBytes = Metric.gauge("t3_server_resident_bytes", {
52 description: "Resident memory of the server process.",
53 attributes: { unit: "By" },
54});
55
56export const serverHeapUsedBytes = Metric.gauge("t3_server_heap_used_bytes", {
57 description: "JavaScript heap in use by the server process.",
58 attributes: { unit: "By" },
59});
60
61export const serverCpuUtilization = Metric.gauge("t3_server_cpu_utilization", {
62 description: "CPU time the server used per second over the last sample; 1 is one full core.",
63});
64
65export const serverEventLoopUtilization = Metric.gauge("t3_server_event_loop_utilization", {
66 description: "Share of the last sample the server's event loop spent busy.",
67});
68
69export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_max", {
70 description: "Longest event loop delay over the last sample.",
71 attributes: { unit: "ms" },
72});
73
74export const runsTotal = Metric.counter("t3_runs_total", {
75 description: "Runs that finished, by outcome. A later rollback does not count again.",
76});
77
78export const runDuration = Metric.timer("t3_run_duration", {
79 description: "Wall time of a run, from when it started to its terminal status.",
80});
81
82export const runQueueWait = Metric.timer("t3_run_queue_wait", {
83 description: "Time a run waited between being requested and starting.",
84});
85
86export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
87 description: "Total git commands executed by the server runtime.",
88});
89
90export const gitCommandDuration = Metric.timer("t3_git_command_duration", {
91 description: "Git command execution duration.",
92});
93
94export const terminalSessionsTotal = Metric.counter("t3_terminal_sessions_total", {
95 description: "Total terminal sessions started.",
96});
97
98export const webhookDeliveriesTotal = Metric.counter("t3_webhook_deliveries_total", {
99 description: "Webhook requests handled, by outcome and source.",
100});
101
102export const webhookDeliveryDuration = Metric.timer("t3_webhook_delivery_duration", {
103 description: "Time to verify, log, and enqueue one webhook request.",
104});
105
106export const webhookHeldDelay = Metric.timer("t3_webhook_held_delay", {
107 description:
108 "Time between the relay receiving a webhook request and the environment handling it.",
109});
110
111export const webhookRunsTotal = Metric.counter("t3_webhook_runs_total", {
112 description: "Runs started from webhook deliveries, by outcome.",
113});
114
115export const secretRequestsTotal = Metric.counter("t3_secret_requests_total", {
116 description: "Secrets agents asked users for, by how each request ended.",
117});
118
119export const secretRefsConsumedTotal = Metric.counter("t3_secret_refs_consumed_total", {
120 description: "Secret refs tools tried to use, by result.",
121});
122
123export const metricAttributes = (
124 attributes: Readonly<Record<string, unknown>>,
125): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));
126
127export const increment = (
128 metric: Metric.Metric<number, unknown>,
129 attributes: Readonly<Record<string, unknown>>,
130 amount = 1,
131) => Metric.update(Metric.withAttributes(metric, metricAttributes(attributes)), amount);
132
133export interface WithMetricsOptions {
134 readonly counter?: Metric.Metric<number, unknown>;
135 readonly timer?: Metric.Metric<Duration.Duration, unknown>;
136 readonly attributes?:
137 | Readonly<Record<string, unknown>>
138 | (() => Readonly<Record<string, unknown>>);
139 readonly outcomeAttributes?: (
140 outcome: ReturnType<typeof outcomeFromExit>,
141 ) => Readonly<Record<string, unknown>>;
142}
143
144const recordMetrics =
145 /* Instrumentation must not create its own span for every recorded operation. */
146 Effect.fnUntraced(function* (
147 options: WithMetricsOptions,
148 startedAt: bigint,
149 exit: Exit.Exit<unknown, unknown>,
150 ) {
151 const duration = Duration.nanos((yield* Clock.monotonicTimeNanos) - startedAt);
152 const baseAttributes =
153 typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});
154
155 if (options.timer) {
156 yield* Metric.update(
157 Metric.withAttributes(options.timer, metricAttributes(baseAttributes)),
158 duration,
159 );
160 }
161
162 if (options.counter) {
163 const outcome = outcomeFromExit(exit);
164 yield* Metric.update(
165 Metric.withAttributes(
166 options.counter,
167 metricAttributes({
168 ...baseAttributes,
169 outcome,
170 ...(options.outcomeAttributes ? options.outcomeAttributes(outcome) : {}),
171 }),
172 ),
173 1,
174 );
175 }
176 });
177
178// Monotonic timing ignores clock corrections; exit finalizers count interruptions.
179const withMetricsImpl = <A, E, R>(
180 effect: Effect.Effect<A, E, R>,
181 options: WithMetricsOptions,
182): Effect.Effect<A, E, R> =>
183 Effect.flatMap(Clock.monotonicTimeNanos, (startedAt) =>
184 Effect.onExit(effect, (exit) => recordMetrics(options, startedAt, exit)),
185 );
186
187export const withMetrics: {
188 (
189 options: WithMetricsOptions,
190 ): <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
191 <A, E, R>(effect: Effect.Effect<A, E, R>, options: WithMetricsOptions): Effect.Effect<A, E, R>;
192} = dual(2, withMetricsImpl);
193
194// Shaped as an `Effect.fn` combinator: the command is the instrumented function's argument.
195export const withCommandMetrics = <A, E, R>(
196 effect: Effect.Effect<A, E, R>,
197 command: { readonly type: string },
198): Effect.Effect<A, E, R> =>
199 withMetrics(effect, {
200 counter: orchestrationCommandsTotal,
201 timer: orchestrationCommandDuration,
202 attributes: { commandType: command.type },
203 });