Skip to content
Source
Sign inSign up
Open History to see the last commit

EventLoopMonitor.ts

259 lines10.0 KB

1// @effect-diagnostics nodeBuiltinImport:off - only node:perf_hooks exposes the event loop delay histogram, and node:os the load average.
2import * as NodeOS from "node:os";
3import * as NodePerfHooks from "node:perf_hooks";
4​
5import type { ServerEventLoopStall } from "@t3tools/contracts";
6import { RotatingFileSink, RotatingFileSinkError } from "@t3tools/shared/logging";
7import * as Context from "effect/Context";
8import * as DateTime from "effect/DateTime";
9import * as Effect from "effect/Effect";
10import * as Layer from "effect/Layer";
11import * as Metric from "effect/Metric";
12import * as Path from "effect/Path";
13import * as Ref from "effect/Ref";
14import * as Schema from "effect/Schema";
15import type * as Scope from "effect/Scope";
16import * as Semaphore from "effect/Semaphore";
17​
18import * as ServerConfig from "../config.ts";
19import * as Metrics from "./Metrics.ts";
20​
21// Subtract the histogram's timer resolution; this can undercount stalls by up to RESOLUTION_MS.
22const RESOLUTION_MS = 1000;
23const STALL_THRESHOLD_MS = 2000;
24const SAMPLE_INTERVAL = "30 seconds";
25const STALL_LOG_MAX_BYTES = 256 * 1024;
26// Long enough that users felt it. Shorter stalls stay in the log only.
27const NOTICEABLE_STALL_MS = 5000;
28// Clients show what they get, so an old stall must not explain a new drop.
29const NOTICEABLE_STALL_MAX_AGE_MS = 5 * 60_000;
30​
31export class EventLoopMonitor extends Context.Reference<{
32 /**
33 * The latest stall of 5 s or more, if seen in the last 5 minutes. It samples
34 * first, so a stall that ended just before a client reconnected is included.
35 */
36 readonly latestNoticeableStall: Effect.Effect<ServerEventLoopStall | undefined>;
37}>("t3/observability/EventLoopMonitor", {
38 defaultValue: () => ({ latestNoticeableStall: Effect.undefined }),
39}) {}
40​
41/** Delay in ns, active time in ms, CPU in µs. */
42export interface EventLoopReadings {
43 readonly delayMaxNs: number;
44 readonly activeMs: number;
45 readonly utilization: number;
46 readonly usage: Pick<
47 NodeJS.ResourceUsage,
48 | "userCPUTime"
49 | "systemCPUTime"
50 | "majorPageFault"
51 | "minorPageFault"
52 | "involuntaryContextSwitches"
53 >;
54 readonly rssBytes: number;
55 readonly heapUsedBytes: number;
56 /** Wall time the readings cover. */
57 readonly elapsedMs: number;
58 /** One-minute load average, undefined on Windows. */
59 readonly loadAverage: number | undefined;
60 readonly cpuCount: number;
62​
63// Node skips the first gap after a reset, so a stall right at a sample boundary can be missed.
64const makeNodeSampler = Effect.gen(function* () {
65 const histogram = yield* Effect.acquireRelease(
66 Effect.sync(() => {
67 const histogram = NodePerfHooks.monitorEventLoopDelay({ resolution: RESOLUTION_MS });
68 histogram.enable();
69 return histogram;
70 }),
71 (histogram) => Effect.sync(() => histogram.disable()),
72 );
73 let elu = NodePerfHooks.performance.eventLoopUtilization();
74 let usage = process.resourceUsage();
75 let sampledAt = NodePerfHooks.performance.now();
76​
77 return Effect.sync(() => {
78 const nextElu = NodePerfHooks.performance.eventLoopUtilization();
79 const nextUsage = process.resourceUsage();
80 const nextSampledAt = NodePerfHooks.performance.now();
81 const memory = process.memoryUsage();
82 const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu);
83 const readings: EventLoopReadings = {
84 delayMaxNs: histogram.max,
85 activeMs: loop.active,
86 utilization: loop.utilization,
87 usage: {
88 userCPUTime: nextUsage.userCPUTime - usage.userCPUTime,
89 systemCPUTime: nextUsage.systemCPUTime - usage.systemCPUTime,
90 majorPageFault: nextUsage.majorPageFault - usage.majorPageFault,
91 minorPageFault: nextUsage.minorPageFault - usage.minorPageFault,
92 involuntaryContextSwitches:
93 nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches,
94 },
95 rssBytes: memory.rss,
96 heapUsedBytes: memory.heapUsed,
97 elapsedMs: nextSampledAt - sampledAt,
98 // Windows has no load average and reports 0, which a running process never sees elsewhere.
99 loadAverage: NodeOS.loadavg()[0] || undefined,
100 cpuCount: NodeOS.availableParallelism(),
101 };
102 histogram.reset();
103 elu = nextElu;
104 usage = nextUsage;
105 sampledAt = nextSampledAt;
106 return readings;
107 });
108});
109​
110export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => {
111 const delayMs = Math.round(delayMaxNs / 1e6) - RESOLUTION_MS;
112 // libuv clocks run during macOS/Windows sleep; idle poll time distinguishes it from stalls.
113 if (delayMs <= STALL_THRESHOLD_MS || activeMs < delayMs) return undefined;
114 return delayMs;
115};
116​
117// Untraced: it runs inside every 30-second tick, which already has the monitor's own span.
118const recordGauges = Effect.fnUntraced(function* (readings: EventLoopReadings) {
119 const cpuMs = (readings.usage.userCPUTime + readings.usage.systemCPUTime) / 1000;
120 yield* Metric.update(Metrics.serverResidentBytes, readings.rssBytes);
121 yield* Metric.update(Metrics.serverHeapUsedBytes, readings.heapUsedBytes);
122 yield* Metric.update(Metrics.serverEventLoopUtilization, readings.utilization);
123 yield* Metric.update(
124 Metrics.serverEventLoopDelayMax,
125 Math.max(0, Math.round(readings.delayMaxNs / 1e6) - RESOLUTION_MS),
126 );
127 if (readings.elapsedMs > 0) {
128 yield* Metric.update(Metrics.serverCpuUtilization, cpuMs / readings.elapsedMs);
129 }
130});
131​
132const encodeStallLine = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
133​
134export interface StallLogOptions {
135 readonly filePath: string;
136 readonly maxBytes: number;
138​
139const layerWith = (
140 makeSampler: Effect.Effect<Effect.Effect<EventLoopReadings>, never, Scope.Scope>,
141 stallLog: StallLogOptions,
142) =>
143 Layer.effect(
144 EventLoopMonitor,
145 Effect.gen(function* () {
146 const sample = yield* makeSampler;
147 // Clients sample on connect as well as the timer, and the sampler resets shared state.
148 const sampling = yield* Semaphore.make(1);
149 const warmedUp = yield* Ref.make(false);
150 const latestNoticeableStall = yield* Ref.make<
151 { readonly stall: ServerEventLoopStall; readonly detectedAtMs: number } | undefined
152 >(undefined);
153 const stallSink = yield* Effect.try({
154 try: () =>
155 new RotatingFileSink({
156 filePath: stallLog.filePath,
157 maxBytes: stallLog.maxBytes,
158 maxFiles: 1,
159 throwOnError: true,
160 }),
161 catch: (cause) =>
162 new RotatingFileSinkError({
163 operation: "initialize",
164 filePath: stallLog.filePath,
165 cause,
166 }),
167 }).pipe(
168 Effect.catch(() =>
169 Effect.logWarning("event loop stall log unavailable", {
170 filePath: stallLog.filePath,
171 }).pipe(Effect.as(undefined)),
172 ),
173 );
174 const tick = Effect.gen(function* () {
175 const readings = yield* sample;
176 yield* recordGauges(readings);
177 const delayMaxMs = stallMs(readings);
178 if (delayMaxMs === undefined) return;
179 const { utilization, usage, rssBytes, loadAverage, cpuCount } = readings;
180 const now = yield* DateTime.now;
181 const detectedAt = DateTime.formatIso(now);
182 if (delayMaxMs >= NOTICEABLE_STALL_MS) {
183 yield* Ref.set(latestNoticeableStall, {
184 detectedAtMs: DateTime.toEpochMillis(now),
185 stall: {
186 detectedAt,
187 durationMs: delayMaxMs,
188 ...(loadAverage === undefined
189 ? {}
190 : { loadAverage: Math.round(loadAverage * 10) / 10 }),
191 cpuCount,
192 },
193 });
194 }
195 const attributes = {
196 delayMaxMs,
197 utilization: Math.round(utilization * 100) / 100,
198 cpuUserMs: Math.round(usage.userCPUTime / 1000),
199 cpuSystemMs: Math.round(usage.systemCPUTime / 1000),
200 majorPageFaults: usage.majorPageFault,
201 minorPageFaults: usage.minorPageFault,
202 involuntaryContextSwitches: usage.involuntaryContextSwitches,
203 rssMb: Math.round(rssBytes / 1024 / 1024),
204 };
205 yield* Effect.logWarning(`event loop stalled for ${delayMaxMs} ms`).pipe(
206 Effect.withSpan("server.eventLoop.stall", { root: true, level: "Warn", attributes }),
207 );
208 if (stallSink !== undefined) {
209 yield* encodeStallLine({ time: detectedAt, ...attributes }).pipe(
210 Effect.flatMap((line) =>
211 Effect.try({
212 try: () => stallSink.write(`${line}\n`),
213 catch: (cause) =>
214 new RotatingFileSinkError({
215 operation: "write",
216 filePath: stallLog.filePath,
217 cause,
218 }),
219 }),
220 ),
221 Effect.catch(() =>
222 Effect.logWarning("event loop stall log write failed", {
223 filePath: stallLog.filePath,
224 }),
225 ),
226 );
227 }
228 }).pipe(sampling.withPermits(1));
229 const wait = Effect.sleep(SAMPLE_INTERVAL);
230 // The first sample includes migrations and bootstrap; skip expected startup stalls.
231 yield* wait.pipe(
232 Effect.andThen(sampling.withPermits(1)(sample)),
233 Effect.andThen(Ref.set(warmedUp, true)),
234 Effect.andThen(wait.pipe(Effect.andThen(tick), Effect.forever)),
235 Effect.forkScoped,
236 );
237​
238 return EventLoopMonitor.of({
239 latestNoticeableStall: Effect.gen(function* () {
240 if (yield* Ref.get(warmedUp)) yield* tick;
241 const latest = yield* Ref.get(latestNoticeableStall);
242 if (latest === undefined) return undefined;
243 const ageMs = DateTime.toEpochMillis(yield* DateTime.now) - latest.detectedAtMs;
244 return ageMs <= NOTICEABLE_STALL_MAX_AGE_MS ? latest.stall : undefined;
245 }),
246 });
247 }),
248 );
249​
250export const layer = Layer.unwrap(
251 Effect.gen(function* () {
252 const { serverTracePath } = yield* ServerConfig.ServerConfig;
253 const path = yield* Path.Path;
254 return layerWith(makeNodeSampler, {
255 filePath: path.join(path.dirname(serverTracePath), "event-loop-stalls.ndjson"),
256 maxBytes: STALL_LOG_MAX_BYTES,
257 });
258 }),
259);