Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

observability.test.ts

675 lines22.9 KB

1import { assert, describe, expect, it } from "@effect/vitest";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import * as Cause from "effect/Cause";
4import * as Effect from "effect/Effect";
5import * as Deferred from "effect/Deferred";
6import * as Fiber from "effect/Fiber";
7import * as Exit from "effect/Exit";
8import * as FileSystem from "effect/FileSystem";
9import { cast } from "effect/Function";
10import * as Layer from "effect/Layer";
11import * as Logger from "effect/Logger";
12import * as Path from "effect/Path";
13import * as Ref from "effect/Ref";
14import * as References from "effect/References";
15import * as Schema from "effect/Schema";
16import * as Tracer from "effect/Tracer";
17import type { OtlpTracer } from "effect/observability";
18​
19import {
20 causeErrorTag,
21 compactTraceAttributes,
22 decodeOtlpTraceRecords,
23 errorTag,
24 makeLocalFileTracer,
25 makeTraceSink,
26 type EffectTraceRecord,
27 type TraceRecord,
28 OtlpHeadersFromString,
29 truncateTraceAttributes,
30 processServiceInstanceId,
31} from "./observability.ts";
32​
33describe("processServiceInstanceId", () => {
34 it("stays the same for the process unless the operator set one", () => {
35 expect(processServiceInstanceId()).toBe(processServiceInstanceId({}));
36 expect(processServiceInstanceId({ "service.instance.id": "pod-7" })).toBe("pod-7");
37 });
38});
39​
40describe("errorTag", () => {
41 it("reports structural tags without retaining arbitrary values", () => {
42 assert.equal(errorTag({ _tag: "AcpRequestError" }), "AcpRequestError");
43 assert.equal(errorTag(new TypeError("secret-token-value")), "TypeError");
44 assert.equal(errorTag({ _tag: "secret token value" }), "TaggedError");
45 });
46});
47​
48describe("causeErrorTag", () => {
49 it("reports the tagged failure value instead of the Cause reason wrapper", () => {
50 assert.equal(
51 causeErrorTag(Cause.fail({ _tag: "ServerAuthInvalidCredentialError" })),
52 "ServerAuthInvalidCredentialError",
53 );
54 });
55});
56​
57const TraceRecordLine = Schema.Struct({
58 name: Schema.String,
59 spanId: Schema.String,
60 parentSpanId: Schema.optional(Schema.String),
61 attributes: Schema.Record(Schema.String, Schema.Unknown),
62 events: Schema.Array(
63 Schema.Struct({
64 name: Schema.String,
65 attributes: Schema.Record(Schema.String, Schema.Unknown),
66 }),
67 ),
68 exit: Schema.optional(
69 Schema.Struct({
70 _tag: Schema.String,
71 }),
72 ),
73});
74​
75const decodeTraceRecordLine = Schema.decodeUnknownSync(Schema.fromJsonString(TraceRecordLine));
76​
77const makeRecord = (name: string, suffix = ""): TraceRecord => ({
78 type: "effect-span",
79 name,
80 traceId: `trace-${name}-${suffix}`,
81 spanId: `span-${name}-${suffix}`,
82 sampled: true,
83 kind: "internal",
84 startTimeUnixNano: "1",
85 endTimeUnixNano: "2",
86 durationMs: 1,
87 attributes: {
88 payload: suffix,
89 },
90 events: [],
91 links: [],
92 exit: {
93 _tag: "Success",
94 },
95});
96​
97const readTraceRecords = Effect.fn("readTraceRecords")(function* (tracePath: string) {
98 const fileSystem = yield* FileSystem.FileSystem;
99 return (yield* fileSystem.readFileString(tracePath))
100 .trim()
101 .split("\n")
102 .filter((line) => line.length > 0)
103 .map((line) => decodeTraceRecordLine(line));
104});
105​
106const layerTest = (tracePath: string) =>
107 Layer.mergeAll(
108 Layer.effect(
109 Tracer.Tracer,
110 makeLocalFileTracer({
111 filePath: tracePath,
112 maxBytes: 1024 * 1024,
113 maxFiles: 2,
114 batchWindowMs: 10_000,
115 }),
116 ),
117 Logger.layer([Logger.tracerLogger], { mergeWithExisting: false }),
118 Layer.succeed(References.MinimumLogLevel, "Info"),
119 );
120​
121const nodeServicesIt = it.layer(NodeServices.layer);
122​
123describe("truncateTraceAttributes", () => {
124 it("clamps oversized strings at any depth without mutating the input", () => {
125 const stack = "s".repeat(2_000);
126 const attributes = {
127 "db.query.text": "q".repeat(2_000),
128 short: "ok",
129 error: { name: "Error", stack, nested: ["a".repeat(2_000)] },
130 };
131 const truncated = truncateTraceAttributes(attributes);
132​
133 assert.equal((truncated["db.query.text"] as string).length, 200 + "…[truncated]".length);
134 assert.equal(truncated["short"], "ok");
135 const error = truncated["error"] as { stack: string; nested: Array<string> };
136 assert.equal(error.stack.length, 500 + "…[truncated]".length);
137 assert.equal(error.nested[0]?.length, 500 + "…[truncated]".length);
138 assert.equal(attributes.error.stack, stack);
139 });
140});
141​
142describe("observability", () => {
143 it.effect(
144 "preserves successful results without retaining them in sampled or unsampled spans",
145 () =>
146 Effect.gen(function* () {
147 for (const sampled of [true, false]) {
148 const delegates: Array<Tracer.Span> = [];
149 const records: Array<EffectTraceRecord> = [];
150 const tracer = yield* makeLocalFileTracer({
151 filePath: "unused",
152 maxBytes: 1024,
153 maxFiles: 1,
154 batchWindowMs: 10_000,
155 sink: {
156 filePath: "unused",
157 push: (record) => {
158 if (record.type === "effect-span") records.push(record);
159 },
160 flush: Effect.void,
161 flushSync: () => {},
162 close: () => Effect.void,
163 },
164 delegate: Tracer.make({
165 span: (options) => {
166 const span = new Tracer.NativeSpan({ ...options, sampled });
167 delegates.push(span);
168 return span;
169 },
170 }),
171 });
172 const payload = { turnItems: [{ output: [{ text: "synthetic-result" }] }] };
173 let span: Tracer.Span | undefined;
174 const result = yield* Effect.gen(function* () {
175 span = yield* Effect.currentSpan;
176 return payload;
177 }).pipe(
178 Effect.withSpan("read-thread-projection"),
179 Effect.provideService(Tracer.Tracer, tracer),
180 );
181 assert.strictEqual(result, payload);
182 assert.isDefined(span);
183 for (const completed of [span!, ...delegates]) {
184 assert.equal(completed.status._tag, "Ended");
185 if (completed.status._tag === "Ended") {
186 assert.deepStrictEqual(completed.status.exit, Exit.void);
187 }
188 }
189 assert.deepStrictEqual(
190 records.map((record) => record.exit),
191 sampled ? [{ _tag: "Success" }] : [],
192 );
193 }
194 }).pipe(Effect.provide(NodeServices.layer)),
195 );
196​
197 it("normalizes circular arrays, maps, and sets without recursing forever", () => {
198 const array: Array<unknown> = ["alpha"];
199 array.push(array);
200​
201 const map = new Map<string, unknown>();
202 map.set("self", map);
203​
204 const set = new Set<unknown>();
205 set.add(set);
206​
207 assert.deepStrictEqual(
208 compactTraceAttributes({
209 array,
210 map,
211 set,
212 }),
213 {
214 array: ["alpha", "[Circular]"],
215 map: { self: "[Circular]" },
216 set: ["[Circular]"],
217 },
218 );
219 });
220​
221 it("keeps an object that appears twice without marking it circular", () => {
222 const shared = { cwd: "/repo" };
223 const sharedMap = new Map([["cwd", "/repo"]]);
224 const sharedSet = new Set(["/repo"]);
225​
226 assert.deepStrictEqual(
227 compactTraceAttributes({
228 ctx: { first: shared, second: shared },
229 list: [shared, shared],
230 maps: [sharedMap, sharedMap],
231 sets: [sharedSet, sharedSet],
232 }),
233 {
234 ctx: { first: { cwd: "/repo" }, second: { cwd: "/repo" } },
235 list: [{ cwd: "/repo" }, { cwd: "/repo" }],
236 maps: [{ cwd: "/repo" }, { cwd: "/repo" }],
237 sets: [["/repo"], ["/repo"]],
238 },
239 );
240 });
241​
242 it("caps shared references that would expand exponentially", () => {
243 let graph: Record<string, unknown> = { cwd: "/repo" };
244 for (let depth = 0; depth < 20; depth++) {
245 graph = { left: graph, right: graph };
246 }
247​
248 const serialized = JSON.stringify(compactTraceAttributes({ graph }));
249​
250 assert.include(serialized, '"[Truncated]"');
251 assert.isBelow(serialized.length, 1_000_000);
252 });
253​
254 nodeServicesIt("node services", (it) => {
255 it.effect(
256 "keeps trace producers responsive during a slow write, bounds the backlog, and drains on close",
257 () =>
258 Effect.scoped(
259 Effect.gen(function* () {
260 const fs = yield* FileSystem.FileSystem;
261 const path = yield* Path.Path;
262 const directory = yield* fs.makeTempDirectoryScoped({
263 prefix: "t3-trace-backpressure-",
264 });
265 const tracePath = path.join(directory, "trace.ndjson");
266 const started = yield* Deferred.make<void>();
267 const release = yield* Deferred.make<void>();
268 const closed = yield* Ref.make(false);
269 const sink = yield* makeTraceSink({
270 filePath: tracePath,
271 maxBytes: 5_000_000,
272 maxFiles: 2,
273 batchWindowMs: 10_000,
274 }).pipe(
275 Effect.provideService(FileSystem.FileSystem, {
276 ...fs,
277 writeFile: (file, data, options) =>
278 Effect.gen(function* () {
279 yield* Deferred.succeed(started, undefined);
280 yield* Deferred.await(release);
281 yield* fs.writeFile(file, data, options);
282 }),
283 }),
284 );
285 sink.push(makeRecord("first"));
286 const flushing = yield* Effect.forkChild(sink.flush);
287 yield* Deferred.await(started);
288 for (let index = 0; index < 2_000; index++)
289 sink.push(makeRecord("small", String(index)));
290 for (let index = 0; index < 3; index++)
291 sink.push({ ...makeRecord("large"), attributes: { payload: "x".repeat(2_000_000) } });
292 const closing = yield* Effect.forkChild(
293 sink.close().pipe(Effect.tap(() => Ref.set(closed, true))),
294 );
295 yield* Effect.yieldNow;
296 assert.isFalse(yield* Ref.get(closed));
297 yield* Deferred.succeed(release, undefined);
298 yield* Fiber.join(flushing);
299 yield* Fiber.join(closing);
300 const records = yield* readTraceRecords(tracePath);
301 assert.deepStrictEqual(
302 records.map((record) => record.name),
303 ["first", ...Array.from({ length: 2_000 }, () => "small"), "large", "large"],
304 );
305 assert.isTrue(yield* Ref.get(closed));
306 }),
307 ),
308 );
309​
310 it.effect("writes buffered records synchronously for process exit", () =>
311 Effect.scoped(
312 Effect.gen(function* () {
313 const fileSystem = yield* FileSystem.FileSystem;
314 const path = yield* Path.Path;
315 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-trace-sink-" });
316 const tracePath = path.join(tempDir, "exit.trace.ndjson");
317 const sink = yield* makeTraceSink({
318 filePath: tracePath,
319 maxBytes: 1_000_000,
320 maxFiles: 2,
321 batchWindowMs: 10_000,
322 });
323​
324 sink.push(makeRecord("before-exit"));
325 sink.flushSync();
326 sink.flushSync();
327​
328 const records = yield* readTraceRecords(tracePath);
329 assert.deepStrictEqual(
330 records.map((record) => record.name),
331 ["before-exit"],
332 );
333 }),
334 ),
335 );
336​
337 it.effect("keeps every trace file within the configured limit for threshold flushes", () =>
338 Effect.scoped(
339 Effect.gen(function* () {
340 const fileSystem = yield* FileSystem.FileSystem;
341 const path = yield* Path.Path;
342 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-trace-sink-" });
343 const tracePath = path.join(tempDir, "shared.trace.ndjson");
344 const maxBytes = 1_024;
345​
346 const sink = yield* makeTraceSink({
347 filePath: tracePath,
348 maxBytes,
349 maxFiles: 2,
350 batchWindowMs: 10_000,
351 });
352​
353 for (let index = 0; index < 256; index += 1) {
354 sink.push(makeRecord("threshold", `${index}-${"x".repeat(48)}`));
355 }
356 yield* sink.close();
357​
358 const matchingFiles = (yield* fileSystem.readDirectory(tempDir)).filter(
359 (entry) => entry === "shared.trace.ndjson" || entry.startsWith("shared.trace.ndjson."),
360 );
361 assert.include(matchingFiles, "shared.trace.ndjson.1");
362 for (const entry of matchingFiles) {
363 const stat = yield* fileSystem.stat(path.join(tempDir, entry));
364 assert.isAtMost(Number(stat.size), maxBytes, entry);
365 }
366 }),
367 ),
368 );
369​
370 it.effect("drops a single trace record that cannot fit within the configured limit", () =>
371 Effect.scoped(
372 Effect.gen(function* () {
373 const fileSystem = yield* FileSystem.FileSystem;
374 const path = yield* Path.Path;
375 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-trace-sink-" });
376 const tracePath = path.join(tempDir, "shared.trace.ndjson");
377 const maxBytes = 1_024;
378​
379 const sink = yield* makeTraceSink({
380 filePath: tracePath,
381 maxBytes,
382 maxFiles: 2,
383 batchWindowMs: 10_000,
384 });
385​
386 sink.push(makeRecord("oversized", "x".repeat(maxBytes * 2)));
387 sink.push(makeRecord("retained"));
388 yield* sink.close();
389​
390 const records = yield* readTraceRecords(tracePath);
391 const stat = yield* fileSystem.stat(tracePath);
392 assert.deepEqual(
393 records.map((record) => record.name),
394 ["retained"],
395 );
396 assert.isAtMost(Number(stat.size), maxBytes);
397 }),
398 ),
399 );
400​
401 it.effect("keeps WebSocket credentials out of trace files and exported spans", () =>
402 Effect.scoped(
403 Effect.gen(function* () {
404 const fileSystem = yield* FileSystem.FileSystem;
405 const path = yield* Path.Path;
406 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-trace-sink-" });
407 const tracePath = path.join(tempDir, "shared.trace.ndjson");
408 const sink = yield* makeTraceSink({
409 filePath: tracePath,
410 maxBytes: 1024 * 1024,
411 maxFiles: 2,
412 batchWindowMs: 10_000,
413 });
414 const delegates: Array<Tracer.NativeSpan> = [];
415 const tracer = yield* makeLocalFileTracer({
416 filePath: tracePath,
417 maxBytes: 1024 * 1024,
418 maxFiles: 2,
419 batchWindowMs: 10_000,
420 sink,
421 delegate: Tracer.make({
422 span: (options) => {
423 const span = new Tracer.NativeSpan(options);
424 delegates.push(span);
425 return span;
426 },
427 }),
428 });
429 const url = "/ws?wsTicket=ticket-secret&dpop=proof-secret&clientSurface=web";
430​
431 yield* Effect.annotateCurrentSpan({
432 "url.full": `http://127.0.0.1:3773${url}`,
433 "url.query": url.slice(4),
434 }).pipe(
435 Effect.withSpan("GET", { kind: "server" }),
436 Effect.provideService(Tracer.Tracer, tracer),
437 );
438 yield* Effect.die(new Error(`RequestParseError (GET ${url})`)).pipe(
439 Effect.tapCause((cause) => Effect.logError(cause)),
440 Effect.withSpan("DeviceHubProxy.proxyWebSocket"),
441 Effect.provideService(Tracer.Tracer, tracer),
442 Effect.provide(Logger.layer([Logger.tracerLogger], { mergeWithExisting: false })),
443 Effect.exit,
444 );
445 yield* sink.close();
446​
447 const written = yield* fileSystem.readFileString(tracePath);
448 const exported = delegates.map((span) =>
449 JSON.stringify({
450 attributes: Object.fromEntries(span.attributes),
451 events: span.events.map(([, , attributes]) => attributes),
452 exit:
453 span.status._tag === "Ended" && Exit.isFailure(span.status.exit)
454 ? Cause.pretty(span.status.exit.cause)
455 : "",
456 }),
457 );
458 for (const text of [written, ...exported]) {
459 assert.notInclude(text, "ticket-secret");
460 assert.notInclude(text, "proof-secret");
461 }
462 assert.include(written, "wsTicket=<redacted>&dpop=<redacted>&clientSurface=web");
463 assert.include(exported.join("\n"), "effect.cause");
464 }),
465 ),
466 );
467​
468 it.effect("drops only the invalid trace record when serialization fails", () =>
469 Effect.scoped(
470 Effect.gen(function* () {
471 const fileSystem = yield* FileSystem.FileSystem;
472 const path = yield* Path.Path;
473 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-trace-sink-" });
474 const tracePath = path.join(tempDir, "shared.trace.ndjson");
475​
476 const sink = yield* makeTraceSink({
477 filePath: tracePath,
478 maxBytes: 1024,
479 maxFiles: 2,
480 batchWindowMs: 10_000,
481 });
482​
483 const circular: Array<unknown> = [];
484 circular.push(circular);
485​
486 sink.push(makeRecord("alpha"));
487 sink.push({
488 ...makeRecord("invalid"),
489 attributes: {
490 circular,
491 },
492 } as TraceRecord);
493 sink.push(makeRecord("beta"));
494 yield* sink.close();
495​
496 const lines = yield* readTraceRecords(tracePath);
497​
498 assert.deepStrictEqual(
499 lines.map((line) => line.name),
500 ["alpha", "beta"],
501 );
502 }),
503 ),
504 );
505​
506 it.effect("writes nested spans to disk and captures log messages as span events", () =>
507 Effect.scoped(
508 Effect.gen(function* () {
509 const fileSystem = yield* FileSystem.FileSystem;
510 const path = yield* Path.Path;
511 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-local-tracer-" });
512 const tracePath = path.join(tempDir, "shared.trace.ndjson");
513​
514 yield* Effect.scoped(
515 Effect.gen(function* () {
516 const program = Effect.gen(function* () {
517 yield* Effect.annotateCurrentSpan({
518 "demo.parent": true,
519 });
520 yield* Effect.logInfo("parent event");
521 yield* Effect.gen(function* () {
522 yield* Effect.annotateCurrentSpan({
523 "demo.child": true,
524 });
525 yield* Effect.logInfo("child event");
526 }).pipe(Effect.withSpan("child-span"));
527 }).pipe(Effect.withSpan("parent-span"));
528​
529 yield* program.pipe(Effect.provide(layerTest(tracePath)));
530 }),
531 );
532​
533 const records = yield* readTraceRecords(tracePath);
534 assert.equal(records.length, 2);
535​
536 const parent = records.find((record) => record.name === "parent-span");
537 const child = records.find((record) => record.name === "child-span");
538​
539 assert.notEqual(parent, undefined);
540 assert.notEqual(child, undefined);
541 if (!parent || !child) {
542 return;
543 }
544​
545 assert.equal(child.parentSpanId, parent.spanId);
546 assert.equal(parent.attributes["demo.parent"], true);
547 assert.equal(child.attributes["demo.child"], true);
548 assert.equal(
549 parent.events.some((event) => event.name === "parent event"),
550 true,
551 );
552 assert.equal(
553 child.events.some((event) => event.name === "child event"),
554 true,
555 );
556 assert.equal(
557 child.events.some((event) => event.attributes["effect.logLevel"] === "INFO"),
558 true,
559 );
560 }),
561 ),
562 );
563​
564 it.effect("serializes interrupted spans with an interrupted exit status", () =>
565 Effect.scoped(
566 Effect.gen(function* () {
567 const fileSystem = yield* FileSystem.FileSystem;
568 const path = yield* Path.Path;
569 const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-local-tracer-" });
570 const tracePath = path.join(tempDir, "shared.trace.ndjson");
571​
572 yield* Effect.scoped(
573 Effect.exit(
574 Effect.interrupt.pipe(
575 Effect.withSpan("interrupt-span"),
576 Effect.provide(layerTest(tracePath)),
577 ),
578 ),
579 );
580​
581 const records = yield* readTraceRecords(tracePath);
582 assert.equal(records.length, 1);
583 assert.equal(records[0]?.name, "interrupt-span");
584 assert.equal(records[0]?.exit?._tag, "Interrupted");
585 }),
586 ),
587 );
588 });
589});
590​
591describe("decodeOtlpTraceRecords", () => {
592 it("decodes spans when OTLP/JSON omits unset and empty fields", () => {
593 // Protobuf JSON leaves out unset and empty fields, so this batch is valid.
594 const payload = cast<unknown, OtlpTracer.TraceData>({
595 resourceSpans: [
596 {},
597 {
598 scopeSpans: [
599 {},
600 {
601 spans: [
602 {
603 traceId: "trace-1",
604 spanId: "span-1",
605 name: "browser.request",
606 startTimeUnixNano: "1000000",
607 endTimeUnixNano: "3000000",
608 attributes: [{ key: "tags", value: { arrayValue: {} } }],
609 },
610 ],
611 },
612 ],
613 },
614 ],
615 });
616​
617 assert.deepStrictEqual(decodeOtlpTraceRecords(cast<unknown, OtlpTracer.TraceData>({})), []);
618 assert.deepStrictEqual(decodeOtlpTraceRecords(payload), [
619 {
620 type: "otlp-span",
621 name: "browser.request",
622 traceId: "trace-1",
623 spanId: "span-1",
624 sampled: true,
625 kind: "internal",
626 startTimeUnixNano: "1000000",
627 endTimeUnixNano: "3000000",
628 durationMs: 2,
629 attributes: { tags: [] },
630 resourceAttributes: {},
631 scope: { attributes: {} },
632 events: [],
633 links: [],
634 status: { code: "0" },
635 },
636 ]);
637 });
638});
639​
640describe("OtlpHeadersFromString", () => {
641 const decode = Schema.decodeUnknownSync(OtlpHeadersFromString);
642​
643 it.each([
644 {
645 name: "decodes percent-encoded values",
646 input: "authorization=Basic%20abc%3D%3D,x-tenant=t3",
647 expected: { authorization: "Basic abc==", "x-tenant": "t3" },
648 },
649 {
650 name: "ignores whitespace around separators",
651 input: "authorization=Basic%20abc%3D%3D, x-tenant = t3 ,",
652 expected: { authorization: "Basic abc==", "x-tenant": "t3" },
653 },
654 {
655 name: "keeps literal equals signs inside a value",
656 input: "authorization=Bearer abc==",
657 expected: { authorization: "Bearer abc==" },
658 },
659 {
660 name: "keeps an empty value",
661 input: "x-empty=",
662 expected: { "x-empty": "" },
663 },
664 ])("$name", ({ input, expected }) => {
665 expect(decode(input)).toEqual(expected);
666 });
667​
668 it.each([
669 { name: "rejects a pair without a separator", input: "authorization" },
670 { name: "rejects a pair without a key", input: "=value" },
671 { name: "rejects a malformed percent-encoding", input: "authorization=%E0" },
672 ])("$name", ({ input }) => {
673 expect(() => decode(input)).toThrow();
674 });
675});