Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.
2import * as NodeFS from "node:fs";
3import * as NodeNet from "node:net";
4
5import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
6import { describe, expect, it } from "@effect/vitest";
7import * as Clock from "effect/Clock";
8import * as Deferred from "effect/Deferred";
9import * as Duration from "effect/Duration";
10import * as Effect from "effect/Effect";
11import * as Fiber from "effect/Fiber";
12import * as Layer from "effect/Layer";
13import * as Queue from "effect/Queue";
14import * as Stream from "effect/Stream";
15import * as TestClock from "effect/testing/TestClock";
16import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
17import { ChildProcessSpawner } from "effect/process";
18import { vi } from "vite-plus/test";
19
20import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
21import * as ConnectReachability from "./ConnectReachability.ts";
22import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
23import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
24
25vi.mock("node:net", { spy: true });
26
27const checkedAt = "2026-10-19T00:00:00.000Z";
28const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
29const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
30const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
31 managedTunnelConfigured: true,
32 connector: connected,
33 endpoint: null,
34 edgeReachable: null,
35 exitNode: null,
36 ...overrides,
37});
38
39describe("classifyReachability", () => {
40 it.each([
41 [{ kind: "status", status: 502 }, "origin_unreachable"],
42 [{ kind: "status", status: 530 }, "no_connector"],
43 [{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"],
44 [{ kind: "status", status: 530, cloudflareError: "1016" }, "unexpected_response"],
45 [{ kind: "network_error" }, "timeout"],
46 ] as const)("maps endpoint result %o to %s", (endpoint, cause) => {
47 expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject(
48 { status: "unreachable", cause },
49 );
50 });
51
52 it("keeps the endpoint cause when the TCP edge test cannot explain it", () => {
53 // QUIC (UDP) can connect while TCP 7844 is blocked.
54 expect(
55 classifyReachability(
56 facts({ endpoint: { kind: "status", status: 502 }, edgeReachable: false }),
57 checkedAt,
58 ),
59 ).toMatchObject({ cause: "origin_unreachable" });
60 expect(
61 classifyReachability(
62 facts({
63 connector: {
64 pid: 1,
65 readyConnections: null,
66 hostname: "env.example.test",
67 protocol: null,
68 },
69 endpoint: { kind: "status", status: 502 },
70 edgeReachable: false,
71 }),
72 checkedAt,
73 ),
74 ).toMatchObject({ cause: "origin_unreachable" });
75 // Failed DNS also fails the edge lookups.
76 expect(
77 classifyReachability(
78 facts({
79 connector: {
80 pid: 1,
81 readyConnections: null,
82 hostname: "env.example.test",
83 protocol: null,
84 },
85 endpoint: { kind: "dns_failed" },
86 edgeReachable: false,
87 }),
88 checkedAt,
89 ),
90 ).toMatchObject({ cause: "dns_failed" });
91 });
92
93 it("waits on a connector that has not connected while the edge answers", () => {
94 expect(
95 classifyReachability(facts({ connector: notConnected, edgeReachable: true }), checkedAt),
96 ).toMatchObject({ cause: "connector_not_ready" });
97 });
98
99 it("does not point an unlinked server at a background service it may not have", () => {
100 // Desktop and `t3 serve` run the server without a background service.
101 const result = classifyReachability(facts({ managedTunnelConfigured: false }), checkedAt);
102 expect(result).toMatchObject({ cause: "not_linked" });
103 expect(result.status === "unreachable" && result.fix).not.toContain("t3 service");
104 });
105});
106
107interface ServedResponse {
108 readonly status: number;
109 readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number };
110}
111
112// What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.
113const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) =>
114 request.headers.accept === "application/json"
115 ? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" })
116 : `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`;
117
118type Served = Effect.Effect<ServedResponse, HttpClientError.HttpClientError>;
119
120const reachabilityLayer = (
121 respond: Served | ((request: HttpClientRequest.HttpClientRequest) => Served),
122 restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>,
123) =>
124 ConnectReachability.layer.pipe(
125 Layer.provide(
126 Layer.mergeAll(
127 Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({
128 inspectConnector: Effect.succeed(connected),
129 readinessChanged: Stream.never,
130 restartConnector,
131 }),
132 Layer.mock(ServerSecretStore.ServerSecretStore)({
133 get: () => Effect.succeedSome(new Uint8Array()),
134 }),
135 Layer.succeed(
136 HttpClient.HttpClient,
137 HttpClient.make((request) =>
138 (Effect.isEffect(respond) ? respond : respond(request)).pipe(
139 Effect.map(({ status, body }) =>
140 HttpClientResponse.fromWeb(
141 request,
142 new Response(
143 typeof body === "object" && "cloudflareError" in body
144 ? cloudflareErrorPage(request, body.cloudflareError)
145 : body,
146 { status },
147 ),
148 ),
149 ),
150 ),
151 ),
152 ),
153 Layer.succeed(
154 ChildProcessSpawner.ChildProcessSpawner,
155 ChildProcessSpawner.make(() => Effect.die("no tailscale")),
156 ),
157 ),
158 ),
159 );
160
161const mockEdgeConnect = () =>
162 vi.spyOn(NodeNet, "connect").mockImplementation(() => {
163 const socket = new NodeNet.Socket();
164 queueMicrotask(() => socket.emit("connect"));
165 return socket;
166 });
167
168describe("ConnectReachability stale connector recovery", () => {
169 it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
170 Effect.gen(function* () {
171 using _connect = mockEdgeConnect();
172 const restarts = yield* Queue.unbounded<number>();
173 // Cloudflare answers 530 while cloudflared still reports four ready connections.
174 let status = 530;
175 let body = { cloudflareError: 1033 };
176 const layer = reachabilityLayer(
177 Effect.sync(() => ({ status, body })),
178 (pid, stillDead) =>
179 stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
180 );
181 // The scheduled checks run in the layer's scope, so the assertions run inside it.
182 yield* Effect.gen(function* () {
183 const reachability = yield* ConnectReachability.ConnectReachability;
184
185 // The first sighting schedules its own confirming check two minutes later.
186 expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });
187 yield* TestClock.adjust(Duration.minutes(1));
188 expect(yield* Queue.size(restarts)).toBe(0);
189 yield* TestClock.adjust(Duration.minutes(1));
190 expect(yield* Queue.take(restarts)).toBe(connected.pid);
191
192 // A restart that did not help doubles the wait.
193 yield* reachability.check;
194 yield* TestClock.adjust(Duration.minutes(2));
195 expect(yield* Queue.size(restarts)).toBe(0);
196 yield* TestClock.adjust(Duration.minutes(2));
197 expect(yield* Queue.take(restarts)).toBe(connected.pid);
198
199 // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.
200 status = 200;
201 expect(yield* reachability.check).toMatchObject({ status: "reachable" });
202 status = 530;
203 body = { cloudflareError: 1016 };
204 expect(yield* reachability.check).toMatchObject({ cause: "unexpected_response" });
205 yield* TestClock.adjust(Duration.minutes(2));
206 expect(yield* Queue.size(restarts)).toBe(0);
207
208 body = { cloudflareError: 1033 };
209 yield* reachability.check;
210 yield* TestClock.adjust(Duration.minutes(2));
211 expect(yield* Queue.take(restarts)).toBe(connected.pid);
212 }).pipe(Effect.provide(layer));
213 }),
214 );
215
216 it.effect("starts a new streak after a healthy check that a newer check overtook", () =>
217 Effect.gen(function* () {
218 using _connect = mockEdgeConnect();
219 const restartTimes = yield* Queue.unbounded<number>();
220 const healthyRequested = yield* Deferred.make<void>();
221 const healthyAnswer = yield* Deferred.make<void>();
222 let status = 530;
223 const layer = reachabilityLayer(
224 Effect.gen(function* () {
225 const served = status;
226 if (served === 200) {
227 yield* Deferred.succeed(healthyRequested, undefined);
228 yield* Deferred.await(healthyAnswer);
229 }
230 return { status: served, body: { cloudflareError: 1033 } };
231 }),
232 (_pid, stillDead) =>
233 stillDead.pipe(
234 Effect.tap((dead) =>
235 dead
236 ? Clock.currentTimeMillis.pipe(
237 Effect.flatMap((now) => Queue.offer(restartTimes, now)),
238 )
239 : Effect.void,
240 ),
241 ),
242 );
243 yield* Effect.gen(function* () {
244 const reachability = yield* ConnectReachability.ConnectReachability;
245 const startedAt = yield* Clock.currentTimeMillis;
246 yield* reachability.check;
247 yield* TestClock.adjust(Duration.minutes(1));
248
249 // The healthy answer arrives after a newer check already reported the tunnel dead.
250 status = 200;
251 const healthy = yield* Effect.forkChild(reachability.check);
252 yield* Deferred.await(healthyRequested);
253 status = 530;
254 yield* reachability.check;
255 yield* Deferred.succeed(healthyAnswer, undefined);
256 expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
257
258 // The grace period starts again at the next dead check, a minute after the first one.
259 yield* reachability.check;
260 yield* TestClock.adjust(Duration.minutes(2));
261 expect((yield* Queue.take(restartTimes)) - startedAt).toBe(
262 Duration.toMillis(Duration.minutes(3)),
263 );
264 }).pipe(Effect.provide(layer));
265 }),
266 );
267
268 it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () =>
269 Effect.gen(function* () {
270 using _connect = mockEdgeConnect();
271 const healthyRequested = yield* Deferred.make<void>();
272 const healthyAnswer = yield* Deferred.make<void>();
273 const restartRequested = yield* Deferred.make<void>();
274 const releaseRestart = yield* Deferred.make<void>();
275 const guardChecks = yield* Queue.unbounded<boolean>();
276 let status = 530;
277 const layer = reachabilityLayer(
278 Effect.gen(function* () {
279 const served = status;
280 if (served === 200) {
281 yield* Deferred.succeed(healthyRequested, undefined);
282 yield* Deferred.await(healthyAnswer);
283 }
284 return { status: served, body: { cloudflareError: 1033 } };
285 }),
286 (_pid, stillDead) =>
287 Effect.gen(function* () {
288 yield* Deferred.succeed(restartRequested, undefined);
289 yield* Deferred.await(releaseRestart);
290 const accepted = yield* stillDead;
291 yield* Queue.offer(guardChecks, accepted);
292 return accepted;
293 }),
294 );
295 yield* Effect.gen(function* () {
296 const reachability = yield* ConnectReachability.ConnectReachability;
297 yield* reachability.check;
298 yield* TestClock.adjust(Duration.seconds(119));
299
300 status = 200;
301 const healthy = yield* Effect.forkChild(reachability.check);
302 yield* Deferred.await(healthyRequested);
303 status = 530;
304 yield* TestClock.adjust(Duration.seconds(1));
305 yield* Deferred.await(restartRequested);
306
307 yield* Deferred.succeed(healthyAnswer, undefined);
308 expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
309 yield* Deferred.succeed(releaseRestart, undefined);
310 expect(yield* Queue.take(guardChecks)).toBe(false);
311
312 yield* reachability.check;
313 yield* TestClock.adjust(Duration.minutes(1));
314 expect(yield* Queue.size(guardChecks)).toBe(0);
315 yield* TestClock.adjust(Duration.minutes(1));
316 expect(yield* Queue.take(guardChecks)).toBe(true);
317 }).pipe(Effect.provide(layer));
318 }),
319 );
320
321 it.effect("drops the confirming check of a streak that already ended", () =>
322 Effect.gen(function* () {
323 using _connect = mockEdgeConnect();
324 const restarts = yield* Queue.unbounded<number>();
325 let status = 530;
326 let requests = 0;
327 const layer = reachabilityLayer(
328 Effect.sync(() => {
329 requests += 1;
330 return { status, body: { cloudflareError: 1033 } };
331 }),
332 (pid, stillDead) =>
333 stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
334 );
335 yield* Effect.gen(function* () {
336 const reachability = yield* ConnectReachability.ConnectReachability;
337 yield* reachability.check;
338 yield* TestClock.adjust(Duration.minutes(1));
339 status = 200;
340 yield* reachability.check;
341 status = 530;
342 yield* reachability.check;
343
344 const before = requests;
345 yield* TestClock.adjust(Duration.minutes(1));
346 expect(requests).toBe(before);
347 yield* TestClock.adjust(Duration.minutes(1));
348 expect(yield* Queue.take(restarts)).toBe(connected.pid);
349 }).pipe(Effect.provide(layer));
350 }),
351 );
352
353 it.effect("keeps a 530 whose error body never finishes", () =>
354 Effect.gen(function* () {
355 using _connect = mockEdgeConnect();
356 let markBodyRead = () => {};
357 const bodyRead = new Promise<void>((resolve) => {
358 markBodyRead = resolve;
359 });
360 const layer = reachabilityLayer(
361 Effect.sync(() => ({
362 status: 530,
363 body: new ReadableStream({
364 pull: () => {
365 markBodyRead();
366 return new Promise<void>(() => {});
367 },
368 }),
369 })),
370 () => Effect.succeed(false),
371 );
372 yield* Effect.gen(function* () {
373 const reachability = yield* ConnectReachability.ConnectReachability;
374 const check = yield* Effect.forkChild(reachability.check);
375 yield* Effect.promise(() => bodyRead);
376 yield* TestClock.adjust(Duration.seconds(2));
377 expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" });
378 }).pipe(Effect.provide(layer));
379 }),
380 );
381});
382
383describe("ConnectReachability DNS failures", () => {
384 // undici wraps getaddrinfo's error in `TypeError: fetch failed`. glibc behind
385 // systemd-resolved can report an NXDOMAIN as EBUSY rather than ENOTFOUND.
386 const lookupFailed = (code: string) =>
387 Effect.fail(
388 new HttpClientError.HttpClientError({
389 reason: new HttpClientError.TransportError({
390 request: HttpClientRequest.get("https://env.example.test"),
391 cause: new TypeError("fetch failed", {
392 cause: Object.assign(new Error(`getaddrinfo ${code} env.example.test`), {
393 code,
394 syscall: "getaddrinfo",
395 hostname: "env.example.test",
396 }),
397 }),
398 }),
399 }),
400 );
401
402 it.effect("reports a getaddrinfo EBUSY as a DNS failure", () =>
403 Effect.gen(function* () {
404 using _connect = mockEdgeConnect();
405 const layer = reachabilityLayer(lookupFailed("EBUSY"), () => Effect.succeed(false));
406 const result = yield* ConnectReachability.ConnectReachability.use(
407 (reachability) => reachability.check,
408 ).pipe(Effect.provide(layer));
409 expect(result).toMatchObject({ status: "unreachable", cause: "dns_failed" });
410 }),
411 );
412
413 // A local "not found" cached before the relay published the record looks like a missing
414 // record. Public DNS tells them apart.
415 it.effect("says whether the name exists in public DNS", () =>
416 Effect.gen(function* () {
417 using _connect = mockEdgeConnect();
418 for (const [lookupCode, dnsStatus, reason] of [
419 ["ENOTFOUND", 0, "old cached answer"],
420 ["ENOTFOUND", 3, "missing from public DNS"],
421 // The local resolver did not answer, so nothing it cached is to blame.
422 ["EAI_AGAIN", 0, "does not resolve from this machine"],
423 ] as const) {
424 const layer = reachabilityLayer(
425 (request) =>
426 request.url.startsWith("https://1.1.1.1/dns-query")
427 ? Effect.succeed({ status: 200, body: JSON.stringify({ Status: dnsStatus }) })
428 : lookupFailed(lookupCode),
429 () => Effect.succeed(false),
430 );
431 const result = yield* ConnectReachability.ConnectReachability.use(
432 (reachability) => reachability.check,
433 ).pipe(Effect.provide(layer));
434 expect(result).toMatchObject({
435 cause: "dns_failed",
436 reason: expect.stringContaining(reason),
437 });
438 }
439 }),
440 );
441});
442
443describe("T3 Connect troubleshooting links", () => {
444 it("each section names a heading in the user guide", () => {
445 const guide = NodeFS.readFileSync(
446 new URL("../../../../docs/user/t3-connect-troubleshooting.md", import.meta.url),
447 "utf8",
448 );
449 // GitHub's heading anchors: lowercase, drop punctuation, spaces become hyphens.
450 const anchors = new Set(
451 [...guide.matchAll(/^#{2,3} (.+)$/gm)].map(([, heading]) =>
452 heading!
453 .toLowerCase()
454 .replace(/[^\p{L}\p{N} _-]/gu, "")
455 .replace(/ /g, "-"),
456 ),
457 );
458 for (const section of T3_CONNECT_TROUBLESHOOTING_SECTIONS) {
459 expect(anchors, section).toContain(section);
460 }
461 });
462});