Open History to see the last commit
1import * as NodeNet from "node:net";
2
3import {
4 connectReachabilityHelpUrl,
5 type EnvironmentConnectReachability,
6 formatConnectReachable,
7} from "@t3tools/contracts";
8import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";
9import * as Clock from "effect/Clock";
10import * as Context from "effect/Context";
11import * as DateTime from "effect/DateTime";
12import * as Duration from "effect/Duration";
13import * as Effect from "effect/Effect";
14import * as Layer from "effect/Layer";
15import * as Option from "effect/Option";
16import * as Ref from "effect/Ref";
17import * as Scope from "effect/Scope";
18import * as Stream from "effect/Stream";
19import * as SynchronizedRef from "effect/SynchronizedRef";
20import * as Schema from "effect/Schema";
21import { HttpClient, HttpClientResponse } from "effect/http";
22import { ChildProcessSpawner } from "effect/process";
23
24import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
25import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts";
26import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
27
28// cloudflared finds the edge through the SRV record `_v2-origintunneld._tcp.argotunnel.com`,
29// which names these two hosts on port 7844. QUIC and HTTP/2 both use that port.
30const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.com"];
31const TUNNEL_EDGE_PORT = 7844;
32const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);
33const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);
34const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);
35// An IP literal, so the answer does not come from this machine's DNS, over HTTPS because
36// some networks intercept port 53.
37const PUBLIC_DNS_OVER_HTTPS = "https://1.1.1.1/dns-query";
38const PUBLIC_DNS_TIMEOUT = Duration.seconds(5);
39// cloudflared registers several connections within a few seconds; they are one change.
40const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);
41// Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).
42const RECHECK_INTERVAL = Duration.minutes(5);
43// cloudflared can keep reporting ready connections the edge already dropped, so its
44// own readiness watch never restarts it. A 530 can also be brief, so restart only once it
45// lasts, and wait longer after each restart that did not help.
46const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);
47const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);
48
49export type EndpointProbe =
50 | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }
51 | {
52 readonly kind: "dns_failed";
53 /** Whether public DNS has the name; absent when it could not be asked. */
54 readonly existsPublicly?: boolean;
55 }
56 | { readonly kind: "timeout" | "network_error" };
57
58/** Everything one check learned. `null` means that step did not run. */
59export interface ReachabilityFacts {
60 readonly managedTunnelConfigured: boolean;
61 readonly connector: ManagedEndpointRuntime.ConnectorFacts | null;
62 readonly endpoint: EndpointProbe | null;
63 readonly edgeReachable: boolean | null;
64 readonly exitNode: TailscaleExitNode | null;
65}
66
67type Unreachable = Extract<EnvironmentConnectReachability, { status: "unreachable" }>;
68
69function exitNodeHint(exitNode: TailscaleExitNode | null): string | null {
70 if (!exitNode) return null;
71 const name = exitNode.name ? ` "${exitNode.name}"` : "";
72 return `Tailscale exit node${name} is on. Traffic to Cloudflare goes through it.`;
73}
74
75/** Turns what one check learned into the answer every surface shows. */
76export function classifyReachability(
77 facts: ReachabilityFacts,
78 checkedAt: string,
79): EnvironmentConnectReachability {
80 const { connector, endpoint, exitNode } = facts;
81 const fail = (
82 cause: Unreachable["cause"],
83 reason: string,
84 fix: string,
85 hint: string | null = null,
86 ): Unreachable => ({ status: "unreachable", checkedAt, cause, reason, hint, fix });
87 const restart = "Restart T3 Code. If it stays, run `t3 connect` again.";
88
89 if (!facts.managedTunnelConfigured) {
90 // A running server reads the link `t3 connect` stores only at startup.
91 return fail(
92 "not_linked",
93 "The server has not set up a T3 Connect tunnel.",
94 "Run `t3 connect`, then restart T3 Code so it applies the link.",
95 );
96 }
97 if (connector === null) {
98 return fail(
99 "connector_not_running",
100 "The tunnel client (cloudflared) is not running.",
101 "Restart T3 Code. If it stays down, check the server log for relay client errors.",
102 );
103 }
104 const status = endpoint?.kind === "status" ? endpoint.status : null;
105 if (status === 200) {
106 return {
107 status: "reachable",
108 checkedAt,
109 readyConnections: connector.readyConnections,
110 protocol: connector.protocol,
111 };
112 }
113 // The edge test is TCP only. With QUIC connections up, a 502 from a working tunnel, or
114 // DNS already failing, a failed TCP test would hide the real cause.
115 if (
116 facts.edgeReachable === false &&
117 status !== 502 &&
118 !(connector.readyConnections && connector.readyConnections > 0) &&
119 endpoint?.kind !== "dns_failed"
120 ) {
121 return fail(
122 "edge_unreachable",
123 `Cloudflare's tunnel port (${TUNNEL_EDGE_PORT}) times out from this machine.`,
124 exitNode
125 ? `tailscale set --exit-node= (or allow outbound ${TUNNEL_EDGE_PORT} on that network)`
126 : `Allow outbound TCP and UDP port ${TUNNEL_EDGE_PORT}, or turn off the VPN or firewall that blocks it.`,
127 exitNodeHint(exitNode) ?? "A VPN, proxy, or firewall may block it.",
128 );
129 }
130 if (status === 502) {
131 return fail(
132 "origin_unreachable",
133 "The tunnel is up, but it cannot reach the T3 server on this machine.",
134 "Restart T3 Code so the tunnel uses the port the server listens on.",
135 );
136 }
137 if (endpoint === null || connector.readyConnections === 0) {
138 return fail(
139 "connector_not_ready",
140 "cloudflared has not connected to Cloudflare yet.",
141 "Wait a minute. If it stays, check the network and the server log.",
142 );
143 }
144 const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;
145 // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other
146 // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.
147 if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {
148 return fail(
149 "no_connector",
150 "Cloudflare has no live connection from this machine for the tunnel.",
151 restart,
152 );
153 }
154 if (status !== null) {
155 const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;
156 return fail(
157 "unexpected_response",
158 `The T3 Connect address answered with HTTP ${status}${code}.`,
159 restart,
160 );
161 }
162 if (endpoint.kind === "dns_failed") {
163 // A lookup made before the relay published the record leaves a cached "not found"
164 // for the zone's negative TTL (30 minutes).
165 if (endpoint.existsPublicly === true) {
166 return fail(
167 "dns_failed",
168 "The T3 Connect address resolves in public DNS, but this network's DNS still has an old cached answer that it does not exist.",
169 "Wait. The cached answer usually clears within about 30 minutes.",
170 exitNodeHint(exitNode),
171 );
172 }
173 if (endpoint.existsPublicly === false) {
174 return fail(
175 "dns_failed",
176 "The T3 Connect address is missing from public DNS. The relay did not publish it.",
177 restart,
178 );
179 }
180 return fail(
181 "dns_failed",
182 "The T3 Connect address does not resolve from this machine.",
183 "Check the DNS settings on this machine or its VPN.",
184 exitNodeHint(exitNode),
185 );
186 }
187 return fail(
188 "timeout",
189 "The T3 Connect address did not answer from this machine.",
190 "Check the network, VPN, or proxy on this machine.",
191 exitNodeHint(exitNode),
192 );
193}
194
195/** The loud diagnosis the CLI prints and the server logs. */
196export function formatReachabilityFailure(result: Unreachable): string {
197 return [
198 "✕ T3 Connect is NOT reachable from other devices",
199 ` ${result.reason}`,
200 ...(result.hint ? [` ${result.hint}`] : []),
201 ` Fix: ${result.fix}`,
202 ` More info: ${connectReachabilityHelpUrl(result.cause)}`,
203 ].join("\n");
204}
205
206/** The code of a failed name lookup, or null. getaddrinfo reports more than the DNS codes, such as EBUSY. */
207function dnsLookupErrorCode(error: unknown): string | null {
208 let current = error;
209 for (let depth = 0; depth < 6 && typeof current === "object" && current !== null; depth++) {
210 const code = "code" in current && typeof current.code === "string" ? current.code : "";
211 if (("syscall" in current && current.syscall === "getaddrinfo") || DNS_ERROR_CODES.has(code)) {
212 return code;
213 }
214 current = "cause" in current ? current.cause : undefined;
215 }
216 return null;
217}
218
219const DNS_ERROR_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "EAI_NONAME"]);
220// Answers that the name does not exist, which a resolver may have cached. EAI_AGAIN means
221// the resolver did not answer. glibc behind systemd-resolved can report NXDOMAIN as EBUSY.
222const NOT_FOUND_CODES = new Set(["ENOTFOUND", "EAI_NONAME", "EBUSY"]);
223
224const DnsJsonAnswer = Schema.Struct({ Status: Schema.Number });
225
226const canOpenTcp = (host: string, port: number) =>
227 Effect.callback<boolean>((resume) => {
228 const socket = NodeNet.connect({ host, port });
229 const finish = (connected: boolean) => {
230 socket.destroy();
231 resume(Effect.succeed(connected));
232 };
233 socket.setTimeout(Duration.toMillis(EDGE_CONNECT_TIMEOUT), () => finish(false));
234 socket.once("connect", () => finish(true));
235 socket.once("error", () => finish(false));
236 return Effect.sync(() => socket.destroy());
237 });
238
239export class ConnectReachability extends Context.Service<
240 ConnectReachability,
241 {
242 /** Runs a check now. The server also runs one when the tunnel's readiness changes. */
243 readonly check: Effect.Effect<EnvironmentConnectReachability>;
244 readonly latest: Effect.Effect<Option.Option<EnvironmentConnectReachability>>;
245 }
246>()("t3/cloud/ConnectReachability") {}
247
248const make = Effect.gen(function* () {
249 const runtime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime;
250 const secrets = yield* ServerSecretStore.ServerSecretStore;
251 const httpClient = yield* HttpClient.HttpClient;
252 // `order` ranks checks by start, so a slow older one cannot replace a newer result.
253 const latestRef = yield* Ref.make(
254 Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
255 );
256 const checkCounter = yield* Ref.make(0);
257 // Checks update this as they finish, even when a newer check already did. A dead streak
258 // or a restart backoff that began before a healthy check no longer counts.
259 const lastHealthyOrder = yield* Ref.make(0);
260 const deadConnectorRef = yield* SynchronizedRef.make<{
261 readonly streak: {
262 readonly pid: number;
263 readonly since: number;
264 readonly order: number;
265 } | null;
266 readonly restartAfterMillis: number;
267 readonly restartOrder: number;
268 }>({
269 streak: null,
270 restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
271 restartOrder: 0,
272 });
273 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
274 const scope = yield* Scope.Scope;
275 const readExitNode = readTailscaleExitNode.pipe(
276 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
277 Effect.orElseSucceed(() => null),
278 );
279
280 // DNS JSON status 0 is NOERROR (the name exists, even with no A record) and 3 is NXDOMAIN.
281 const existsInPublicDns = Effect.fn("ConnectReachability.existsInPublicDns")(
282 function* (hostname: string) {
283 const response = yield* httpClient.get(PUBLIC_DNS_OVER_HTTPS, {
284 urlParams: { name: hostname, type: "A" },
285 headers: { accept: "application/dns-json" },
286 });
287 const { Status } = yield* HttpClientResponse.schemaBodyJson(DnsJsonAnswer)(response);
288 return Status === 0 ? true : Status === 3 ? false : null;
289 },
290 Effect.scoped,
291 Effect.timeoutOption(PUBLIC_DNS_TIMEOUT),
292 Effect.map(Option.getOrNull),
293 Effect.orElseSucceed(() => null),
294 );
295
296 const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(
297 function* (hostname: string) {
298 const response = yield* httpClient
299 .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })
300 .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));
301 if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;
302 const { status } = response.value;
303 if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;
304 // Cloudflare's error page is HTML unless the client accepts JSON, which carries
305 // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.
306 const body = yield* response.value.text.pipe(
307 Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),
308 Effect.orElseSucceed(() => Option.none<string>()),
309 );
310 const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];
311 return (
312 code === undefined
313 ? { kind: "status", status }
314 : { kind: "status", status, cloudflareError: code }
315 ) satisfies EndpointProbe;
316 },
317 Effect.scoped,
318 (effect, hostname) =>
319 effect.pipe(
320 Effect.catch((error) => {
321 const code = dnsLookupErrorCode(error);
322 if (code === null) return Effect.succeed<EndpointProbe>({ kind: "network_error" });
323 if (!NOT_FOUND_CODES.has(code))
324 return Effect.succeed<EndpointProbe>({ kind: "dns_failed" });
325 return existsInPublicDns(hostname).pipe(
326 Effect.map((existsPublicly): EndpointProbe =>
327 existsPublicly === null
328 ? { kind: "dns_failed" }
329 : { kind: "dns_failed", existsPublicly },
330 ),
331 );
332 }),
333 ),
334 );
335
336 const gatherFacts = Effect.gen(function* () {
337 const managedTunnelConfigured = Option.isSome(
338 yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG).pipe(Effect.orElseSucceed(Option.none)),
339 );
340 const connector = managedTunnelConfigured ? yield* runtime.inspectConnector : null;
341 const endpoint =
342 connector?.hostname && connector.readyConnections !== 0
343 ? yield* probeEndpoint(connector.hostname)
344 : null;
345 const facts: ReachabilityFacts = {
346 managedTunnelConfigured,
347 connector,
348 endpoint,
349 edgeReachable: null,
350 exitNode: null,
351 };
352 if (connector === null || (endpoint?.kind === "status" && endpoint.status === 200)) {
353 return facts;
354 }
355 const [edgeResults, exitNode] = yield* Effect.all(
356 [
357 Effect.forEach(TUNNEL_EDGE_HOSTS, (host) => canOpenTcp(host, TUNNEL_EDGE_PORT), {
358 concurrency: "unbounded",
359 }),
360 readExitNode,
361 ],
362 { concurrency: 2 },
363 );
364 return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
365 });
366
367 const isLatest = (order: number) =>
368 Ref.get(latestRef).pipe(
369 Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),
370 );
371
372 // Runs inside the check span; the runtime restart has its own. Returns the streak a dead
373 // connector just started and when to confirm it.
374 const restartDeadConnector = Effect.fnUntraced(function* (
375 facts: ReachabilityFacts,
376 result: EnvironmentConnectReachability,
377 order: number,
378 ) {
379 const now = yield* Clock.currentTimeMillis;
380 const { connector } = facts;
381 const dead =
382 result.status === "unreachable" &&
383 result.cause === "no_connector" &&
384 connector !== null &&
385 (connector.readyConnections ?? 0) > 0
386 ? connector
387 : null;
388 return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>
389 Effect.gen(function* () {
390 const healthy = yield* Ref.get(lastHealthyOrder);
391 const state = {
392 streak: current.streak && current.streak.order > healthy ? current.streak : null,
393 restartAfterMillis:
394 current.restartOrder > healthy
395 ? current.restartAfterMillis
396 : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
397 restartOrder: current.restartOrder,
398 };
399 if (!(yield* isLatest(order))) return [null, state] as const;
400 if (dead === null) return [null, { ...state, streak: null }] as const;
401 if (state.streak?.pid !== dead.pid) {
402 const started = { ...state, streak: { pid: dead.pid, since: now, order } };
403 return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;
404 }
405 if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;
406 // Checks can invalidate the streak while this request waits for the runtime's lock.
407 const streakOrder = state.streak.order;
408 const stillDead = Effect.gen(function* () {
409 if (!(yield* isLatest(order))) return false;
410 return (yield* Ref.get(lastHealthyOrder)) < streakOrder;
411 });
412 if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {
413 return [null, state] as const;
414 }
415 const restarted = {
416 streak: null,
417 restartAfterMillis: Math.min(
418 state.restartAfterMillis * 2,
419 Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
420 ),
421 restartOrder: order,
422 };
423 return [null, restarted] as const;
424 }),
425 );
426 });
427
428 const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {
429 const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);
430 const facts = yield* gatherFacts;
431 const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));
432 if (result.status === "reachable") {
433 yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));
434 }
435 const previous = yield* Ref.modify(latestRef, (current) =>
436 Option.isSome(current) && current.value.order > order
437 ? [null, current]
438 : [current, Option.some({ order, result })],
439 );
440 if (previous === null) return result;
441 const started = yield* restartDeadConnector(facts, result, order);
442 // Nothing else would look again until the next periodic check.
443 if (started !== null) {
444 const confirm = SynchronizedRef.get(deadConnectorRef).pipe(
445 Effect.flatMap(({ streak }) =>
446 streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,
447 ),
448 );
449 yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));
450 }
451 const changed = Option.match(previous, {
452 onNone: () => true,
453 onSome: ({ result: last }) =>
454 last.status !== result.status ||
455 (last.status === "unreachable" &&
456 result.status === "unreachable" &&
457 last.cause !== result.cause),
458 });
459 if (!changed || (result.status === "unreachable" && result.cause === "not_linked")) {
460 return result;
461 }
462 if (result.status === "unreachable") {
463 // A connector that has not connected yet is usually still starting; the
464 // runtime restarts it, and an unreachable edge is reported as its own cause.
465 const log = result.cause === "connector_not_ready" ? Effect.logWarning : Effect.logError;
466 yield* log(`\n${formatReachabilityFailure(result)}`).pipe(
467 Effect.annotateLogs({ cause: result.cause }),
468 );
469 } else {
470 yield* Effect.logInfo(
471 `T3 Connect is reachable from other devices. ${formatConnectReachable(result)}`,
472 );
473 }
474 return result;
475 }).pipe(Effect.withSpan("ConnectReachability.check"));
476
477 yield* runtime.readinessChanged.pipe(
478 Stream.debounce(READINESS_CHANGE_DEBOUNCE),
479 Stream.runForEach(() => check),
480 Effect.forkScoped,
481 );
482 yield* Effect.sleep(RECHECK_INTERVAL).pipe(
483 Effect.andThen(check),
484 Effect.forever,
485 Effect.forkScoped,
486 );
487
488 return ConnectReachability.of({
489 check,
490 latest: Ref.get(latestRef).pipe(Effect.map(Option.map(({ result }) => result))),
491 });
492});
493
494export const layer = Layer.effect(ConnectReachability, make);