1// @effect-diagnostics nodeBuiltinImport:off - Runs the generated runner in a real shell.
2import * as NodeChildProcess from "node:child_process";
3import * as NodeFS from "node:fs";
4import * as NodeOS from "node:os";
5import * as NodePath from "node:path";
6
7import { assert, describe, it } from "@effect/vitest";
8import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
9import * as NodeServices from "@effect/platform-node/NodeServices";
10import * as HostProcess from "@t3tools/shared/HostProcess";
11import * as NetService from "@t3tools/shared/Net";
12import * as Clock from "effect/Clock";
13import * as Deferred from "effect/Deferred";
14import * as Duration from "effect/Duration";
15import * as Effect from "effect/Effect";
16import * as Fiber from "effect/Fiber";
17import * as FileSystem from "effect/FileSystem";
18import * as Layer from "effect/Layer";
19import * as Option from "effect/Option";
20import * as Path from "effect/Path";
21import * as PlatformError from "effect/PlatformError";
22import * as Schema from "effect/Schema";
23import * as Result from "effect/Result";
24import * as Sink from "effect/Sink";
25import * as Stream from "effect/Stream";
26import { TestClock } from "effect/testing";
27import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
28import { ChildProcess, ChildProcessSpawner } from "effect/process";
29
30import { captureLauncher, captureLauncherFor, runLauncher } from "./launcherTestHarness.ts";
31import * as SshAuth from "./auth.ts";
32import { remoteStateKey } from "./command.ts";
33import { SshCommandError, SshPasswordPromptError, SshReadinessError } from "./errors.ts";
34import * as SshTunnel from "./tunnel.ts";
35
36const isSshCommandError = Schema.is(SshCommandError);
37
38const makeSuccessfulProcess = (stdout: string) => {
39 const stdoutStream = Stream.make(new TextEncoder().encode(stdout));
40 return ChildProcessSpawner.makeHandle({
41 pid: ChildProcessSpawner.ProcessId(123),
42 stdout: stdoutStream,
43 stderr: Stream.empty,
44 all: stdoutStream,
45 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)),
46 isRunning: Effect.succeed(false),
47 kill: () => Effect.void,
48 stdin: Sink.drain,
49 getInputFd: () => Sink.drain,
50 getOutputFd: () => Stream.empty,
51 unref: Effect.succeed(Effect.void),
52 });
53};
54
55const makeRunningProcess = (onKill: () => void) => {
56 let finish: ((exitCode: ChildProcessSpawner.ExitCode) => void) | null = null;
57 return ChildProcessSpawner.makeHandle({
58 pid: ChildProcessSpawner.ProcessId(123),
59 stdout: Stream.empty,
60 stderr: Stream.empty,
61 all: Stream.empty,
62 exitCode: Effect.callback<ChildProcessSpawner.ExitCode>((resume) => {
63 finish = (exitCode) => resume(Effect.succeed(exitCode));
64 return Effect.sync(() => {
65 finish = null;
66 });
67 }),
68 isRunning: Effect.succeed(true),
69 kill: () =>
70 Effect.sync(() => {
71 onKill();
72 finish?.(ChildProcessSpawner.ExitCode(143));
73 }),
74 stdin: Sink.drain,
75 getInputFd: () => Sink.drain,
76 getOutputFd: () => Stream.empty,
77 unref: Effect.succeed(Effect.void),
78 });
79};
80
81const testHttpClient = HttpClient.make((request) =>
82 Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status: 200 }))),
83);
84
85const hangingHttpClient = HttpClient.make(() => Effect.never);
86
87const testNetService = NetService.NetService.of({
88 canListenOnHost: () => Effect.succeed(true),
89 isPortAvailableOnLoopback: () => Effect.succeed(true),
90 hasListenerOnHost: () => Effect.succeed(true),
91 reserveLoopbackPort: () => Effect.succeed(41_773),
92 findAvailablePort: (preferred) => Effect.succeed(preferred),
93});
94
95function commandArgs(command: ChildProcess.Command): ReadonlyArray<string> {
96 return command._tag === "StandardCommand" ? command.args.map(String) : [];
97}
98
99const ARCHIVE = { archiveVersion: "1.2.3-preview.20260911.4" } as const;
100
101// Emulates `t3 serve`: it takes the home owner lock at startup, then binds
102// once a second server has tried the same home (or after 2s), so a concurrent
103// launch always sees the home still unclaimed. Like the real server, it
104// publishes server-runtime.json after it starts answering HTTP, if at all.
105const fakeT3Serve = (publishDelayMs: number | null) => `const fs = require("node:fs");
106const http = require("node:http");
107const path = require("node:path");
108const args = process.argv.slice(2);
109const port = Number(args[args.indexOf("--port") + 1]);
110const userdata = path.join(args[args.indexOf("--base-dir") + 1], "userdata");
111const servers = path.join(process.env.HOME, "servers");
112fs.mkdirSync(userdata, { recursive: true });
113fs.appendFileSync(servers, process.pid + "\\n");
114try {
115 fs.writeFileSync(path.join(userdata, "server.lock"), String(process.pid), { flag: "wx" });
116} catch {
117 console.error("ServerAlreadyRunningError: another server owns " + userdata);
118 process.exit(1);
119}
120setTimeout(() => process.exit(0), 60_000);
121const serve = (tries) => {
122 if (tries > 0 && fs.readFileSync(servers, "utf8").trim().split("\\n").length < 2) {
123 setTimeout(() => serve(tries - 1), 50);
124 return;
125 }
126 http.createServer((_, response) => response.end()).listen(port, "127.0.0.1", () => {
127 const origin = "http://127.0.0.1:" + port;
128 ${publishDelayMs === null ? "return;" : ""}
129 setTimeout(() => {
130 fs.writeFileSync(path.join(userdata, "server-runtime.json"), JSON.stringify({ pid: process.pid, port, origin }));
131 }, ${publishDelayMs});
132 });
133};
134serve(40);
135`;
136
137// The launch's POSIX bootstrap without the account login shell around it, so the
138// test machine's login profile cannot move HOME.
139const launchStateKey = (command: ChildProcess.Command) =>
140 /exec sh -s -- (\S+)"/u.exec(commandArgs(command).join(" "))?.[1];
141
142// Runs the remote half of an SSH launch as a local shell
143// whose HOME is the given directory. Every launch after the first starts once
144// a server is running (or after 10s), which lines up the race deterministically.
145const localRemoteShellLayer = (home: string) =>
146 Layer.effect(
147 ChildProcessSpawner.ChildProcessSpawner,
148 Effect.gen(function* () {
149 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
150 let launches = 0;
151 return ChildProcessSpawner.make((command) => {
152 if (command._tag !== "StandardCommand") return spawner.spawn(command);
153 const gate =
154 launches++ === 0
155 ? ""
156 : 'i=0; while [ ! -s "$HOME/servers" ] && [ "$i" -lt 200 ]; do sleep 0.05; i=$((i+1)); done; ';
157 return spawner.spawn(
158 ChildProcess.make(
159 "sh",
160 ["-c", `${gate}exec sh -s -- "$1"`, "sh", launchStateKey(command) ?? ""],
161 {
162 ...command.options,
163 env: { ...command.options.env, HOME: home },
164 // Like an SSH session ending, the shell's exit leaves its nohup'd server running.
165 detached: false,
166 },
167 ),
168 );
169 });
170 }),
171 );
172
173describe("ssh tunnel scripts", () => {
174 it.live("distinguishes an incompatible remote Node from a missing executable", () =>
175 Effect.gen(function* () {
176 const fs = yield* FileSystem.FileSystem;
177 const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-node-" });
178 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
179 for (const missing of [false, true]) {
180 const nodeFixture = missing
181 ? "command() { return 1; }; "
182 : 'node() { cat >/dev/null; printf "Remote node 18.20.8 does not satisfy required range >=999.0.0.\\n" >&2; return 1; }; ';
183 const remoteShell = ChildProcessSpawner.make((command) => {
184 if (command._tag !== "StandardCommand") return spawner.spawn(command);
185 return spawner.spawn(
186 ChildProcess.make(
187 "sh",
188 ["-c", `${nodeFixture}eval "$(cat)"`, "sh", ...commandArgs(command).slice(-1)],
189 {
190 ...command.options,
191 env: {
192 ...command.options.env,
193 HOME: home,
194 NVM_DIR: NodePath.join(home, ".nvm"),
195 FNM_DIR: NodePath.join(home, ".fnm"),
196 VOLTA_HOME: NodePath.join(home, ".volta"),
197 },
198 },
199 ),
200 );
201 });
202 const result = yield* SshTunnel.launchOrReuseRemoteServer(
203 { alias: "devbox", hostname: "devbox", username: null, port: null },
204 undefined,
205 { nodeScriptPath: "/unused/server.ts", nodeEngineRange: ">=999.0.0" },
206 ).pipe(
207 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, remoteShell),
208 Effect.result,
209 );
210 assert.isTrue(Result.isFailure(result));
211 if (!Result.isFailure(result)) return;
212 assert.isTrue(isSshCommandError(result.failure));
213 if (!isSshCommandError(result.failure)) return;
214 assert.equal(result.failure.exitCode, 1);
215 assert.equal(result.failure.stderr.includes("missing node on PATH"), missing);
216 assert.equal(
217 result.failure.stderr.includes("does not satisfy required range >=999.0.0"),
218 !missing,
219 );
220 }
221 }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
222 );
223
224 it("rejects archive versions that are not a single exact version segment", () => {
225 for (const archiveVersion of [
226 "../other",
227 "1.2.3/evil",
228 "1.2.3\\evil",
229 "1.2.3-preview.1 x",
230 "1.2.3-preview.1\nrm -rf /",
231 "v1.2.3",
232 ]) {
233 assert.throws(
234 () => SshTunnel.buildRemoteT3RunnerScript({ archiveVersion }),
235 SshTunnel.SshInvalidArchiveVersionError,
236 undefined,
237 archiveVersion,
238 );
239 }
240 assert.include(
241 SshTunnel.buildRemoteT3RunnerScript(ARCHIVE),
242 "T3_ARCHIVE_VERSION='1.2.3-preview.20260911.4'",
243 );
244 });
245
246 it.each([
247 { host: "Darwin x86_64", system: "Darwin", machine: "x86_64", downloads: false },
248 { host: "Darwin arm64", system: "Darwin", machine: "arm64", downloads: true },
249 ])(
250 "only downloads a release archive the host can run: $host",
251 ({ system, machine, downloads }) => {
252 const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-ssh-runner-"));
253 try {
254 const bin = NodePath.join(root, "bin");
255 NodeFS.mkdirSync(bin);
256 NodeFS.mkdirSync(NodePath.join(root, "home"));
257 NodeFS.writeFileSync(
258 NodePath.join(bin, "uname"),
259 `#!/bin/sh\ncase "$1" in -s) echo ${system} ;; -m) echo ${machine} ;; esac\n`,
260 { mode: 0o755 },
261 );
262 NodeFS.writeFileSync(
263 NodePath.join(bin, "curl"),
264 "#!/bin/sh\necho 'download attempt' >&2\necho 'curl: (6) Could not resolve host: github.com' >&2\nexit 6\n",
265 { mode: 0o755 },
266 );
267 const runner = NodePath.join(root, "run-t3.sh");
268 NodeFS.writeFileSync(runner, SshTunnel.buildRemoteT3RunnerScript(ARCHIVE));
269 const result = NodeChildProcess.spawnSync("sh", [runner, "--version"], {
270 encoding: "utf8",
271 env: { HOME: NodePath.join(root, "home"), PATH: `${bin}:${process.env.PATH ?? ""}` },
272 });
273 assert.notEqual(result.status, 0);
274 if (downloads) {
275 assert.include(result.stderr, "download attempt");
276 // A bare curl error does not tell the user that t3 itself could not be downloaded.
277 assert.include(result.stderr, ARCHIVE.archiveVersion);
278 assert.include(result.stderr, "https://github.com/pingdotgg/t3code/releases/download");
279 } else {
280 assert.notInclude(result.stderr, "download attempt");
281 assert.include(result.stderr, "has no t3 release archive.");
282 }
283 } finally {
284 NodeFS.rmSync(root, { recursive: true, force: true });
285 }
286 },
287 );
288
289 it("names gzip when a host with tar but no gzip cannot unpack the release archive", () => {
290 const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-ssh-runner-"));
291 try {
292 const bin = NodePath.join(root, "bin");
293 const release = NodePath.join(root, "release");
294 const archive = `t3-${ARCHIVE.archiveVersion}-linux-x64.tar.gz`;
295 NodeFS.mkdirSync(bin);
296 NodeFS.mkdirSync(NodePath.join(root, "home"));
297 NodeFS.mkdirSync(NodePath.join(release, "t3"), { recursive: true });
298 NodeFS.writeFileSync(NodePath.join(release, "t3", "t3"), "#!/bin/sh\necho ok\n", {
299 mode: 0o755,
300 });
301 NodeChildProcess.execFileSync("tar", ["-czf", archive, "t3"], { cwd: release });
302 NodeChildProcess.execFileSync("sh", ["-c", `sha256sum ${archive} > SHA256SUMS`], {
303 cwd: release,
304 });
305 for (const tool of [
306 "sh",
307 "mkdir",
308 "mktemp",
309 "cat",
310 "rm",
311 "grep",
312 "cut",
313 "sha256sum",
314 "tar",
315 "mv",
316 ]) {
317 NodeFS.symlinkSync(
318 NodeChildProcess.execFileSync("sh", ["-c", `command -v ${tool}`], {
319 encoding: "utf8",
320 }).trim(),
321 NodePath.join(bin, tool),
322 );
323 }
324 NodeFS.writeFileSync(
325 NodePath.join(bin, "uname"),
326 '#!/bin/sh\ncase "$1" in -s) echo Linux ;; -m) echo x86_64 ;; esac\n',
327 { mode: 0o755 },
328 );
329 NodeFS.writeFileSync(
330 NodePath.join(bin, "curl"),
331 `#!/bin/sh\nfor last; do :; done\nfor a; do case "$a" in http*) src="${release}/\${a##*/}" ;; esac; done\nexec cat "$src" > "$last"\n`,
332 { mode: 0o755 },
333 );
334 const runner = NodePath.join(root, "run-t3.sh");
335 NodeFS.writeFileSync(runner, SshTunnel.buildRemoteT3RunnerScript(ARCHIVE));
336 const result = NodeChildProcess.spawnSync(NodePath.join(bin, "sh"), [runner, "--version"], {
337 encoding: "utf8",
338 env: { HOME: NodePath.join(root, "home"), PATH: bin },
339 });
340 assert.notEqual(result.status, 0);
341 assert.match(result.stderr, /(missing|needs) gzip/);
342 assert.notInclude(result.stderr, "Cannot exec");
343 } finally {
344 NodeFS.rmSync(root, { recursive: true, force: true });
345 }
346 });
347
348 it.effect("accepts launch JSON after remote shell startup noise", () => {
349 const target = {
350 alias: "devbox",
351 hostname: "devbox.example.com",
352 username: "julius",
353 port: 2222,
354 } as const;
355 const spawner = ChildProcessSpawner.make(() =>
356 Effect.succeed(makeSuccessfulProcess('loaded nvm default\n{"remotePort":3774}\n')),
357 );
358 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
359 const layerProcess = Layer.merge(
360 Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
361 layerSpawner,
362 );
363
364 return Effect.gen(function* () {
365 const result = yield* SshTunnel.launchOrReuseRemoteServer(target, undefined, ARCHIVE);
366 assert.equal(result.remotePort, 3774);
367 }).pipe(Effect.provide(layerProcess));
368 });
369
370 it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
371 "loads login settings without changing the POSIX bootstrap or its state key",
372 () =>
373 Effect.gen(function* () {
374 const fs = yield* FileSystem.FileSystem;
375 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-login-" });
376 const startupFile = `${directory}/bash-env`;
377 const bash = `${directory}/bash`;
378 // System profiles can reset HOME. Suppress profile reads while BASH_ENV
379 // checks that the launcher still requests Bash login mode.
380 yield* fs.writeFileString(bash, '#!/bin/sh\nexec /bin/bash --noprofile "$@"\n');
381 yield* fs.chmod(bash, 0o755);
382 yield* fs.writeFileString(
383 startupFile,
384 "if shopt -q login_shell; then export T3_SSH_LOGIN_ENV=loaded; fi\n",
385 );
386 const target = { alias: "login-test", hostname: "login-test", username: null, port: null };
387 let remoteCommand = "";
388 const capture = ChildProcessSpawner.make((command) =>
389 Effect.sync(() => {
390 const args = commandArgs(command);
391 remoteCommand = args.slice(args.indexOf(target.alias) + 1).join(" ");
392 return makeSuccessfulProcess('{"remotePort":3774}\n');
393 }),
394 );
395 yield* SshTunnel.launchOrReuseRemoteServer(target, undefined, ARCHIVE).pipe(
396 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, capture),
397 );
398 yield* fs.writeFileString(
399 `${directory}/.zprofile`,
400 "if [[ -o login ]]; then export T3_SSH_LOGIN_ENV=loaded; fi\n",
401 );
402 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
403 const cases = [
404 { shell: bash, outerShell: "/bin/sh", expected: "loaded" },
405 { shell: "/bin/sh", outerShell: "/bin/sh", expected: "missing" },
406 { shell: "", outerShell: "/bin/sh", expected: "missing" },
407 { shell: "/unavailable/csh", outerShell: "/bin/sh", expected: "missing" },
408 { shell: "/unavailable/tcsh", outerShell: "/bin/sh", expected: "missing" },
409 { shell: "/unavailable/pwsh", outerShell: "/bin/sh", expected: "missing" },
410 ];
411 if (yield* fs.exists("/bin/zsh")) {
412 cases.push({ shell: "/bin/zsh", outerShell: "/bin/zsh", expected: "loaded" });
413 }
414 if (yield* fs.exists("/bin/tcsh")) {
415 cases.push({ shell: "/bin/tcsh", outerShell: "/bin/tcsh", expected: "missing" });
416 }
417 const environment = yield* HostProcess.Environment;
418 for (const folder of (environment.PATH ?? "").split(":")) {
419 const nu = `${folder}/nu`;
420 if (folder && (yield* fs.exists(nu))) {
421 cases.push({ shell: nu, outerShell: nu, expected: "missing" });
422 break;
423 }
424 }
425 for (const { shell, outerShell, expected } of cases) {
426 const child = yield* spawner.spawn(
427 ChildProcess.make(outerShell, ["-c", remoteCommand], {
428 env: {
429 PATH: environment.PATH ?? "/usr/bin:/bin",
430 HOME: directory,
431 XDG_CONFIG_HOME: directory,
432 SHELL: shell,
433 BASH_ENV: startupFile,
434 ZDOTDIR: directory,
435 T3_SSH_LOGIN_ENV: "",
436 },
437 stdin: {
438 stream: Stream.make(
439 new TextEncoder().encode('printf "%s:%s" "${T3_SSH_LOGIN_ENV:-missing}" "$1"'),
440 ),
441 endOnDone: true,
442 },
443 }),
444 );
445 const [stdout, stderr, exitCode] = yield* Effect.all(
446 [
447 child.stdout.pipe(
448 Stream.decodeText(),
449 Stream.runFold(
450 () => "",
451 (a, b) => a + b,
452 ),
453 ),
454 child.stderr.pipe(
455 Stream.decodeText(),
456 Stream.runFold(
457 () => "",
458 (a, b) => a + b,
459 ),
460 ),
461 child.exitCode,
462 ],
463 { concurrency: "unbounded" },
464 );
465 assert.equal(Number(exitCode), 0, stderr);
466 // First-run shell messages can precede the bootstrap response.
467 const response = stdout.trim().split(/\r?\n/u).at(-1) ?? "";
468 assert.match(response, new RegExp(`^${expected}:[a-f0-9]{16}$`), shell || "unset SHELL");
469 }
470 }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeCrypto.layer))),
471 );
472
473 it.effect("accepts pretty-printed pairing JSON after remote shell startup noise", () => {
474 const target = {
475 alias: "devbox",
476 hostname: "devbox.example.com",
477 username: "julius",
478 port: 2222,
479 } as const;
480 const spawner = ChildProcessSpawner.make(() =>
481 Effect.succeed(
482 makeSuccessfulProcess(`loaded nvm default
483{
484 "id": "88941235-6ed5-4184-a2ff-5339e2075958",
485 "credential": "LCL4R2TPHDKQ",
486 "scopes": ["orchestration:read"],
487 "expiresAt": "2026-04-29T01:01:20.994Z"
488}
489
490`),
491 ),
492 );
493 const layerSpawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner);
494 const layerProcess = Layer.merge(
495 Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
496 layerSpawner,
497 );
498 return Effect.gen(function* () {
499 const result = yield* SshTunnel.issueRemotePairingToken(target, undefined, ARCHIVE);
500 assert.equal(result.credential, "LCL4R2TPHDKQ");
501 }).pipe(Effect.provide(layerProcess));
502 });
503
504 it.effect("retains a bounded diagnostic tail when the tunnel exits before readiness", () =>
505 Effect.gen(function* () {
506 const diagnostic = "Connection refused: café";
507 const output = new TextEncoder().encode(
508 "old diagnostics\n" + "x".repeat(32_768) + "\n" + diagnostic,
509 );
510 const spawner = ChildProcessSpawner.make((command) =>
511 Effect.succeed(
512 commandArgs(command).includes("-N")
513 ? {
514 ...makeSuccessfulProcess(""),
515 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)),
516 stderr: Stream.make(output.subarray(0, output.length - 1), output.subarray(-1)),
517 }
518 : makeSuccessfulProcess('{"remotePort":3773}\n'),
519 ),
520 );
521 const layer = Layer.mergeAll(
522 NodeServices.layer,
523 NodeCrypto.layer,
524 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
525 Layer.succeed(HttpClient.HttpClient, hangingHttpClient),
526 Layer.succeed(NetService.NetService, testNetService),
527 SshAuth.SshPasswordPrompt.disabledLayer,
528 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
529 );
530 yield* Effect.gen(function* () {
531 const manager = yield* SshTunnel.SshEnvironmentManager;
532 const result = yield* Effect.result(
533 manager.ensureEnvironment({
534 alias: "devbox",
535 hostname: "devbox",
536 username: null,
537 port: null,
538 }),
539 );
540 assert.isTrue(Result.isFailure(result));
541 if (Result.isFailure(result)) {
542 assert.instanceOf(result.failure, SshCommandError);
543 if (isSshCommandError(result.failure)) {
544 assert.equal(result.failure.exitCode, 1);
545 assert.equal(result.failure.stderr.length, 16_384);
546 assert.isTrue(result.failure.stderr.endsWith(diagnostic));
547 assert.equal(result.failure.message, result.failure.stderr.trim());
548 }
549 }
550 }).pipe(Effect.provide(layer), Effect.scoped);
551 }),
552 );
553
554 it.effect("reports the real exit when reading tunnel diagnostics fails", () =>
555 Effect.gen(function* () {
556 const diagnostic = "Permission denied (publickey)";
557 const spawner = ChildProcessSpawner.make((command) =>
558 Effect.succeed(
559 commandArgs(command).includes("-N")
560 ? {
561 ...makeSuccessfulProcess(""),
562 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(255)),
563 stderr: Stream.make(new TextEncoder().encode(diagnostic)).pipe(
564 Stream.concat(
565 Stream.fail(
566 PlatformError.systemError({
567 _tag: "Unknown",
568 module: "Stream",
569 method: "read",
570 description: "stderr read failed",
571 }),
572 ),
573 ),
574 ),
575 }
576 : makeSuccessfulProcess('{"remotePort":3773}\n'),
577 ),
578 );
579 const layer = Layer.mergeAll(
580 NodeServices.layer,
581 NodeCrypto.layer,
582 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
583 Layer.succeed(HttpClient.HttpClient, hangingHttpClient),
584 Layer.succeed(NetService.NetService, testNetService),
585 SshAuth.SshPasswordPrompt.disabledLayer,
586 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
587 );
588 yield* Effect.gen(function* () {
589 const manager = yield* SshTunnel.SshEnvironmentManager;
590 const result = yield* Effect.result(
591 manager.ensureEnvironment({
592 alias: "devbox",
593 hostname: "devbox",
594 username: null,
595 port: null,
596 }),
597 );
598 assert.isTrue(Result.isFailure(result));
599 if (Result.isFailure(result)) {
600 assert.instanceOf(result.failure, SshCommandError);
601 if (isSshCommandError(result.failure)) {
602 assert.equal(result.failure.exitCode, 255);
603 assert.equal(result.failure.stderr, diagnostic);
604 }
605 }
606 }).pipe(Effect.provide(layer), Effect.scoped);
607 }),
608 );
609
610 it.effect("waits for a tunnel whose ssh connection outlasts the readiness timeout", () =>
611 Effect.gen(function* () {
612 const LOCAL_PORT = 41_773;
613 // ssh spends ConnectTimeout on each unreachable address before the forward listens.
614 const SSH_CONNECT_MS = 25_000;
615 let forwardListensAt = Number.POSITIVE_INFINITY;
616 const tunnelSpawned = yield* Deferred.make<void>();
617 const forwardListening = (port: number) =>
618 Clock.currentTimeMillis.pipe(
619 Effect.map((now) => port === LOCAL_PORT && now >= forwardListensAt),
620 );
621 const spawner = ChildProcessSpawner.make((command) =>
622 commandArgs(command).includes("-N")
623 ? Effect.gen(function* () {
624 forwardListensAt = (yield* Clock.currentTimeMillis) + SSH_CONNECT_MS;
625 yield* Deferred.succeed(tunnelSpawned, undefined);
626 return makeRunningProcess(() => undefined);
627 })
628 : Effect.succeed(makeSuccessfulProcess('{"remotePort":3773}\n')),
629 );
630 const httpClient = HttpClient.make((request) =>
631 forwardListening(LOCAL_PORT).pipe(
632 Effect.flatMap((listening) =>
633 listening
634 ? Effect.succeed(
635 HttpClientResponse.fromWeb(request, new Response("", { status: 200 })),
636 )
637 : Effect.fail(
638 new HttpClientError.HttpClientError({
639 reason: new HttpClientError.TransportError({
640 request,
641 description: `connect ECONNREFUSED 127.0.0.1:${LOCAL_PORT}`,
642 }),
643 }),
644 ),
645 ),
646 ),
647 );
648 const net = NetService.NetService.of({
649 ...testNetService,
650 reserveLoopbackPort: () => Effect.succeed(LOCAL_PORT),
651 canListenOnHost: (port) =>
652 forwardListening(port).pipe(Effect.map((listening) => !listening)),
653 isPortAvailableOnLoopback: (port) =>
654 forwardListening(port).pipe(Effect.map((listening) => !listening)),
655 hasListenerOnHost: (port) => forwardListening(port),
656 });
657 const layer = Layer.mergeAll(
658 NodeServices.layer,
659 NodeCrypto.layer,
660 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
661 Layer.succeed(HttpClient.HttpClient, httpClient),
662 Layer.succeed(NetService.NetService, net),
663 SshAuth.SshPasswordPrompt.disabledLayer,
664 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
665 );
666 yield* Effect.gen(function* () {
667 const manager = yield* SshTunnel.SshEnvironmentManager;
668 const ensure = yield* Effect.forkChild(
669 manager.ensureEnvironment({
670 alias: "devbox",
671 hostname: "devbox",
672 username: null,
673 port: null,
674 }),
675 );
676 yield* Deferred.await(tunnelSpawned);
677 yield* TestClock.adjust(Duration.seconds(60));
678 const environment = yield* Fiber.join(ensure);
679 assert.equal(environment.httpBaseUrl, `http://127.0.0.1:${LOCAL_PORT}/`);
680 }).pipe(Effect.provide(layer), Effect.scoped);
681 }),
682 );
683
684 it.effect("connects and reuses a tunnel whose responses take over a second", () =>
685 Effect.gen(function* () {
686 // Over a ~1s RTT link each request through the forward takes a few round trips.
687 const RESPONSE_MS = 2_500;
688 const tunnelSpawned = yield* Deferred.make<void>();
689 const reconnectProbed = yield* Deferred.make<void>();
690 const forwardReplaced = yield* Deferred.make<void>();
691 let forwards = 0;
692 let connected = false;
693 const spawner = ChildProcessSpawner.make((command) =>
694 commandArgs(command).includes("-N")
695 ? Effect.sync(() => ++forwards).pipe(
696 Effect.flatMap((count) =>
697 Deferred.succeed(count === 1 ? tunnelSpawned : forwardReplaced, undefined),
698 ),
699 Effect.as(makeRunningProcess(() => undefined)),
700 )
701 : Effect.succeed(makeSuccessfulProcess('{"remotePort":3773}\n')),
702 );
703 const httpClient = HttpClient.make((request) =>
704 Effect.gen(function* () {
705 if (connected) yield* Deferred.succeed(reconnectProbed, undefined);
706 yield* Effect.sleep(Duration.millis(RESPONSE_MS));
707 return HttpClientResponse.fromWeb(request, new Response("", { status: 200 }));
708 }),
709 );
710 const layer = Layer.mergeAll(
711 NodeServices.layer,
712 NodeCrypto.layer,
713 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
714 Layer.succeed(HttpClient.HttpClient, httpClient),
715 Layer.succeed(NetService.NetService, testNetService),
716 SshAuth.SshPasswordPrompt.disabledLayer,
717 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
718 );
719 yield* Effect.gen(function* () {
720 const manager = yield* SshTunnel.SshEnvironmentManager;
721 const ensure = yield* Effect.forkChild(
722 Effect.result(
723 manager.ensureEnvironment({
724 alias: "devbox",
725 hostname: "devbox",
726 username: null,
727 port: null,
728 }),
729 ),
730 );
731 yield* Deferred.await(tunnelSpawned);
732 yield* TestClock.adjust(Duration.seconds(60));
733 const result = yield* Fiber.join(ensure);
734 assert.isTrue(
735 Result.isSuccess(result),
736 Result.isFailure(result) ? result.failure.message : undefined,
737 );
738 if (Result.isFailure(result)) return;
739
740 connected = true;
741 const reconnect = yield* Effect.forkChild(manager.ensureEnvironment(result.success.target));
742 yield* Deferred.await(reconnectProbed);
743 yield* TestClock.adjust(Duration.seconds(60));
744 const outcome = yield* Effect.raceFirst(
745 Fiber.join(reconnect).pipe(Effect.map((environment) => environment.httpBaseUrl)),
746 Deferred.await(forwardReplaced).pipe(Effect.as("replaced the forward")),
747 );
748 assert.equal(outcome, result.success.httpBaseUrl);
749 }).pipe(Effect.provide(layer), Effect.scoped);
750 }),
751 );
752
753 it.effect("reconnects a saved alias through the port its ssh config names now", () =>
754 Effect.gen(function* () {
755 let configPort = 53_791;
756 const tunnelPorts: Array<string | undefined> = [];
757 const spawner = ChildProcessSpawner.make((command) => {
758 const args = commandArgs(command);
759 if (args.includes("-G")) {
760 return Effect.succeed(
761 makeSuccessfulProcess(`hostname 127.0.0.1\nuser developer\nport ${configPort}\n`),
762 );
763 }
764 if (args.includes("-N")) {
765 tunnelPorts.push(args[args.indexOf("-p") + 1]);
766 return Effect.succeed(makeRunningProcess(() => undefined));
767 }
768 return Effect.succeed(makeSuccessfulProcess('{"remotePort":3773}\n'));
769 });
770 const layer = Layer.mergeAll(
771 NodeServices.layer,
772 NodeCrypto.layer,
773 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
774 Layer.succeed(HttpClient.HttpClient, testHttpClient),
775 Layer.succeed(NetService.NetService, testNetService),
776 SshAuth.SshPasswordPrompt.disabledLayer,
777 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
778 );
779 yield* Effect.gen(function* () {
780 const manager = yield* SshTunnel.SshEnvironmentManager;
781 const added = yield* manager.ensureEnvironment({
782 alias: "lima-sandbox",
783 hostname: "lima-sandbox",
784 username: null,
785 port: null,
786 });
787 // The VM restarts on a new port; the client reconnects with the target it saved.
788 yield* manager.disconnectEnvironment(added.target);
789 configPort = 53_812;
790 yield* manager.ensureEnvironment(added.target);
791 assert.deepStrictEqual(tunnelPorts, ["53791", "53812"]);
792 }).pipe(Effect.provide(layer), Effect.scoped);
793 }),
794 );
795
796 it.effect("fails with a readiness error when ssh never connects", () =>
797 Effect.gen(function* () {
798 const tunnelSpawned = yield* Deferred.make<void>();
799 let tunnelKilled = false;
800 const spawner = ChildProcessSpawner.make((command) =>
801 commandArgs(command).includes("-N")
802 ? Deferred.succeed(tunnelSpawned, undefined).pipe(
803 Effect.as(
804 makeRunningProcess(() => {
805 tunnelKilled = true;
806 }),
807 ),
808 )
809 : Effect.succeed(makeSuccessfulProcess('{"remotePort":3773}\n')),
810 );
811 const layer = Layer.mergeAll(
812 NodeServices.layer,
813 NodeCrypto.layer,
814 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
815 Layer.succeed(HttpClient.HttpClient, hangingHttpClient),
816 Layer.succeed(
817 NetService.NetService,
818 NetService.NetService.of({
819 ...testNetService,
820 hasListenerOnHost: () => Effect.succeed(false),
821 }),
822 ),
823 SshAuth.SshPasswordPrompt.disabledLayer,
824 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
825 );
826 yield* Effect.gen(function* () {
827 const manager = yield* SshTunnel.SshEnvironmentManager;
828 const ensure = yield* Effect.forkChild(
829 Effect.result(
830 manager.ensureEnvironment({
831 alias: "devbox",
832 hostname: "devbox",
833 username: null,
834 port: null,
835 }),
836 ),
837 );
838 yield* Deferred.await(tunnelSpawned);
839 yield* TestClock.adjust(Duration.minutes(2));
840 const result = yield* Fiber.join(ensure);
841 assert.isTrue(Result.isFailure(result));
842 if (Result.isFailure(result)) {
843 assert.instanceOf(result.failure, SshReadinessError);
844 assert.include(result.failure.message, "SSH connection to devbox did not come up");
845 }
846 assert.isTrue(tunnelKilled);
847 }).pipe(Effect.provide(layer), Effect.scoped);
848 }),
849 );
850
851 it.effect.each([
852 {
853 name: "does not ask for a password when the agent cannot sign an accepted key",
854 stderr:
855 'sign_and_send_pubkey: signing failed for ED25519 "/home/julius/.ssh/id_ed25519" from agent: communication with agent failed\r\n' +
856 "julius@devbox: Permission denied (publickey,password).\r\n",
857 expectedPromptAttempts: [],
858 },
859 {
860 name: "asks for a password when signing failure has no agent provenance",
861 stderr:
862 'sign_and_send_pubkey: signing failed for ED25519-CERT "key-cert.pub": agent refused operation\n' +
863 "julius@devbox: Permission denied (publickey,password).\n",
864 expectedPromptAttempts: [1],
865 },
866 {
867 name: "does not ask for a password when the host key cannot be verified",
868 stderr: "Host key verification failed.\r\n",
869 expectedPromptAttempts: [],
870 },
871 {
872 name: "asks for a password when the server rejects every key",
873 stderr: "julius@devbox: Permission denied (publickey,password).\r\n",
874 expectedPromptAttempts: [1],
875 },
876 ])("$name", ({ stderr, expectedPromptAttempts }) =>
877 Effect.gen(function* () {
878 const promptAttempts: Array<number> = [];
879 const spawner = ChildProcessSpawner.make((command) =>
880 Effect.succeed(
881 commandArgs(command).includes("-G")
882 ? makeSuccessfulProcess("")
883 : {
884 ...makeSuccessfulProcess(""),
885 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(255)),
886 stderr: Stream.make(new TextEncoder().encode(stderr)),
887 },
888 ),
889 );
890 const layer = Layer.mergeAll(
891 NodeServices.layer,
892 NodeCrypto.layer,
893 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
894 Layer.succeed(HttpClient.HttpClient, testHttpClient),
895 Layer.succeed(NetService.NetService, testNetService),
896 Layer.succeed(
897 SshAuth.SshPasswordPrompt,
898 SshAuth.SshPasswordPrompt.of({
899 isAvailable: true,
900 request: (request) =>
901 Effect.sync(() => {
902 promptAttempts.push(request.attempt);
903 return null;
904 }),
905 }),
906 ),
907 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
908 );
909 yield* Effect.gen(function* () {
910 const manager = yield* SshTunnel.SshEnvironmentManager;
911 const result = yield* Effect.result(
912 manager.ensureEnvironment({
913 alias: "devbox",
914 hostname: "devbox",
915 username: null,
916 port: null,
917 }),
918 );
919 assert.isTrue(Result.isFailure(result));
920 assert.deepEqual(promptAttempts, expectedPromptAttempts);
921 if (Result.isFailure(result)) {
922 if (expectedPromptAttempts.length === 0) {
923 assert.instanceOf(result.failure, SshCommandError);
924 if (isSshCommandError(result.failure)) {
925 assert.equal(result.failure.stderr, stderr);
926 }
927 } else {
928 assert.instanceOf(result.failure, SshPasswordPromptError);
929 }
930 }
931 }).pipe(Effect.provide(layer), Effect.scoped);
932 }),
933 );
934
935 it.effect.each(["local tunnel", "remote server"] as const)(
936 "waits for %s shutdown before reconnecting the same target",
937 (stalledStep) =>
938 Effect.gen(function* () {
939 const shutdownStarted = yield* Deferred.make<void>();
940 const finishShutdown = yield* Deferred.make<void>();
941 const reconnectsStarted = yield* Deferred.make<void>();
942 const pauseShutdown = Deferred.succeed(shutdownStarted, undefined).pipe(
943 Effect.andThen(Deferred.await(finishShutdown)),
944 );
945 let resolutions = 0;
946 let launches = 0;
947 let tunnels = 0;
948 let stops = 0;
949 let remoteRunning = false;
950 const target = { alias: "devbox", hostname: "devbox", username: null, port: null };
951 const spawner = ChildProcessSpawner.make((command) =>
952 Effect.gen(function* () {
953 const args = commandArgs(command);
954 const isTarget = args.includes(target.alias);
955 if (args.includes("-G")) {
956 if (isTarget && ++resolutions === 4) {
957 yield* Deferred.succeed(reconnectsStarted, undefined);
958 }
959 return makeSuccessfulProcess("");
960 }
961 if (args.includes("-N")) {
962 const tunnel = makeRunningProcess(() => undefined);
963 if (isTarget && ++tunnels === 1 && stalledStep === "local tunnel") {
964 return {
965 ...tunnel,
966 kill: (options?: ChildProcess.KillOptions) =>
967 pauseShutdown.pipe(Effect.andThen(tunnel.kill(options))),
968 };
969 }
970 return tunnel;
971 }
972 if (args.includes("-c")) {
973 if (isTarget) {
974 launches += 1;
975 remoteRunning = true;
976 }
977 return makeSuccessfulProcess('{"remotePort":3773}\n');
978 }
979 const stop = makeSuccessfulProcess('{"stopped":true}\n');
980 if (!isTarget) return stop;
981 const pause = ++stops === 1 && stalledStep === "remote server";
982 return {
983 ...stop,
984 exitCode: (pause ? pauseShutdown : Effect.void).pipe(
985 Effect.andThen(
986 Effect.sync(() => {
987 remoteRunning = false;
988 return ChildProcessSpawner.ExitCode(0);
989 }),
990 ),
991 ),
992 };
993 }),
994 );
995 const layer = Layer.mergeAll(
996 NodeServices.layer,
997 NodeCrypto.layer,
998 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
999 Layer.succeed(HttpClient.HttpClient, testHttpClient),
1000 Layer.succeed(NetService.NetService, testNetService),
1001 SshAuth.SshPasswordPrompt.disabledLayer,
1002 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
1003 );
1004 yield* Effect.gen(function* () {
1005 const manager = yield* SshTunnel.SshEnvironmentManager;
1006 yield* manager.ensureEnvironment(target);
1007 const disconnect = yield* Effect.forkChild(manager.disconnectEnvironment(target));
1008 yield* Deferred.await(shutdownStarted);
1009 const firstReconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
1010 const secondReconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
1011 yield* Deferred.await(reconnectsStarted);
1012
1013 yield* manager.ensureEnvironment({
1014 alias: "other",
1015 hostname: "other",
1016 username: null,
1017 port: null,
1018 });
1019 yield* TestClock.adjust(Duration.zero);
1020 const launchesBeforeShutdown = launches;
1021 yield* Deferred.succeed(finishShutdown, undefined);
1022 yield* Fiber.join(disconnect);
1023 const first = yield* Fiber.join(firstReconnect);
1024 const second = yield* Fiber.join(secondReconnect);
1025
1026 assert.equal(launchesBeforeShutdown, 1);
1027 assert.equal(launches, 2);
1028 assert.equal(tunnels, 2);
1029 assert.isTrue(remoteRunning);
1030 assert.equal(first.httpBaseUrl, second.httpBaseUrl);
1031 }).pipe(
1032 Effect.ensuring(Deferred.succeed(finishShutdown, undefined)),
1033 Effect.provide(layer),
1034 Effect.scoped,
1035 );
1036 }),
1037 );
1038
1039 it.live("concurrent launches on one remote home record the server that owns it", () =>
1040 // The second row's server never publishes server-runtime.json.
1041 Effect.forEach([1_500, null], (publishDelayMs) =>
1042 Effect.gen(function* () {
1043 const fs = yield* FileSystem.FileSystem;
1044 const path = yield* Path.Path;
1045 const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
1046 const serversFile = path.join(home, "servers");
1047 yield* Effect.addFinalizer(() =>
1048 fs.readFileString(serversFile).pipe(
1049 Effect.map((pids) => {
1050 for (const pid of pids.split("\n").filter(Boolean)) {
1051 try {
1052 process.kill(Number(pid), "SIGKILL");
1053 } catch {}
1054 }
1055 }),
1056 Effect.ignore,
1057 ),
1058 );
1059 const serverScript = path.join(home, "fake-t3.cjs");
1060 yield* fs.writeFileString(serverScript, fakeT3Serve(publishDelayMs));
1061 // A launch that died holding the lock must not block or double-admit later ones.
1062 const launchLock = path.join(home, ".t3", "ssh-launch", ".launch.lock");
1063 yield* fs.makeDirectory(path.dirname(launchLock), { recursive: true });
1064 yield* fs.writeFileString(launchLock, "999999999\n");
1065 const targets = [
1066 { alias: "devbox", hostname: "devbox", username: null, port: null },
1067 { alias: "devbox-tailnet", hostname: "devbox-tailnet", username: null, port: null },
1068 ];
1069
1070 const launches = yield* Effect.forEach(
1071 targets,
1072 (target) =>
1073 SshTunnel.launchOrReuseRemoteServer(target, undefined, {
1074 nodeScriptPath: serverScript,
1075 }),
1076 { concurrency: "unbounded" },
1077 ).pipe(Effect.provide(localRemoteShellLayer(home)));
1078
1079 const ownerPid = Number(
1080 yield* fs.readFileString(path.join(home, ".t3", "userdata", "server.lock")),
1081 );
1082 assert.sameMembers(
1083 launches.map((launch) => launch.remoteServerKind),
1084 ["managed", "external"],
1085 );
1086 assert.equal(launches[0]!.remotePort, launches[1]!.remotePort);
1087 assert.deepEqual((yield* fs.readFileString(serversFile)).trim().split("\n"), [
1088 String(ownerPid),
1089 ]);
1090 for (const [index, launch] of launches.entries()) {
1091 if (launch.remoteServerKind !== "managed") continue;
1092 const stateKey = yield* remoteStateKey(targets[index]!);
1093 const recordedPid = yield* fs.readFileString(
1094 path.join(home, ".t3", "ssh-launch", stateKey, "pid"),
1095 );
1096 assert.equal(
1097 Number(recordedPid.trim()),
1098 ownerPid,
1099 `${targets[index]!.alias} reported a managed server, but its recorded PID is not the server that owns the remote home`,
1100 );
1101 }
1102 assert.isFalse(yield* fs.exists(launchLock));
1103 }).pipe(Effect.scoped),
1104 ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1105 );
1106});
1107
1108// On Linux kill -0 accepts a thread ID and kill <tid> signals its whole process,
1109// but ps lists only thread-group leaders. After a reboot the managed pid file can
1110// name a thread or a whole process that is not our server.
1111describe.runIf(HostProcess.Platform.defaultValue() === "linux")("stale managed pid file", () => {
1112 const FAKE_SERVER = `const fs = require("node:fs");
1113const http = require("node:http");
1114const path = require("node:path");
1115const args = process.argv.slice(2);
1116const flag = (name) => args[args.indexOf(name) + 1];
1117const server = http.createServer((_request, response) => response.end("ok"));
1118server.listen(Number(flag("--port") ?? 0), "127.0.0.1", () => {
1119 const port = server.address().port;
1120 const base = flag("--base-dir");
1121 if (base) {
1122 fs.mkdirSync(path.join(base, "userdata"), { recursive: true });
1123 fs.writeFileSync(
1124 path.join(base, "userdata", "server-runtime.json"),
1125 JSON.stringify({ version: 1, pid: process.pid, port, origin: "http://127.0.0.1:" + port }),
1126 );
1127 }
1128 console.log(port);
1129});
1130`;
1131 const target = { alias: "devbox", hostname: "devbox", username: null, port: null } as const;
1132 const staleIds = ["a thread of an unrelated process", "an unrelated process"] as const;
1133
1134 const isRunning = (pid: number) =>
1135 Effect.flatMap(FileSystem.FileSystem, (fs) => fs.readFileString(`/proc/${pid}/stat`)).pipe(
1136 Effect.map((stat) => !/^\d+ \(.*\) Z/u.test(stat)),
1137 Effect.orElseSucceed(() => false),
1138 );
1139
1140 const prepare = (staleId: (typeof staleIds)[number]) =>
1141 Effect.gen(function* () {
1142 const fs = yield* FileSystem.FileSystem;
1143 const realSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1144 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-stale-pid-" });
1145 const home = `${root}/home`;
1146 const fakeServer = `${root}/fake-t3.cjs`;
1147 yield* fs.makeDirectory(home, { recursive: true });
1148 yield* fs.writeFileString(fakeServer, FAKE_SERVER);
1149 // Runs the remote command locally under the temp HOME. setsid keeps a launched server alive
1150 // when the command's process group goes away, as sshd would.
1151 const sshAsLocalShell = ChildProcessSpawner.make((command) => {
1152 if (command._tag !== "StandardCommand" || command.command !== "ssh") {
1153 return realSpawner.spawn(command);
1154 }
1155 if (command.args.includes("-G")) return Effect.succeed(makeSuccessfulProcess(""));
1156 const stateKey = launchStateKey(command);
1157 const remoteArgs =
1158 stateKey === undefined
1159 ? command.args.slice(command.args.indexOf("--") + 2)
1160 : ["sh", "-s", "--", stateKey];
1161 return realSpawner.spawn(
1162 ChildProcess.make("setsid", ["-w", ...remoteArgs], {
1163 ...command.options,
1164 env: { PATH: process.env.PATH ?? "", HOME: home },
1165 extendEnv: false,
1166 }),
1167 );
1168 });
1169 const holder = yield* realSpawner.spawn(ChildProcess.make("node", [fakeServer]));
1170 const holderPort = Number(
1171 Option.getOrThrow(
1172 yield* holder.stdout.pipe(Stream.decodeText(), Stream.splitLines, Stream.runHead),
1173 ),
1174 );
1175 const tid = Number(
1176 (yield* fs.readDirectory(`/proc/${holder.pid}/task`)).find(
1177 (task) => task !== `${holder.pid}`,
1178 ),
1179 );
1180 assert.isTrue(Number.isInteger(tid));
1181 return {
1182 fs,
1183 home,
1184 fakeServer,
1185 holder,
1186 holderPort,
1187 sshAsLocalShell,
1188 staleId: staleId === "an unrelated process" ? holder.pid : tid,
1189 stateDir: `${home}/.t3/ssh-launch/${yield* remoteStateKey(target)}`,
1190 };
1191 });
1192
1193 it.live.each(staleIds)("launches a fresh server when the managed pid file names %s", (kind) =>
1194 Effect.gen(function* () {
1195 const { fs, home, fakeServer, holderPort, sshAsLocalShell, staleId, stateDir } =
1196 yield* prepare(kind);
1197 const launch = SshTunnel.launchOrReuseRemoteServer(target, undefined, {
1198 nodeScriptPath: fakeServer,
1199 }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, sshAsLocalShell));
1200 const readPid = fs
1201 .readFileString(`${stateDir}/pid`)
1202 .pipe(Effect.map((pid) => Number(pid.trim())));
1203 const launchedPids: Array<number> = [];
1204 yield* Effect.addFinalizer(() =>
1205 Effect.forEach(launchedPids, (pid) =>
1206 Effect.sync(() => {
1207 try {
1208 process.kill(pid, "SIGKILL");
1209 } catch {}
1210 }),
1211 ),
1212 );
1213
1214 const first = yield* launch;
1215 const firstPid = yield* readPid;
1216 launchedPids.push(firstPid);
1217 assert.deepEqual(yield* launch, first);
1218 assert.equal(yield* readPid, firstPid);
1219
1220 // Reboot: our server is gone and its recorded PID now names something that
1221 // answers on the recorded port.
1222 process.kill(firstPid, "SIGKILL");
1223 yield* fs.remove(`${home}/.t3/userdata/server-runtime.json`);
1224 yield* fs.writeFileString(`${stateDir}/pid`, `${staleId}\n`);
1225 yield* fs.writeFileString(`${stateDir}/port`, `${holderPort}\n`);
1226
1227 const relaunched = yield* launch;
1228 const pid = yield* readPid;
1229 if (pid !== staleId) launchedPids.push(pid);
1230 assert.notEqual(
1231 relaunched.remotePort,
1232 holderPort,
1233 `the launcher kept ${kind} as its running server`,
1234 );
1235 assert.isTrue(yield* isRunning(pid));
1236 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1237 );
1238
1239 it.live.each(staleIds)("disconnecting does not signal %s named by the managed pid file", (kind) =>
1240 Effect.gen(function* () {
1241 const { fs, holder, sshAsLocalShell, staleId, stateDir } = yield* prepare(kind);
1242 yield* fs.makeDirectory(stateDir, { recursive: true });
1243 yield* fs.writeFileString(`${stateDir}/pid`, `${staleId}\n`);
1244 yield* fs.writeFileString(`${stateDir}/managed`, "managed\n");
1245 const layer = Layer.mergeAll(
1246 NodeServices.layer,
1247 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, sshAsLocalShell),
1248 Layer.succeed(HttpClient.HttpClient, testHttpClient),
1249 Layer.succeed(NetService.NetService, testNetService),
1250 SshAuth.SshPasswordPrompt.disabledLayer,
1251 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
1252 );
1253
1254 yield* Effect.flatMap(SshTunnel.SshEnvironmentManager, (manager) =>
1255 manager.disconnectEnvironment(target),
1256 ).pipe(Effect.provide(layer));
1257 assert.isTrue(yield* isRunning(holder.pid), `disconnecting signalled ${kind}`);
1258 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1259 );
1260});
1261
1262// On macOS, Node can bind 127.0.0.1:<port> while another process (such as the
1263// desktop app with network access on) listens on *:<port>. Linux refuses that
1264// bind unless both sockets set SO_REUSEPORT, so on Linux the preload adds it to
1265// the picker's probe and the holder sets it too.
1266
1267describe("SSH launch during desktop update", () => {
1268 it.effect("archive launcher reuses the replacement without host Node", () =>
1269 Effect.gen(function* () {
1270 const script = yield* captureLauncherFor({ archiveVersion: "1.2.3" });
1271 const result = runLauncher(script, {
1272 archive: true,
1273 marker: { startedAt: 0, expiresAt: 60_000 },
1274 readyAt: 2_000,
1275 });
1276 assert.deepEqual(result.output, { remotePort: 4888, serverKind: "external" });
1277 assert.equal(result.now, 2_000);
1278 assert.isFalse(result.events.some((event) => event.startsWith("start ")));
1279 }),
1280 );
1281
1282 it.effect.each(["managed", "external"] as const)(
1283 "reuses recorded %s server without runtime metadata or an update wait",
1284 (recordedKind) =>
1285 Effect.gen(function* () {
1286 const result = runLauncher(yield* captureLauncher, {
1287 marker: { startedAt: 0, expiresAt: 60_000 },
1288 recordedKind,
1289 });
1290 assert.deepEqual(result.output, { remotePort: 5999, serverKind: recordedKind });
1291 assert.equal(result.now, 0);
1292 assert.isFalse(result.events.some((event) => event.startsWith("kill ")));
1293 assert.isFalse(result.events.some((event) => event.startsWith("start ")));
1294 }),
1295 );
1296
1297 it.effect.each([
1298 ["malformed", "broken"],
1299 ["future", { startedAt: 1_000, expiresAt: 61_000 }],
1300 ["expired", { startedAt: -60_000, expiresAt: 0 }],
1301 ] as const)("preserves %s markers without waiting", ([, marker]) =>
1302 Effect.gen(function* () {
1303 const result = runLauncher(yield* captureLauncher, { marker });
1304 assert.equal(result.now, 0);
1305 assert.isTrue(result.markerExists);
1306 assert.deepEqual(result.output, { remotePort: 5999, serverKind: "managed" });
1307 }),
1308 );
1309
1310 it.effect("bounds the wait when the host clock jumps back", () =>
1311 Effect.gen(function* () {
1312 const result = runLauncher(yield* captureLauncher, {
1313 marker: { startedAt: 0, expiresAt: 60_000 },
1314 clockJump: -30_000,
1315 });
1316 assert.deepEqual(result.output, { remotePort: 5999, serverKind: "managed" });
1317 assert.isBelow(result.events.filter((event) => event.startsWith("runtime ")).length, 62);
1318 }),
1319 );
1320
1321 it.effect("refreshes runtime after readiness removes the marker between polls", () =>
1322 Effect.gen(function* () {
1323 const result = runLauncher(yield* captureLauncher, {
1324 marker: { startedAt: 0, expiresAt: 60_000 },
1325 readyAt: 8_000,
1326 clearMarkerOnReady: true,
1327 });
1328 assert.deepEqual(result.output, { remotePort: 4888, serverKind: "external" });
1329 assert.equal(result.now, 8_000);
1330 assert.isFalse(result.markerExists);
1331 assert.isFalse(result.events.some((event) => event.startsWith("start ")));
1332 assert.isFalse(result.events.some((event) => event.startsWith("kill ")));
1333 }),
1334 );
1335
1336 it.effect("reuses a runtime published before the first marker read", () =>
1337 Effect.gen(function* () {
1338 const result = runLauncher(yield* captureLauncher, {
1339 marker: { startedAt: 0, expiresAt: 60_000 },
1340 publishAfterFirstRuntimeRead: true,
1341 });
1342 assert.deepEqual(result.output, { remotePort: 4888, serverKind: "external" });
1343 assert.equal(result.now, 0);
1344 assert.isFalse(result.events.some((event) => event.startsWith("start ")));
1345 }),
1346 );
1347
1348 it.effect("waits across the eight-second installer gap and reuses the replacement server", () =>
1349 Effect.gen(function* () {
1350 const result = runLauncher(yield* captureLauncher, {
1351 marker: { startedAt: 0, expiresAt: 60_000 },
1352 readyAt: 8_000,
1353 });
1354 assert.deepEqual(result.output, { remotePort: 4888, serverKind: "external" });
1355 assert.equal(result.now, 8_000);
1356 assert.isFalse(result.events.some((event) => event.startsWith("start ")));
1357 assert.isAtLeast(result.events.filter((event) => event.startsWith("runtime ")).length, 2);
1358 }),
1359 );
1360
1361 it.effect("keeps waiting while a stale runtime record points at a dead port", () =>
1362 Effect.gen(function* () {
1363 const result = runLauncher(yield* captureLauncher, {
1364 marker: { startedAt: 0, expiresAt: 60_000 },
1365 initialPort: 3773,
1366 readyAt: 3_000,
1367 });
1368 assert.deepEqual(result.output, { remotePort: 4888, serverKind: "external" });
1369 assert.equal(result.now, 3_000);
1370 assert.isFalse(result.events.some((event) => event.startsWith("start ")));
1371 }),
1372 );
1373
1374 it.effect("starts a server when the marker expires without a replacement becoming ready", () =>
1375 Effect.gen(function* () {
1376 const result = runLauncher(yield* captureLauncher, {
1377 marker: { startedAt: 0, expiresAt: 60_000 },
1378 });
1379 assert.deepEqual(result.output, { remotePort: 5999, serverKind: "managed" });
1380 assert.isAtLeast(result.now, 60_000);
1381 assert.isAtMost(result.now, 61_000);
1382 assert.isTrue(result.markerExists, "an expired marker is ignored, not deleted");
1383 assert.equal(result.events.filter((event) => event.startsWith("start ")).length, 1);
1384 }),
1385 );
1386
1387 it.effect("starts immediately without a marker", () =>
1388 Effect.gen(function* () {
1389 const result = runLauncher(yield* captureLauncher, {});
1390 assert.deepEqual(result.output, { remotePort: 5999, serverKind: "managed" });
1391 assert.equal(result.now, 0);
1392 assert.equal(result.events.filter((event) => event.startsWith("start ")).length, 1);
1393 assert.equal(result.events.filter((event) => event.startsWith("marker ")).length, 1);
1394 assert.equal(result.events.filter((event) => event.startsWith("runtime ")).length, 1);
1395 }),
1396 );
1397});