1// @effect-diagnostics nodeBuiltinImport:off
2// @effect-diagnostics globalTimers:off
3// The launcher must survive unloadable server versions; use Node built-ins without Effect.
4import * as NodeChildProcess from "node:child_process";
5import * as NodeCrypto from "node:crypto";
6import * as NodeFS from "node:fs";
7import * as NodeFSP from "node:fs/promises";
8import * as NodePath from "node:path";
9import * as NodeUtil from "node:util";
10import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts";
11
12import type {
13 PendingServiceUpdate,
14 ServiceLauncherChildMessage,
15 ServiceLauncherContext,
16 ServiceLauncherParentMessage,
17 ServiceState,
18 ServiceUpdateRecord,
19} from "./cloud/serviceProtocol.ts";
20import {
21 compareExactServiceVersions,
22 decodeServiceLauncherChildMessage,
23 isExactServiceVersion,
24 parseLegacyServiceState,
25 disabledServiceTask,
26 parseServiceState,
27 serviceTaskIsDisabled,
28 serviceUnitBaseDirOf,
29 serviceUnitLogPathOf,
30 SERVICE_ENV_FILE,
31 SERVICE_LAUNCHER_LOCK_FILE,
32 SERVICE_LAUNCHER_CONTEXT_ENV,
33 SERVICE_LAUNCHER_PROTOCOL,
34 SERVICE_STATE_FILE,
35 SERVICE_RESTART_PENDING_FILE,
36 SERVICE_STOP_MARKER_FILE,
37 SERVICE_UNIT_ENV,
38} from "./cloud/serviceProtocol.ts";
39
40const HANDOFF_DELAY_MS = 2_000;
41/**
42 * Migrations count toward it and can scale with the database: tens of GB on a cold page cache
43 * take minutes. A missed window rolls back the database, so the next attempt pays the same cost.
44 */
45const PREPARED_TIMEOUT_MS = 30 * 60_000;
46const TERMINATE_GRACE_MS = 5_000;
47// A replacement started by a handoff waits up to two minutes for the stopping launcher.
48const LOCK_RETRY_MS = 500;
49const LOCK_ATTEMPTS = 240;
50// Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect.
51const STATE_DIR_OWNED_EXIT_CODE = 78;
52
53type TerminalStatus = "committed" | "rolled-back" | "failed";
54type ChildRole = "active" | "trial";
55
56interface ManagedChild {
57 readonly version: string;
58 role: ChildRole;
59 readonly process: NodeChildProcess.ChildProcess;
60}
61
62const runtimePaths = (baseDir: string, version: string) => {
63 const versionDir = NodePath.join(baseDir, "runtime", "versions", version);
64 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
65 const executableName = process.platform === "win32" ? "t3.exe" : "t3";
66 return {
67 versionDir,
68 entryPath: NodePath.join(versionDir, executableName),
69 sentinelPath: NodePath.join(versionDir, ".install-complete"),
70 };
71};
72
73const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
74 command: paths.entryPath,
75 args: ["serve"],
76});
77
78/**
79 * The unit or launch agent that `t3 service install` wrote. Its program names
80 * the runtime that ran the install, and a remote update never replaces it, so
81 * after a commit the launcher points it at the committed runtime. Otherwise the
82 * next start lands on a runtime that nothing in the service state records.
83 */
84export interface ServiceUnit {
85 readonly path: string;
86 /** systemd and Task Scheduler keep a loaded copy; launchd re-reads the plist at bootstrap. */
87 readonly reload: () => Promise<void>;
88}
89
90const runSchtasks = (...args: ReadonlyArray<string>) =>
91 NodeUtil.promisify(NodeChildProcess.execFile)("schtasks", args, {
92 windowsHide: true,
93 timeout: 30_000,
94 });
95
96/**
97 * Registering replaces the task, so one the CLI disabled to keep the service stopped is
98 * registered disabled. The caller holds the unit lease, as the CLI does to disable it.
99 */
100export async function registerServiceTask(
101 unitPath: string,
102 taskName: string,
103 schtasks: (...args: ReadonlyArray<string>) => Promise<{ readonly stdout: string }> = runSchtasks,
104): Promise<void> {
105 const register = (xmlPath: string) => schtasks("/Create", "/XML", xmlPath, "/TN", taskName, "/F");
106 const registered = await schtasks("/Query", "/TN", taskName, "/XML");
107 if (!serviceTaskIsDisabled(registered.stdout)) {
108 await register(unitPath);
109 return;
110 }
111 const disabledPath = `${unitPath}.${process.pid}.disabled`;
112 try {
113 const unit = await NodeFSP.readFile(unitPath, "utf8");
114 await NodeFSP.writeFile(disabledPath, disabledServiceTask(unit));
115 await register(disabledPath);
116 } finally {
117 await NodeFSP.rm(disabledPath, { force: true });
118 }
119}
120
121function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined {
122 const unitName = env[SERVICE_UNIT_ENV]?.trim();
123 if (!unitName) return undefined;
124 if (unitName.endsWith(".xml")) {
125 const localAppData = env.LOCALAPPDATA?.trim();
126 if (!localAppData) return undefined;
127 const path = NodePath.win32.join(localAppData, "t3code", unitName);
128 const taskName = serviceTaskName(env.USERDOMAIN ?? "", env.USERNAME ?? "");
129 return { path, reload: () => registerServiceTask(path, taskName) };
130 }
131 const home = env.HOME?.trim();
132 if (!home) return undefined;
133 if (unitName.endsWith(".plist")) {
134 return {
135 path: NodePath.join(home, "Library", "LaunchAgents", unitName),
136 reload: () => Promise.resolve(),
137 };
138 }
139 // The manager's own environment, which decides where it reads user units.
140 const xdgConfigHome = env.XDG_CONFIG_HOME?.trim();
141 const configHome =
142 xdgConfigHome && NodePath.isAbsolute(xdgConfigHome)
143 ? xdgConfigHome
144 : NodePath.join(home, ".config");
145 return {
146 path: NodePath.join(configHome, "systemd", "user", unitName),
147 reload: () =>
148 new Promise((resolve, reject) => {
149 NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], {
150 stdio: "ignore",
151 timeout: 30_000,
152 })
153 .once("error", reject)
154 .once("exit", (code, signal) =>
155 code === 0
156 ? resolve()
157 : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)),
158 );
159 }),
160 };
161}
162
163// systemd's ExecStart value, or the plist string or task argument before __service-launcher.
164// Only the version segment changes, so quoted or escaped paths stay as rendered.
165// The launch agent's PATH names a versions directory without the executable; leave it.
166const SERVICE_UNIT_PROGRAMS = [
167 /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m,
168 /(<string>[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*<string>__service-launcher<\/string>)/,
169 /(<Arguments>--headless "[^"<]*?[\\/]runtime[\\/]versions[\\/])([^\\/"<]+)([\\/]t3\.exe" "__service-launcher")/,
170] as const;
171
172/**
173 * The unit contents with their program pointed at `version`'s runtime. A unit
174 * already at that version or a newer one is returned unchanged: `t3 update`
175 * may have repointed it to a later version while this commit was in flight.
176 */
177function repointServiceUnit(contents: string, version: string): string {
178 for (const program of SERVICE_UNIT_PROGRAMS) {
179 const match = program.exec(contents);
180 if (match === null) continue;
181 const [, prefix = "", current = "", suffix = ""] = match;
182 if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) {
183 return contents;
184 }
185 return contents.replace(program, () => `${prefix}${version}${suffix}`);
186 }
187 return contents;
188}
189
190const SERVICE_LOG_MAX_BYTES = 10 * 1024 * 1024;
191const SERVICE_LOG_CHECK_INTERVAL_MS = 10 * 60_000;
192
193/**
194 * The service manager holds the log open for appending, so it is copied aside and emptied in
195 * place rather than renamed. The copy is best effort: on a full disk the truncate frees space.
196 */
197async function boundServiceLog(logPath: string): Promise<void> {
198 const size = await NodeFSP.stat(logPath).then(
199 (stats) => stats.size,
200 () => 0,
201 );
202 if (size <= SERVICE_LOG_MAX_BYTES) return;
203 const archivePath = `${logPath}.1`;
204 const pendingArchivePath = `${archivePath}.tmp`;
205 await NodeFSP.copyFile(logPath, pendingArchivePath)
206 .then(() => NodeFSP.rename(pendingArchivePath, archivePath))
207 .catch(() => NodeFSP.rm(pendingArchivePath, { force: true }).catch(() => undefined));
208 await NodeFSP.truncate(logPath, 0).catch(() => undefined);
209}
210
211async function serviceLogPath(
212 options: LauncherOptions,
213 serviceUnit: ServiceUnit | undefined,
214): Promise<string | undefined> {
215 if (options.logFile !== undefined) return options.logFile;
216 if (serviceUnit === undefined) return undefined;
217 const contents = await NodeFSP.readFile(serviceUnit.path, "utf8").catch(() => undefined);
218 return contents === undefined ? undefined : serviceUnitLogPathOf(contents);
219}
220
221const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
222const RESTORE_MARKER = ".restore-pending";
223
224const databaseBackupDir = (baseDir: string, updateId: string) =>
225 NodePath.join(baseDir, "runtime", "db-backup", updateId);
226
227const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>
228 NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);
229
230async function pathExists(target: string): Promise<boolean> {
231 try {
232 await NodeFSP.access(target);
233 return true;
234 } catch (cause) {
235 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;
236 throw cause;
237 }
238}
239
240const MANAGED_SERVICE_ENV_NAMES = new Set([
241 "PATH",
242 "T3CODE_HOME",
243 "T3_BOOT_SERVICE_UNIT",
244 SERVICE_LAUNCHER_CONTEXT_ENV,
245]);
246
247export async function readServiceEnvironment(
248 baseDir: string,
249 inherited: NodeJS.ProcessEnv = process.env,
250): Promise<NodeJS.ProcessEnv> {
251 const environment = { ...inherited };
252 let contents: string;
253 try {
254 contents = await NodeFSP.readFile(NodePath.join(baseDir, SERVICE_ENV_FILE), "utf8");
255 } catch (cause) {
256 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return environment;
257 throw cause;
258 }
259 for (const [name, value] of Object.entries(NodeUtil.parseEnv(contents))) {
260 if (
261 /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
262 !MANAGED_SERVICE_ENV_NAMES.has(name.toUpperCase())
263 ) {
264 environment[name] = value;
265 }
266 }
267 return environment;
268}
269
270// Opened read-write: Windows refuses to flush a handle without write access.
271async function syncFile(filePath: string): Promise<void> {
272 const handle = await NodeFSP.open(filePath, "r+");
273 try {
274 await handle.sync();
275 } finally {
276 await handle.close();
277 }
278}
279
280// Fsync makes renames durable; Windows returns EPERM and NTFS journals them itself.
281async function syncDirectory(directory: string): Promise<void> {
282 const handle = await NodeFSP.open(directory, "r");
283 try {
284 await handle.sync();
285 } catch (error) {
286 if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error;
287 } finally {
288 await handle.close();
289 }
290}
291
292function runCp(args: ReadonlyArray<string>): Promise<boolean> {
293 return new Promise((resolve) => {
294 NodeChildProcess.execFile("cp", args, (error) => resolve(error === null));
295 });
296}
297
298// macOS fs.copyFile clone flags fall back to full copies.
299async function copyDatabaseFile(source: string, destination: string): Promise<void> {
300 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
301 const platform = process.platform;
302 // cp -c replaces files and breaks hard links; copy into hard-linked destinations instead.
303 const hardLinked =
304 platform === "darwin" &&
305 ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1;
306 const args =
307 platform === "darwin" && !hardLinked
308 ? ["-c", source, destination]
309 : platform === "linux"
310 ? ["--reflink=auto", source, destination]
311 : undefined;
312 if (args === undefined || !(await runCp(args))) await NodeFSP.copyFile(source, destination);
313}
314
315const GIB = 1024 ** 3;
316// Cloned backups still need headroom for trial writes and migrations to diverge.
317const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
318const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
319// While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
320const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
321
322async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
323 const stats: Array<NodeFS.Stats> = [];
324 for (const suffix of DB_FILE_SUFFIXES) {
325 const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => {
326 if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined;
327 throw cause;
328 });
329 if (stat !== undefined) stats.push(stat);
330 }
331 return stats;
332}
333
334// cp -c silently copies across volumes; require one device matching the APFS system volume's filesystem type.
335async function backupCanClone(dbPath: string, backupRoot: string): Promise<boolean> {
336 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
337 const platform = process.platform;
338 if (platform === "darwin") {
339 const [backupDirectory, backupFs, systemFs] = await Promise.all([
340 NodeFSP.stat(backupRoot),
341 NodeFSP.statfs(backupRoot),
342 NodeFSP.statfs("/"),
343 ]);
344 if (backupFs.type !== systemFs.type) return false;
345 return (await statDatabaseFiles(dbPath)).every(
346 (source) => source.dev === backupDirectory.dev && source.nlink === 1,
347 );
348 }
349 if (platform !== "linux") return false;
350 const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`);
351 try {
352 for (const suffix of DB_FILE_SUFFIXES) {
353 const source = `${dbPath}${suffix}`;
354 if (suffix !== "" && !(await pathExists(source))) continue;
355 if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false;
356 }
357 return true;
358 } finally {
359 for (const suffix of DB_FILE_SUFFIXES) {
360 await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined);
361 }
362 }
363}
364
365export interface BackupSpaceProbe {
366 readonly readFreeBytes: (directory: string) => Promise<number>;
367 readonly canClone: (dbPath: string, backupRoot: string) => Promise<boolean>;
368}
369
370const filesystemBackupSpaceProbe: BackupSpaceProbe = {
371 readFreeBytes: async (directory) => {
372 const stats = await NodeFSP.statfs(directory);
373 return stats.bavail * stats.bsize;
374 },
375 canClone: backupCanClone,
376};
377
378// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.
379// Update requests are refused while one is pending, so no staging directory is still in use here.
380async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {
381 for (const entry of await NodeFSP.readdir(backupRoot)) {
382 if (!entry.endsWith(".staging")) continue;
383 await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(
384 () => undefined,
385 );
386 }
387}
388
389/** Check before stopping the server so full disks refuse updates without an outage. */
390export async function databaseBackupBlocker(
391 baseDir: string,
392 dbPath: string,
393 probe: Partial<BackupSpaceProbe> = {},
394): Promise<string | undefined> {
395 const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
396 const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
397 await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
398 await removeAbandonedBackupCopies(backupRoot);
399 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
400 const size = files.reduce((total, file) => total + file.size, 0);
401 const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
402 if (free >= fullCopyBytes) return undefined;
403 const clonedBytes = Math.max(
404 CLONED_BACKUP_MIN_HEADROOM_BYTES,
405 size * CLONED_BACKUP_HEADROOM_FRACTION,
406 );
407 const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes;
408 if (free >= needed) return undefined;
409 const gb = (bytes: number) => (bytes / GIB).toFixed(1);
410 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
411}
412
413// A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
414async function copyWithoutOpening(source: string, destination: string): Promise<void> {
415 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
416 if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
417 if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
418}
419
420// Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
421// which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
422// `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
423// a write-heavy server's new writes wait, uncommitted, until the server stops.
424async function startOnlineDatabaseBackup(
425 baseDir: string,
426 pending: PendingServiceUpdate,
427 probe: Partial<BackupSpaceProbe> = {},
428): Promise<(() => Promise<void>) | undefined> {
429 const backupDir = databaseBackupDir(baseDir, pending.id);
430 if (await pathExists(backupDir)) return undefined;
431 const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
432 if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
433 return undefined;
434 const NodeSqlite = await import("node:sqlite").catch(() => undefined);
435 if (NodeSqlite === undefined) return undefined;
436
437 const stagingDir = `${backupDir}.staging`;
438 const discardStaging = () =>
439 NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
440 const fallBack = async (cause: unknown) => {
441 await discardStaging();
442 process.stderr.write(
443 `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
444 );
445 return undefined;
446 };
447 let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
448 // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
449 let dbPath = pending.dbPath;
450 const release = () => {
451 try {
452 snapshot?.close();
453 } catch {}
454 snapshot = undefined;
455 };
456 try {
457 await discardStaging();
458 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
459 dbPath = await NodeFSP.realpath(pending.dbPath);
460 snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
461 const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
462 if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
463 snapshot.exec("BEGIN");
464 snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
465 const mainCopy = databaseBackupFile(stagingDir, "");
466 await copyWithoutOpening(dbPath, mainCopy);
467 await syncFile(mainCopy);
468 } catch (cause) {
469 release();
470 return fallBack(cause);
471 }
472 return async () => {
473 try {
474 const wal = `${dbPath}-wal`;
475 if (!(await pathExists(wal)))
476 throw new Error("the WAL disappeared while the snapshot was open.");
477 const walCopy = databaseBackupFile(stagingDir, "-wal");
478 await copyWithoutOpening(wal, walCopy);
479 await syncFile(walCopy);
480 release();
481 await NodeFSP.rename(stagingDir, backupDir);
482 await syncDirectory(NodePath.dirname(backupDir));
483 } catch (cause) {
484 release();
485 await fallBack(cause);
486 }
487 };
488}
489
490// Preserve completed backups; restarted trials may already have changed the live database.
491async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
492 const backupDir = databaseBackupDir(baseDir, pending.id);
493 if (await pathExists(backupDir)) return;
494
495 const stagingDir = `${backupDir}.staging`;
496 await NodeFSP.rm(stagingDir, { recursive: true, force: true });
497 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
498 try {
499 for (const suffix of DB_FILE_SUFFIXES) {
500 const source = `${pending.dbPath}${suffix}`;
501 if (suffix !== "" && !(await pathExists(source))) continue;
502 const destination = databaseBackupFile(stagingDir, suffix);
503 await copyDatabaseFile(source, destination);
504 await syncFile(destination);
505 }
506 await NodeFSP.rename(stagingDir, backupDir);
507 await syncDirectory(NodePath.dirname(backupDir));
508 } catch (cause) {
509 await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
510 throw cause;
511 }
512}
513
514const restoreMarkerPath = (baseDir: string, updateId: string) =>
515 NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);
516
517const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>
518 pathExists(restoreMarkerPath(baseDir, pending.id));
519
520// Mark before live writes so recovery cannot boot a partial restore.
521async function markDatabaseRestorePending(backupDir: string): Promise<void> {
522 const markerPath = NodePath.join(backupDir, RESTORE_MARKER);
523 if (!(await pathExists(markerPath))) {
524 const handle = await NodeFSP.open(markerPath, "wx", 0o600);
525 try {
526 await handle.sync();
527 } finally {
528 await handle.close();
529 }
530 await syncDirectory(backupDir);
531 }
532}
533
534// Resolve symlink targets even if a crashed cp -c left the final target missing.
535async function restoreTarget(path: string): Promise<string> {
536 let current = path;
537 for (let hops = 0; hops < 40; hops++) {
538 try {
539 return await NodeFSP.realpath(current);
540 } catch (cause) {
541 if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
542 }
543 const link = await NodeFSP.readlink(current).catch((cause: unknown) => {
544 const code = (cause as NodeJS.ErrnoException).code;
545 if (code === "ENOENT" || code === "EINVAL") return undefined;
546 throw cause;
547 });
548 if (link === undefined) return current;
549 // Do not normalize .. before following directory symlinks; match filesystem resolution.
550 current = NodePath.isAbsolute(link)
551 ? link
552 : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`;
553 }
554 throw new Error(`Too many symbolic links resolving ${path}.`);
555}
556
557async function restoreDatabaseBackup(
558 baseDir: string,
559 pending: PendingServiceUpdate,
560): Promise<void> {
561 const backupDir = databaseBackupDir(baseDir, pending.id);
562 if (!(await pathExists(backupDir))) return;
563
564 await markDatabaseRestorePending(backupDir);
565 const directories = new Set([NodePath.dirname(pending.dbPath)]);
566 // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
567 const mainTarget = await restoreTarget(pending.dbPath);
568 for (const suffix of DB_FILE_SUFFIXES) {
569 const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
570 if (livePath !== `${pending.dbPath}${suffix}`) {
571 await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
572 }
573 const source = databaseBackupFile(backupDir, suffix);
574 if (await pathExists(source)) {
575 // Restore into the symlink target; cloning would replace the link.
576 const target = await restoreTarget(livePath);
577 await copyDatabaseFile(source, target);
578 await syncFile(target);
579 directories.add(NodePath.dirname(target));
580 } else {
581 await NodeFSP.rm(livePath, { force: true });
582 }
583 }
584 for (const directory of directories) await syncDirectory(directory);
585}
586
587async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
588 const backupDir = databaseBackupDir(baseDir, updateId);
589 if (!(await pathExists(backupDir))) return;
590 await NodeFSP.rm(backupDir, { recursive: true, force: true });
591 await syncDirectory(NodePath.dirname(backupDir));
592}
593
594export async function readServiceState(filePath: string): Promise<ServiceState> {
595 const contents = await NodeFSP.readFile(filePath, "utf8");
596 const state = parseServiceState(contents) ?? adoptLegacyServiceState(contents);
597 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
598 return state;
599}
600
601// An older coordinator can write protocol 2 beside a newer runtime. Adopt
602// only its install record, in memory: rewriting it outside the selection lock
603// could replace state a concurrent install wrote after this read.
604function adoptLegacyServiceState(contents: string): ServiceState | undefined {
605 const legacy = parseLegacyServiceState(contents);
606 return legacy && { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: legacy.activeVersion };
607}
608
609export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
610 await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 });
611 await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`);
612}
613
614async function replaceFileDurably(filePath: string, contents: string): Promise<void> {
615 const directory = NodePath.dirname(filePath);
616 const tempPath = NodePath.join(
617 directory,
618 `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
619 );
620 let handle: NodeFSP.FileHandle | undefined;
621 try {
622 handle = await NodeFSP.open(tempPath, "wx", 0o600);
623 await handle.writeFile(contents, "utf8");
624 await handle.sync();
625 await handle.close();
626 handle = undefined;
627 await NodeFSP.rename(tempPath, filePath);
628 await syncDirectory(directory);
629 } finally {
630 await handle?.close().catch(() => undefined);
631 await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined);
632 }
633}
634
635async function runtimeExists(baseDir: string, version: string): Promise<boolean> {
636 const paths = runtimePaths(baseDir, version);
637 try {
638 const [entry, sentinel] = await Promise.all([
639 NodeFSP.stat(paths.entryPath),
640 NodeFSP.readFile(paths.sentinelPath, "utf8"),
641 ]);
642 return entry.isFile() && sentinel.trim() === version;
643 } catch {
644 return false;
645 }
646}
647
648function terminalUpdate<S extends TerminalStatus>(input: {
649 readonly pending: PendingServiceUpdate;
650 readonly status: S;
651 readonly reason?: string;
652}): Exclude<ServiceUpdateRecord, PendingServiceUpdate> & { readonly status: S } {
653 return {
654 id: input.pending.id,
655 fromVersion: input.pending.fromVersion,
656 targetVersion: input.pending.targetVersion,
657 status: input.status,
658 ...(input.reason === undefined ? {} : { reason: input.reason }),
659 };
660}
661
662function sendMessage(
663 child: NodeChildProcess.ChildProcess,
664 message: ServiceLauncherParentMessage,
665): Promise<void> {
666 return new Promise((resolve, reject) => {
667 if (!child.connected || child.send === undefined) {
668 reject(new Error("service child IPC is disconnected."));
669 return;
670 }
671 child.send(message, (error) => (error === null ? resolve() : reject(error)));
672 });
673}
674
675function waitForExit(child: NodeChildProcess.ChildProcess): Promise<void> {
676 if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
677 return new Promise((resolve) => child.once("exit", () => resolve()));
678}
679
680async function terminateChild(
681 child: NodeChildProcess.ChildProcess,
682 signal: NodeJS.Signals,
683 overIpc: boolean,
684): Promise<void> {
685 if (child.exitCode !== null || child.signalCode !== null) return;
686 if (overIpc) {
687 await sendMessage(child, { type: "stop" }).catch(() => child.kill(signal));
688 } else {
689 child.kill(signal);
690 }
691 const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS);
692 try {
693 await waitForExit(child);
694 } finally {
695 clearTimeout(force);
696 }
697}
698
699/** Task Scheduler names are machine-wide, so each Windows account runs its own task. */
700export const serviceTaskName = (domain: string, userName: string) => `t3code-${domain}-${userName}`;
701
702export const serviceLauncherLockPath = (baseDir: string) =>
703 NodePath.join(baseDir, "runtime", SERVICE_LAUNCHER_LOCK_FILE);
704
705// An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies.
706async function tryAcquireLauncherLock(
707 lockPath: string,
708): Promise<{ readonly close: () => void } | undefined> {
709 const { DatabaseSync } = await import("node:sqlite");
710 const db = new DatabaseSync(lockPath);
711 try {
712 db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
713 return db;
714 } catch (cause) {
715 db.close();
716 if (cause instanceof Error && "errcode" in cause && cause.errcode === 5) return undefined;
717 throw cause;
718 }
719}
720
721export async function serviceLauncherIsRunning(lockPath: string): Promise<boolean> {
722 if (!NodeFS.existsSync(lockPath)) return false;
723 const lock = await tryAcquireLauncherLock(lockPath);
724 lock?.close();
725 return lock === undefined;
726}
727
728/** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */
729export async function waitForServiceLauncherExit(
730 lockPath: string,
731 signal: AbortSignal,
732): Promise<void> {
733 while (await serviceLauncherIsRunning(lockPath)) {
734 signal.throwIfAborted();
735 await new Promise((resolve) => setTimeout(resolve, 100));
736 }
737}
738
739const stopMarkerPath = (baseDir: string) =>
740 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
741const restartPendingPath = (baseDir: string) =>
742 NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE);
743
744export class Launcher {
745 readonly #baseDir: string;
746 readonly #statePath: string;
747 #state: ServiceState;
748 #child: ManagedChild | null = null;
749 #timer: NodeJS.Timeout | undefined;
750 #transitions: Promise<void> = Promise.resolve();
751 #stopRequested = false;
752 #stopping = false;
753 #done = false;
754 readonly #completion = Promise.withResolvers<void>();
755 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
756 readonly #serviceUnit: ServiceUnit | undefined;
757 readonly #environment: Readonly<NodeJS.ProcessEnv>;
758 readonly #output: number | undefined;
759 readonly #lockPath: string | undefined;
760 readonly #stopChildOverIpc: boolean;
761
762 constructor(
763 baseDir: string,
764 state: ServiceState,
765 options: {
766 readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
767 readonly serviceUnit?: ServiceUnit;
768 readonly environment?: Readonly<NodeJS.ProcessEnv>;
769 /** Launcher and server output, for managers that cannot redirect it. */
770 readonly output?: number;
771 /** Makes this the only launcher for its home; a replacement waits for the one stopping. */
772 readonly lockPath?: string;
773 /** Windows `kill()` is TerminateProcess, which skips the server's shutdown finalizers. */
774 readonly stopChildOverIpc?: boolean;
775 } = {},
776 ) {
777 this.#baseDir = baseDir;
778 this.#backupSpaceProbe = options.backupSpaceProbe;
779 this.#serviceUnit = options.serviceUnit;
780 this.#environment = { ...(options.environment ?? process.env) };
781 this.#output = options.output;
782 this.#lockPath = options.lockPath;
783 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
784 this.#stopChildOverIpc = options.stopChildOverIpc ?? process.platform === "win32";
785 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
786 this.#state = state;
787 }
788
789 async run(): Promise<void> {
790 const onSigterm = () => void this.stop("SIGTERM");
791 const onSigint = () => void this.stop("SIGINT");
792 process.once("SIGTERM", onSigterm);
793 process.once("SIGINT", onSigint);
794 try {
795 const lock =
796 this.#lockPath === undefined ? undefined : await this.#acquireLock(this.#lockPath);
797 try {
798 // The launcher that held the lock may have moved the state since main read it.
799 if (lock !== undefined) this.#state = await readServiceState(this.#statePath);
800 // A stop that arrived before this point leaves nothing to start or recover.
801 if (!this.#stopRequested) this.#enqueue(() => this.#recover());
802 await this.#completion.promise;
803 } finally {
804 if (lock !== undefined) {
805 // Releasing the lock says this launcher is done with the home.
806 await this.#transitions.catch(() => undefined);
807 lock.close();
808 }
809 }
810 } finally {
811 process.off("SIGTERM", onSigterm);
812 process.off("SIGINT", onSigint);
813 }
814 }
815
816 async #acquireLock(lockPath: string): Promise<{ readonly close: () => void } | undefined> {
817 for (let attempt = 1; !this.#stopRequested; attempt++) {
818 const lock = await tryAcquireLauncherLock(lockPath);
819 if (lock !== undefined) return lock;
820 if (attempt >= LOCK_ATTEMPTS) {
821 throw new Error("Another T3 Code service launcher still runs for this T3 home.");
822 }
823 await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS));
824 }
825 return undefined;
826 }
827
828 #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") {
829 return terminateChild(child, signal, this.#stopChildOverIpc);
830 }
831
832 #log(line: string): void {
833 if (this.#output === undefined) process.stderr.write(line);
834 else NodeFS.writeSync(this.#output, line);
835 }
836
837 #enqueue(transition: () => Promise<void>): void {
838 this.#transitions = this.#transitions
839 .then(transition, transition)
840 .catch((cause: unknown) =>
841 this.#fatal(cause instanceof Error ? cause : new Error(String(cause))),
842 );
843 }
844
845 async #fatal(error: Error): Promise<void> {
846 if (this.#done) return;
847 this.#done = true;
848 this.#stopping = true;
849 this.#clearTimer();
850 const child = this.#child?.process;
851 this.#child = null;
852 if (child !== undefined) await this.#terminate(child);
853 this.#completion.reject(error);
854 }
855
856 async stop(signal: NodeJS.Signals): Promise<void> {
857 // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer.
858 try {
859 NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 });
860 } catch {}
861 if (this.#stopRequested || this.#stopping) {
862 await this.#completion.promise.catch(() => undefined);
863 return;
864 }
865 this.#stopRequested = true;
866 this.#clearTimer();
867 this.#enqueue(async () => {
868 // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup.
869 this.#stopping = true;
870 const child = this.#child?.process;
871 this.#child = null;
872 if (child !== undefined) await this.#terminate(child, signal);
873 this.#done = true;
874 this.#completion.resolve();
875 });
876 await this.#completion.promise.catch(() => undefined);
877 }
878
879 #clearTimer(): void {
880 clearTimeout(this.#timer);
881 this.#timer = undefined;
882 }
883
884 async #selectState(next: ServiceState): Promise<void> {
885 const release = await acquireServiceSelectionLock(this.#statePath);
886 try {
887 const selected = await readServiceState(this.#statePath);
888 // Local installs replace the selection and clear its update record. An old
889 // launcher must not restore its own selection when asked to update again.
890 if (
891 selected.activeVersion !== this.#state.activeVersion ||
892 selected.update?.id !== this.#state.update?.id ||
893 selected.update?.status !== this.#state.update?.status
894 ) {
895 throw new Error("Service selection changed. Restart the service before updating.");
896 }
897 await writeServiceState(this.#statePath, next);
898 this.#state = next;
899 } finally {
900 release();
901 }
902 }
903
904 async #recover(): Promise<void> {
905 // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
906 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
907 const restartPending = restartPendingPath(this.#baseDir);
908 try {
909 const release = await acquireServiceSelectionLock(this.#statePath);
910 try {
911 const selected = await readServiceState(this.#statePath);
912 const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(
913 () => undefined,
914 );
915 const awaited = awaitedVersion?.trim();
916 if (
917 selected.activeVersion === this.#state.activeVersion &&
918 selected.update?.id === this.#state.update?.id &&
919 selected.update?.status === this.#state.update?.status &&
920 awaited !== undefined &&
921 isExactServiceVersion(awaited) &&
922 compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0
923 ) {
924 await NodeFSP.rm(restartPending, { force: true });
925 }
926 } finally {
927 release();
928 }
929 } catch (error) {
930 process.stderr.write(
931 `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`,
932 );
933 }
934 const update = this.#state.update;
935 if (update?.status !== "pending") {
936 if (update !== undefined) {
937 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
938 }
939 try {
940 await this.#startChild(this.#state.activeVersion, "active", update);
941 } finally {
942 if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion);
943 }
944 return;
945 }
946 if (await databaseRestorePending(this.#baseDir, update)) {
947 await this.#returnToPrevious(update, "failed", "rollback-interrupted");
948 return;
949 }
950 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {
951 await this.#returnToPrevious(update, "failed", "target-runtime-missing");
952 return;
953 }
954 await this.#startTrial(update);
955 }
956
957 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
958 try {
959 await backupDatabaseOnce(this.#baseDir, pending);
960 } catch {
961 await this.#returnToPrevious(pending, "failed", "db-backup-failed");
962 return;
963 }
964 try {
965 await this.#startChild(pending.targetVersion, "trial", pending);
966 } catch {
967 await this.#returnToPrevious(pending, "failed", "candidate-start-failed");
968 }
969 }
970
971 async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
972 if (this.#stopping) return;
973 if (!(await runtimeExists(this.#baseDir, version))) {
974 throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
975 }
976 if (this.#stopping) return;
977 const paths = runtimePaths(this.#baseDir, version);
978 const context: ServiceLauncherContext = {
979 protocol: SERVICE_LAUNCHER_PROTOCOL,
980 childVersion: version,
981 ...(update === undefined ? {} : { update }),
982 };
983 const spawnArguments = runtimeSpawnArguments(paths);
984 const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, {
985 env: { ...this.#environment, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) },
986 stdio:
987 this.#output === undefined
988 ? ["inherit", "inherit", "inherit", "ipc"]
989 : ["ignore", this.#output, this.#output, "ipc"],
990 });
991 await new Promise<void>((resolve, reject) => {
992 const onError = (error: Error) => reject(error);
993 child.once("error", onError);
994 child.once("spawn", () => {
995 child.removeListener("error", onError);
996 child.on("error", (error) => this.#enqueue(() => Promise.reject(error)));
997 resolve();
998 });
999 });
1000 if (this.#stopping) {
1001 await this.#terminate(child);
1002 return;
1003 }
1004
1005 const managed: ManagedChild = {
1006 version,
1007 role,
1008 process: child,
1009 };
1010 this.#child = managed;
1011 child.on("message", (value) => {
1012 const message = decodeServiceLauncherChildMessage(value);
1013 if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message));
1014 });
1015 child.once("exit", (code, signal) =>
1016 this.#enqueue(() => this.#handleExit(managed, code, signal)),
1017 );
1018
1019 if (role === "trial") {
1020 this.#timer = setTimeout(
1021 () => this.#enqueue(() => this.#handlePreparedTimeout(managed)),
1022 PREPARED_TIMEOUT_MS,
1023 );
1024 }
1025 }
1026
1027 async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise<void> {
1028 if (this.#child !== child || this.#stopping) return;
1029 if (message.type === "request-update") {
1030 await this.#handleUpdateRequest(child, message);
1031 return;
1032 }
1033 await this.#handlePrepared(child, message.updateId);
1034 }
1035
1036 async #handleUpdateRequest(
1037 child: ManagedChild,
1038 message: Extract<ServiceLauncherChildMessage, { readonly type: "request-update" }>,
1039 ): Promise<void> {
1040 const reject = (reason: string) =>
1041 sendMessage(child.process, { type: "update-rejected", reason });
1042 if (child.role !== "active") {
1043 await reject("Only the active server can request an update.");
1044 return;
1045 }
1046 if (child.version !== this.#state.activeVersion) {
1047 await reject("The requesting server is not the selected active version.");
1048 return;
1049 }
1050 if (this.#state.update?.status === "pending") {
1051 await reject("Another server update is already pending.");
1052 return;
1053 }
1054 if (!isExactServiceVersion(message.targetVersion)) {
1055 await reject("The requested target is not an exact version.");
1056 return;
1057 }
1058 if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) {
1059 await reject("Remote updates must select a newer server version.");
1060 return;
1061 }
1062 if (!NodePath.isAbsolute(message.dbPath)) {
1063 await reject("The requested database path is not absolute.");
1064 return;
1065 }
1066 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {
1067 await reject("The requested target runtime is missing or incomplete.");
1068 return;
1069 }
1070 // Failed checks must refuse updates rather than stop the server with unknown backup capacity.
1071 const backupBlocker = await databaseBackupBlocker(
1072 this.#baseDir,
1073 message.dbPath,
1074 this.#backupSpaceProbe,
1075 ).catch(
1076 (error: unknown) =>
1077 `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`,
1078 );
1079 if (!child.process.connected) return;
1080 if (backupBlocker !== undefined) {
1081 await reject(backupBlocker);
1082 return;
1083 }
1084
1085 const pending: PendingServiceUpdate = {
1086 id: NodeCrypto.randomUUID(),
1087 fromVersion: child.version,
1088 targetVersion: message.targetVersion,
1089 dbPath: message.dbPath,
1090 status: "pending",
1091 };
1092 const next: ServiceState = { ...this.#state, update: pending };
1093 try {
1094 await this.#selectState(next);
1095 } catch (error) {
1096 await reject(error instanceof Error ? error.message : String(error));
1097 return;
1098 }
1099 await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
1100 this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
1101 }
1102
1103 async #beginTrial(child: ManagedChild): Promise<void> {
1104 const pending = this.#state.update;
1105 if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") {
1106 return;
1107 }
1108 this.#timer = undefined;
1109 // Anything thrown outside its own fallback still leaves the offline backup to run.
1110 const finishOnlineBackup = await startOnlineDatabaseBackup(
1111 this.#baseDir,
1112 pending,
1113 this.#backupSpaceProbe,
1114 ).catch(() => undefined);
1115 this.#child = null;
1116 await this.#terminate(child.process);
1117 await finishOnlineBackup?.();
1118 // A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
1119 if (this.#stopRequested) return;
1120 await this.#startTrial(pending);
1121 }
1122
1123 async #handlePrepared(child: ManagedChild, updateId: string): Promise<void> {
1124 const pending = this.#state.update;
1125 if (
1126 child.role !== "trial" ||
1127 pending?.status !== "pending" ||
1128 pending.id !== updateId ||
1129 pending.targetVersion !== child.version
1130 ) {
1131 if (child.role === "trial" && pending?.status === "pending") {
1132 await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child);
1133 return;
1134 }
1135 throw new Error("Trial child reported prepared for an unexpected update.");
1136 }
1137 this.#clearTimer();
1138 const committed = terminalUpdate({ pending, status: "committed" });
1139 const next: ServiceState = {
1140 ...this.#state,
1141 activeVersion: pending.targetVersion,
1142 update: committed,
1143 };
1144 await this.#selectState(next);
1145 child.role = "active";
1146 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
1147 try {
1148 await sendMessage(child.process, { type: "committed", updateId: committed.id });
1149 } finally {
1150 await this.#repointServiceUnit(committed.targetVersion);
1151 }
1152 }
1153
1154 // Selection is durable before this best-effort unit reconciliation.
1155 async #repointServiceUnit(version: string): Promise<void> {
1156 const unit = this.#serviceUnit;
1157 if (unit === undefined) return;
1158 try {
1159 const release = await acquireServiceSelectionLock(unit.path);
1160 try {
1161 // The installer writes state before the unit under this lease. Do not
1162 // acquire its state lease here: the installer takes that lease first.
1163 const selected = await readServiceState(this.#statePath);
1164 if (
1165 selected.activeVersion !== this.#state.activeVersion ||
1166 selected.update?.id !== this.#state.update?.id ||
1167 selected.update?.status !== this.#state.update?.status
1168 ) {
1169 return;
1170 }
1171 const contents = await NodeFSP.readFile(unit.path, "utf8");
1172 const installedBaseDir = serviceUnitBaseDirOf(contents);
1173 if (
1174 installedBaseDir === undefined ||
1175 NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir)
1176 ) {
1177 return;
1178 }
1179 const repointed = repointServiceUnit(contents, version);
1180 if (repointed !== contents) await replaceFileDurably(unit.path, repointed);
1181 // A previous rename may have succeeded before reload failed.
1182 await unit.reload();
1183 } finally {
1184 release();
1185 }
1186 } catch (error) {
1187 process.stderr.write(
1188 `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`,
1189 );
1190 }
1191 }
1192
1193 async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
1194 const pending = this.#state.update;
1195 if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") {
1196 return;
1197 }
1198 this.#timer = undefined;
1199 await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child);
1200 }
1201
1202 async #handleExit(
1203 child: ManagedChild,
1204 code: number | null,
1205 signal: NodeJS.Signals | null,
1206 ): Promise<void> {
1207 if (this.#child !== child || this.#stopping) return;
1208 this.#child = null;
1209 if (code === STATE_DIR_OWNED_EXIT_CODE) {
1210 await this.#idleWhileStateDirOwned();
1211 return;
1212 }
1213 if (child.role === "trial") {
1214 this.#clearTimer();
1215 const pending = this.#state.update;
1216 if (pending?.status !== "pending") {
1217 throw new Error("Trial child exited without matching pending state.");
1218 }
1219 await this.#returnToPrevious(
1220 pending,
1221 "rolled-back",
1222 `candidate-exited:${String(code ?? signal ?? "unknown")}`,
1223 );
1224 return;
1225 }
1226
1227 this.#clearTimer();
1228 const pending = this.#state.update;
1229 if (pending?.status === "pending") {
1230 await this.#startTrial(pending);
1231 return;
1232 }
1233 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
1234 }
1235
1236 // Stay idle to avoid supervisor restart loops; never restore backups over another owner's possible writes.
1237 async #idleWhileStateDirOwned(): Promise<void> {
1238 this.#clearTimer();
1239 const pending = this.#state.update;
1240 if (pending?.status === "pending") {
1241 const next: ServiceState = {
1242 ...this.#state,
1243 activeVersion: pending.fromVersion,
1244 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
1245 };
1246 await this.#selectState(next);
1247 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1248 await this.#repointServiceUnit(next.activeVersion);
1249 }
1250 // Signal listeners alone do not keep Node alive.
1251 this.#timer = setInterval(() => {}, 2_147_483_647);
1252 this.#log(
1253 "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n",
1254 );
1255 }
1256
1257 async #returnToPrevious(
1258 pending: PendingServiceUpdate,
1259 status: "rolled-back" | "failed",
1260 reason: string,
1261 child?: ManagedChild,
1262 ): Promise<void> {
1263 if (child !== undefined) {
1264 this.#child = null;
1265 await this.#terminate(child.process);
1266 }
1267 await restoreDatabaseBackup(this.#baseDir, pending);
1268 const outcome = terminalUpdate({ pending, status, reason });
1269 const next: ServiceState = {
1270 ...this.#state,
1271 activeVersion: pending.fromVersion,
1272 update: outcome,
1273 };
1274 await this.#selectState(next);
1275 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1276 try {
1277 await this.#startChild(next.activeVersion, "active", outcome);
1278 } finally {
1279 await this.#repointServiceUnit(next.activeVersion);
1280 }
1281 }
1282}
1283
1284/** Task Scheduler cannot set variables or redirect output, so a Windows task passes them as flags. */
1285export interface LauncherOptions {
1286 readonly baseDir?: string | undefined;
1287 readonly logFile?: string | undefined;
1288 readonly serviceUnit?: string | undefined;
1289}
1290
1291export async function main(options: LauncherOptions = {}): Promise<void> {
1292 const baseDir = (options.baseDir ?? process.env.T3CODE_HOME)?.trim();
1293 if (baseDir === undefined || baseDir === "") {
1294 throw new Error("T3CODE_HOME is required by the T3 Code service launcher.");
1295 }
1296 const output =
1297 options.logFile === undefined ? undefined : NodeFS.openSync(options.logFile, "a", 0o600);
1298 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
1299 const windows = process.platform === "win32";
1300 let logTimer: NodeJS.Timeout | undefined;
1301 try {
1302 // What the service manager started the launcher with. It locates the unit; service.env
1303 // changes only what the server gets.
1304 const managerEnvironment = {
1305 ...process.env,
1306 T3CODE_HOME: baseDir,
1307 ...(options.serviceUnit === undefined ? {} : { [SERVICE_UNIT_ENV]: options.serviceUnit }),
1308 };
1309 const environment = await readServiceEnvironment(baseDir, managerEnvironment);
1310 const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
1311 const contents = await NodeFSP.readFile(statePath, "utf8");
1312 let state = parseServiceState(contents);
1313 if (state === undefined) {
1314 state = adoptLegacyServiceState(contents);
1315 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
1316 if (!(await runtimeExists(baseDir, state.activeVersion))) {
1317 throw new Error(`Selected t3@${state.activeVersion} runtime is missing or incomplete.`);
1318 }
1319 }
1320 const serviceUnit = serviceUnitFromEnvironment(managerEnvironment);
1321 const logPath = await serviceLogPath(options, serviceUnit);
1322 if (logPath !== undefined) {
1323 await boundServiceLog(logPath);
1324 logTimer = setInterval(
1325 () => void boundServiceLog(logPath),
1326 SERVICE_LOG_CHECK_INTERVAL_MS,
1327 ).unref();
1328 }
1329 const launcher = new Launcher(baseDir, state, {
1330 environment,
1331 ...(serviceUnit === undefined ? {} : { serviceUnit }),
1332 ...(output === undefined ? {} : { output }),
1333 ...(windows ? { lockPath: serviceLauncherLockPath(baseDir) } : {}),
1334 });
1335 // Ending a scheduled task kills its console host without a signal; the console read fails instead.
1336 const watchConsole = windows && process.stdin.isTTY === true;
1337 const onConsoleLost = () => void launcher.stop("SIGTERM");
1338 if (watchConsole) {
1339 process.stdin.on("error", () => undefined);
1340 process.stdin.once("close", onConsoleLost);
1341 process.stdin.resume();
1342 }
1343 try {
1344 await launcher.run();
1345 } finally {
1346 if (watchConsole) {
1347 process.stdin.off("close", onConsoleLost);
1348 process.stdin.destroy();
1349 }
1350 }
1351 } catch (cause) {
1352 if (output !== undefined) {
1353 NodeFS.writeSync(
1354 output,
1355 `[service-launcher] ${cause instanceof Error ? cause.message : String(cause)}\n`,
1356 );
1357 }
1358 // Agents the server left behind keep the console host, and so the task, running; ending it
1359 // lets the next repeating trigger start the service again.
1360 if (windows && options.serviceUnit !== undefined) {
1361 const taskName = serviceTaskName(process.env.USERDOMAIN ?? "", process.env.USERNAME ?? "");
1362 NodeChildProcess.spawnSync("schtasks", ["/End", "/TN", taskName], {
1363 stdio: "ignore",
1364 windowsHide: true,
1365 });
1366 }
1367 throw cause;
1368 } finally {
1369 clearInterval(logTimer);
1370 if (output !== undefined) NodeFS.closeSync(output);
1371 }
1372}