Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

serviceLauncher.ts

1,458 lines55.2 KB

1// @effect-diagnostics nodeBuiltinImport:off
2// @effect-diagnostics globalTimers:off
3// The launcher must survive unloadable server versions; use Node built-ins without Effect.
4import * as NodeChildProcess from "node:child_process";
5import * as NodeCrypto from "node:crypto";
6import * as NodeFS from "node:fs";
7import * as NodeFSP from "node:fs/promises";
8import * as NodePath from "node:path";
9import * as NodeUtil from "node:util";
10import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts";
11​
12import type {
13 PendingServiceUpdate,
14 ServiceLauncherChildMessage,
15 ServiceLauncherContext,
16 ServiceLauncherParentMessage,
17 ServiceState,
18 ServiceUpdateRecord,
19} from "./cloud/serviceProtocol.ts";
20import {
21 compareExactServiceVersions,
22 decodeServiceLauncherChildMessage,
23 isExactServiceVersion,
24 parseLegacyServiceState,
25 disabledServiceTask,
26 parseServiceState,
27 serviceTaskIsDisabled,
28 serviceUnitBaseDirOf,
29 serviceUnitLogPathOf,
30 SERVICE_ENV_FILE,
31 SERVICE_LAUNCHER_LOCK_FILE,
32 SERVICE_LAUNCHER_CONTEXT_ENV,
33 SERVICE_LAUNCHER_PROTOCOL,
34 SERVICE_STATE_FILE,
35 SERVICE_RESTART_PENDING_FILE,
36 SERVICE_STOP_MARKER_FILE,
37 SERVICE_UNIT_ENV,
38} from "./cloud/serviceProtocol.ts";
39import {
40 legacyOwnerStartedBeforeRecord,
41 legacyOwnerStartTimeCommand,
42} from "./legacyServerOwner.ts";
43​
44const HANDOFF_DELAY_MS = 2_000;
45/**
46 * Migrations count toward it and can scale with the database: tens of GB on a cold page cache
47 * take minutes. A missed window rolls back the database, so the next attempt pays the same cost.
48 */
49const PREPARED_TIMEOUT_MS = 30 * 60_000;
50const TERMINATE_GRACE_MS = 5_000;
51// A replacement started by a handoff waits up to two minutes for the stopping launcher.
52const LOCK_RETRY_MS = 500;
53const LOCK_ATTEMPTS = 240;
54// Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect.
55const STATE_DIR_OWNED_EXIT_CODE = 78;
56const STATE_DIR_OWNER_POLL_MS = 30_000;
57​
58type TerminalStatus = "committed" | "rolled-back" | "failed";
59type ChildRole = "active" | "trial";
60​
61interface ManagedChild {
62 readonly version: string;
63 role: ChildRole;
64 readonly process: NodeChildProcess.ChildProcess;
66​
67const runtimePaths = (baseDir: string, version: string) => {
68 const versionDir = NodePath.join(baseDir, "runtime", "versions", version);
69 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
70 const executableName = process.platform === "win32" ? "t3.exe" : "t3";
71 return {
72 versionDir,
73 entryPath: NodePath.join(versionDir, executableName),
74 sentinelPath: NodePath.join(versionDir, ".install-complete"),
75 };
76};
77​
78const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
79 command: paths.entryPath,
80 args: ["serve"],
81});
82​
83/**
84 * The unit or launch agent that `t3 service install` wrote. Its program names
85 * the runtime that ran the install, and a remote update never replaces it, so
86 * after a commit the launcher points it at the committed runtime. Otherwise the
87 * next start lands on a runtime that nothing in the service state records.
88 */
89export interface ServiceUnit {
90 readonly path: string;
91 /** systemd and Task Scheduler keep a loaded copy; launchd re-reads the plist at bootstrap. */
92 readonly reload: () => Promise<void>;
94​
95const runSchtasks = (...args: ReadonlyArray<string>) =>
96 NodeUtil.promisify(NodeChildProcess.execFile)("schtasks", args, {
97 windowsHide: true,
98 timeout: 30_000,
99 });
100​
101/**
102 * Registering replaces the task, so one the CLI disabled to keep the service stopped is
103 * registered disabled. The caller holds the unit lease, as the CLI does to disable it.
104 */
105export async function registerServiceTask(
106 unitPath: string,
107 taskName: string,
108 schtasks: (...args: ReadonlyArray<string>) => Promise<{ readonly stdout: string }> = runSchtasks,
109): Promise<void> {
110 const register = (xmlPath: string) => schtasks("/Create", "/XML", xmlPath, "/TN", taskName, "/F");
111 const registered = await schtasks("/Query", "/TN", taskName, "/XML");
112 if (!serviceTaskIsDisabled(registered.stdout)) {
113 await register(unitPath);
114 return;
115 }
116 const disabledPath = `${unitPath}.${process.pid}.disabled`;
117 try {
118 const unit = await NodeFSP.readFile(unitPath, "utf8");
119 await NodeFSP.writeFile(disabledPath, disabledServiceTask(unit));
120 await register(disabledPath);
121 } finally {
122 await NodeFSP.rm(disabledPath, { force: true });
123 }
125​
126function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined {
127 const unitName = env[SERVICE_UNIT_ENV]?.trim();
128 if (!unitName) return undefined;
129 if (unitName.endsWith(".xml")) {
130 const localAppData = env.LOCALAPPDATA?.trim();
131 if (!localAppData) return undefined;
132 const path = NodePath.win32.join(localAppData, "t3code", unitName);
133 const taskName = serviceTaskName(env.USERDOMAIN ?? "", env.USERNAME ?? "");
134 return { path, reload: () => registerServiceTask(path, taskName) };
135 }
136 const home = env.HOME?.trim();
137 if (!home) return undefined;
138 if (unitName.endsWith(".plist")) {
139 return {
140 path: NodePath.join(home, "Library", "LaunchAgents", unitName),
141 reload: () => Promise.resolve(),
142 };
143 }
144 // The manager's own environment, which decides where it reads user units.
145 const xdgConfigHome = env.XDG_CONFIG_HOME?.trim();
146 const configHome =
147 xdgConfigHome && NodePath.isAbsolute(xdgConfigHome)
148 ? xdgConfigHome
149 : NodePath.join(home, ".config");
150 return {
151 path: NodePath.join(configHome, "systemd", "user", unitName),
152 reload: () =>
153 new Promise((resolve, reject) => {
154 NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], {
155 stdio: "ignore",
156 timeout: 30_000,
157 })
158 .once("error", reject)
159 .once("exit", (code, signal) =>
160 code === 0
161 ? resolve()
162 : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)),
163 );
164 }),
165 };
167​
168// systemd's ExecStart value, or the plist string or task argument before __service-launcher.
169// Only the version segment changes, so quoted or escaped paths stay as rendered.
170// The launch agent's PATH names a versions directory without the executable; leave it.
171const SERVICE_UNIT_PROGRAMS = [
172 /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m,
173 /(<string>[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*<string>__service-launcher<\/string>)/,
174 /(<Arguments>--headless "[^"<]*?[\\/]runtime[\\/]versions[\\/])([^\\/"<]+)([\\/]t3\.exe" "__service-launcher")/,
175] as const;
176​
177/**
178 * The unit contents with their program pointed at `version`'s runtime. A unit
179 * already at that version or a newer one is returned unchanged: `t3 update`
180 * may have repointed it to a later version while this commit was in flight.
181 */
182function repointServiceUnit(contents: string, version: string): string {
183 for (const program of SERVICE_UNIT_PROGRAMS) {
184 const match = program.exec(contents);
185 if (match === null) continue;
186 const [, prefix = "", current = "", suffix = ""] = match;
187 if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) {
188 return contents;
189 }
190 return contents.replace(program, () => `${prefix}${version}${suffix}`);
191 }
192 return contents;
194​
195const SERVICE_LOG_MAX_BYTES = 10 * 1024 * 1024;
196const SERVICE_LOG_CHECK_INTERVAL_MS = 10 * 60_000;
197​
198/**
199 * The service manager holds the log open for appending, so it is copied aside and emptied in
200 * place rather than renamed. The copy is best effort: on a full disk the truncate frees space.
201 */
202async function boundServiceLog(logPath: string): Promise<void> {
203 const size = await NodeFSP.stat(logPath).then(
204 (stats) => stats.size,
205 () => 0,
206 );
207 if (size <= SERVICE_LOG_MAX_BYTES) return;
208 const archivePath = `${logPath}.1`;
209 const pendingArchivePath = `${archivePath}.tmp`;
210 await NodeFSP.copyFile(logPath, pendingArchivePath)
211 .then(() => NodeFSP.rename(pendingArchivePath, archivePath))
212 .catch(() => NodeFSP.rm(pendingArchivePath, { force: true }).catch(() => undefined));
213 await NodeFSP.truncate(logPath, 0).catch(() => undefined);
215​
216async function serviceLogPath(
217 options: LauncherOptions,
218 serviceUnit: ServiceUnit | undefined,
219): Promise<string | undefined> {
220 if (options.logFile !== undefined) return options.logFile;
221 if (serviceUnit === undefined) return undefined;
222 const contents = await NodeFSP.readFile(serviceUnit.path, "utf8").catch(() => undefined);
223 return contents === undefined ? undefined : serviceUnitLogPathOf(contents);
225​
226const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
227const RESTORE_MARKER = ".restore-pending";
228​
229const databaseBackupDir = (baseDir: string, updateId: string) =>
230 NodePath.join(baseDir, "runtime", "db-backup", updateId);
231​
232const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>
233 NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);
234​
235async function pathExists(target: string): Promise<boolean> {
236 try {
237 await NodeFSP.access(target);
238 return true;
239 } catch (cause) {
240 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;
241 throw cause;
242 }
244​
245const MANAGED_SERVICE_ENV_NAMES = new Set([
246 "PATH",
247 "T3CODE_HOME",
248 "T3_BOOT_SERVICE_UNIT",
249 SERVICE_LAUNCHER_CONTEXT_ENV,
250]);
251​
252export async function readServiceEnvironment(
253 baseDir: string,
254 inherited: NodeJS.ProcessEnv = process.env,
255): Promise<NodeJS.ProcessEnv> {
256 const environment = { ...inherited };
257 let contents: string;
258 try {
259 contents = await NodeFSP.readFile(NodePath.join(baseDir, SERVICE_ENV_FILE), "utf8");
260 } catch (cause) {
261 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return environment;
262 throw cause;
263 }
264 for (const [name, value] of Object.entries(NodeUtil.parseEnv(contents))) {
265 if (
266 /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
267 !MANAGED_SERVICE_ENV_NAMES.has(name.toUpperCase())
268 ) {
269 environment[name] = value;
270 }
271 }
272 return environment;
274​
275// Opened read-write: Windows refuses to flush a handle without write access.
276async function syncFile(filePath: string): Promise<void> {
277 const handle = await NodeFSP.open(filePath, "r+");
278 try {
279 await handle.sync();
280 } finally {
281 await handle.close();
282 }
284​
285// Fsync makes renames durable; Windows returns EPERM and NTFS journals them itself.
286async function syncDirectory(directory: string): Promise<void> {
287 const handle = await NodeFSP.open(directory, "r");
288 try {
289 await handle.sync();
290 } catch (error) {
291 if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error;
292 } finally {
293 await handle.close();
294 }
296​
297function runCp(args: ReadonlyArray<string>): Promise<boolean> {
298 return new Promise((resolve) => {
299 NodeChildProcess.execFile("cp", args, (error) => resolve(error === null));
300 });
302​
303// macOS fs.copyFile clone flags fall back to full copies.
304async function copyDatabaseFile(source: string, destination: string): Promise<void> {
305 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
306 const platform = process.platform;
307 // cp -c replaces files and breaks hard links; copy into hard-linked destinations instead.
308 const hardLinked =
309 platform === "darwin" &&
310 ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1;
311 const args =
312 platform === "darwin" && !hardLinked
313 ? ["-c", source, destination]
314 : platform === "linux"
315 ? ["--reflink=auto", source, destination]
316 : undefined;
317 if (args === undefined || !(await runCp(args))) await NodeFSP.copyFile(source, destination);
319​
320const GIB = 1024 ** 3;
321// Cloned backups still need headroom for trial writes and migrations to diverge.
322const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
323const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
324// While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
325const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
326​
327async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
328 const stats: Array<NodeFS.Stats> = [];
329 for (const suffix of DB_FILE_SUFFIXES) {
330 const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => {
331 if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined;
332 throw cause;
333 });
334 if (stat !== undefined) stats.push(stat);
335 }
336 return stats;
338​
339// cp -c silently copies across volumes; require one device matching the APFS system volume's filesystem type.
340async function backupCanClone(dbPath: string, backupRoot: string): Promise<boolean> {
341 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
342 const platform = process.platform;
343 if (platform === "darwin") {
344 const [backupDirectory, backupFs, systemFs] = await Promise.all([
345 NodeFSP.stat(backupRoot),
346 NodeFSP.statfs(backupRoot),
347 NodeFSP.statfs("/"),
348 ]);
349 if (backupFs.type !== systemFs.type) return false;
350 return (await statDatabaseFiles(dbPath)).every(
351 (source) => source.dev === backupDirectory.dev && source.nlink === 1,
352 );
353 }
354 if (platform !== "linux") return false;
355 const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`);
356 try {
357 for (const suffix of DB_FILE_SUFFIXES) {
358 const source = `${dbPath}${suffix}`;
359 if (suffix !== "" && !(await pathExists(source))) continue;
360 if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false;
361 }
362 return true;
363 } finally {
364 for (const suffix of DB_FILE_SUFFIXES) {
365 await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined);
366 }
367 }
369​
370export interface BackupSpaceProbe {
371 readonly readFreeBytes: (directory: string) => Promise<number>;
372 readonly canClone: (dbPath: string, backupRoot: string) => Promise<boolean>;
374​
375const filesystemBackupSpaceProbe: BackupSpaceProbe = {
376 readFreeBytes: async (directory) => {
377 const stats = await NodeFSP.statfs(directory);
378 return stats.bavail * stats.bsize;
379 },
380 canClone: backupCanClone,
381};
382​
383// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.
384// Update requests are refused while one is pending, so no staging directory is still in use here.
385async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {
386 for (const entry of await NodeFSP.readdir(backupRoot)) {
387 if (!entry.endsWith(".staging")) continue;
388 await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(
389 () => undefined,
390 );
391 }
393​
394/** Check before stopping the server so full disks refuse updates without an outage. */
395export async function databaseBackupBlocker(
396 baseDir: string,
397 dbPath: string,
398 probe: Partial<BackupSpaceProbe> = {},
399): Promise<string | undefined> {
400 const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
401 const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
402 await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
403 await removeAbandonedBackupCopies(backupRoot);
404 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
405 const size = files.reduce((total, file) => total + file.size, 0);
406 const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
407 if (free >= fullCopyBytes) return undefined;
408 const clonedBytes = Math.max(
409 CLONED_BACKUP_MIN_HEADROOM_BYTES,
410 size * CLONED_BACKUP_HEADROOM_FRACTION,
411 );
412 const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes;
413 if (free >= needed) return undefined;
414 const gb = (bytes: number) => (bytes / GIB).toFixed(1);
415 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
417​
418// A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
419async function copyWithoutOpening(source: string, destination: string): Promise<void> {
420 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
421 if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
422 if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
424​
425// Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
426// which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
427// `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
428// a write-heavy server's new writes wait, uncommitted, until the server stops.
429async function startOnlineDatabaseBackup(
430 baseDir: string,
431 pending: PendingServiceUpdate,
432 probe: Partial<BackupSpaceProbe> = {},
433): Promise<(() => Promise<void>) | undefined> {
434 const backupDir = databaseBackupDir(baseDir, pending.id);
435 if (await pathExists(backupDir)) return undefined;
436 const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
437 if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
438 return undefined;
439 const NodeSqlite = await import("node:sqlite").catch(() => undefined);
440 if (NodeSqlite === undefined) return undefined;
441​
442 const stagingDir = `${backupDir}.staging`;
443 const discardStaging = () =>
444 NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
445 const fallBack = async (cause: unknown) => {
446 await discardStaging();
447 process.stderr.write(
448 `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
449 );
450 return undefined;
451 };
452 let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
453 // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
454 let dbPath = pending.dbPath;
455 const release = () => {
456 try {
457 snapshot?.close();
458 } catch {}
459 snapshot = undefined;
460 };
461 try {
462 await discardStaging();
463 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
464 dbPath = await NodeFSP.realpath(pending.dbPath);
465 snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
466 const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
467 if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
468 snapshot.exec("BEGIN");
469 snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
470 const mainCopy = databaseBackupFile(stagingDir, "");
471 await copyWithoutOpening(dbPath, mainCopy);
472 await syncFile(mainCopy);
473 } catch (cause) {
474 release();
475 return fallBack(cause);
476 }
477 return async () => {
478 try {
479 const wal = `${dbPath}-wal`;
480 if (!(await pathExists(wal)))
481 throw new Error("the WAL disappeared while the snapshot was open.");
482 const walCopy = databaseBackupFile(stagingDir, "-wal");
483 await copyWithoutOpening(wal, walCopy);
484 await syncFile(walCopy);
485 release();
486 await NodeFSP.rename(stagingDir, backupDir);
487 await syncDirectory(NodePath.dirname(backupDir));
488 } catch (cause) {
489 release();
490 await fallBack(cause);
491 }
492 };
494​
495// Preserve completed backups; restarted trials may already have changed the live database.
496async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
497 const backupDir = databaseBackupDir(baseDir, pending.id);
498 if (await pathExists(backupDir)) return;
499​
500 const stagingDir = `${backupDir}.staging`;
501 await NodeFSP.rm(stagingDir, { recursive: true, force: true });
502 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
503 try {
504 for (const suffix of DB_FILE_SUFFIXES) {
505 const source = `${pending.dbPath}${suffix}`;
506 if (suffix !== "" && !(await pathExists(source))) continue;
507 const destination = databaseBackupFile(stagingDir, suffix);
508 await copyDatabaseFile(source, destination);
509 await syncFile(destination);
510 }
511 await NodeFSP.rename(stagingDir, backupDir);
512 await syncDirectory(NodePath.dirname(backupDir));
513 } catch (cause) {
514 await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
515 throw cause;
516 }
518​
519const restoreMarkerPath = (baseDir: string, updateId: string) =>
520 NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);
521​
522const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>
523 pathExists(restoreMarkerPath(baseDir, pending.id));
524​
525// Mark before live writes so recovery cannot boot a partial restore.
526async function markDatabaseRestorePending(backupDir: string): Promise<void> {
527 const markerPath = NodePath.join(backupDir, RESTORE_MARKER);
528 if (!(await pathExists(markerPath))) {
529 const handle = await NodeFSP.open(markerPath, "wx", 0o600);
530 try {
531 await handle.sync();
532 } finally {
533 await handle.close();
534 }
535 await syncDirectory(backupDir);
536 }
538​
539// Resolve symlink targets even if a crashed cp -c left the final target missing.
540async function restoreTarget(path: string): Promise<string> {
541 let current = path;
542 for (let hops = 0; hops < 40; hops++) {
543 try {
544 return await NodeFSP.realpath(current);
545 } catch (cause) {
546 if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
547 }
548 const link = await NodeFSP.readlink(current).catch((cause: unknown) => {
549 const code = (cause as NodeJS.ErrnoException).code;
550 if (code === "ENOENT" || code === "EINVAL") return undefined;
551 throw cause;
552 });
553 if (link === undefined) return current;
554 // Do not normalize .. before following directory symlinks; match filesystem resolution.
555 current = NodePath.isAbsolute(link)
556 ? link
557 : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`;
558 }
559 throw new Error(`Too many symbolic links resolving ${path}.`);
561​
562async function restoreDatabaseBackup(
563 baseDir: string,
564 pending: PendingServiceUpdate,
565): Promise<void> {
566 const backupDir = databaseBackupDir(baseDir, pending.id);
567 if (!(await pathExists(backupDir))) return;
568​
569 await markDatabaseRestorePending(backupDir);
570 const directories = new Set([NodePath.dirname(pending.dbPath)]);
571 // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
572 const mainTarget = await restoreTarget(pending.dbPath);
573 for (const suffix of DB_FILE_SUFFIXES) {
574 const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
575 if (livePath !== `${pending.dbPath}${suffix}`) {
576 await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
577 }
578 const source = databaseBackupFile(backupDir, suffix);
579 if (await pathExists(source)) {
580 // Restore into the symlink target; cloning would replace the link.
581 const target = await restoreTarget(livePath);
582 await copyDatabaseFile(source, target);
583 await syncFile(target);
584 directories.add(NodePath.dirname(target));
585 } else {
586 await NodeFSP.rm(livePath, { force: true });
587 }
588 }
589 for (const directory of directories) await syncDirectory(directory);
591​
592async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
593 const backupDir = databaseBackupDir(baseDir, updateId);
594 if (!(await pathExists(backupDir))) return;
595 await NodeFSP.rm(backupDir, { recursive: true, force: true });
596 await syncDirectory(NodePath.dirname(backupDir));
598​
599export async function readServiceState(filePath: string): Promise<ServiceState> {
600 const contents = await NodeFSP.readFile(filePath, "utf8");
601 const state = parseServiceState(contents) ?? adoptLegacyServiceState(contents);
602 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
603 return state;
605​
606// An older coordinator can write protocol 2 beside a newer runtime. Adopt
607// only its install record, in memory: rewriting it outside the selection lock
608// could replace state a concurrent install wrote after this read.
609function adoptLegacyServiceState(contents: string): ServiceState | undefined {
610 const legacy = parseLegacyServiceState(contents);
611 return legacy && { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: legacy.activeVersion };
613​
614export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
615 await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 });
616 await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`);
618​
619async function replaceFileDurably(filePath: string, contents: string): Promise<void> {
620 const directory = NodePath.dirname(filePath);
621 const tempPath = NodePath.join(
622 directory,
623 `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
624 );
625 let handle: NodeFSP.FileHandle | undefined;
626 try {
627 handle = await NodeFSP.open(tempPath, "wx", 0o600);
628 await handle.writeFile(contents, "utf8");
629 await handle.sync();
630 await handle.close();
631 handle = undefined;
632 await NodeFSP.rename(tempPath, filePath);
633 await syncDirectory(directory);
634 } finally {
635 await handle?.close().catch(() => undefined);
636 await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined);
637 }
639​
640async function runtimeExists(baseDir: string, version: string): Promise<boolean> {
641 const paths = runtimePaths(baseDir, version);
642 try {
643 const [entry, sentinel] = await Promise.all([
644 NodeFSP.stat(paths.entryPath),
645 NodeFSP.readFile(paths.sentinelPath, "utf8"),
646 ]);
647 return entry.isFile() && sentinel.trim() === version;
648 } catch {
649 return false;
650 }
652​
653function terminalUpdate<S extends TerminalStatus>(input: {
654 readonly pending: PendingServiceUpdate;
655 readonly status: S;
656 readonly reason?: string;
657}): Exclude<ServiceUpdateRecord, PendingServiceUpdate> & { readonly status: S } {
658 return {
659 id: input.pending.id,
660 fromVersion: input.pending.fromVersion,
661 targetVersion: input.pending.targetVersion,
662 status: input.status,
663 ...(input.reason === undefined ? {} : { reason: input.reason }),
664 };
666​
667function sendMessage(
668 child: NodeChildProcess.ChildProcess,
669 message: ServiceLauncherParentMessage,
670): Promise<void> {
671 return new Promise((resolve, reject) => {
672 if (!child.connected || child.send === undefined) {
673 reject(new Error("service child IPC is disconnected."));
674 return;
675 }
676 child.send(message, (error) => (error === null ? resolve() : reject(error)));
677 });
679​
680function waitForExit(child: NodeChildProcess.ChildProcess): Promise<void> {
681 if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
682 return new Promise((resolve) => child.once("exit", () => resolve()));
684​
685async function terminateChild(
686 child: NodeChildProcess.ChildProcess,
687 signal: NodeJS.Signals,
688 overIpc: boolean,
689): Promise<void> {
690 if (child.exitCode !== null || child.signalCode !== null) return;
691 if (overIpc) {
692 await sendMessage(child, { type: "stop" }).catch(() => child.kill(signal));
693 } else {
694 child.kill(signal);
695 }
696 const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS);
697 try {
698 await waitForExit(child);
699 } finally {
700 clearTimeout(force);
701 }
703​
704/** Task Scheduler names are machine-wide, so each Windows account runs its own task. */
705export const serviceTaskName = (domain: string, userName: string) => `t3code-${domain}-${userName}`;
706​
707export const serviceLauncherLockPath = (baseDir: string) =>
708 NodePath.join(baseDir, "runtime", SERVICE_LAUNCHER_LOCK_FILE);
709​
710// An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies.
711async function tryAcquireSqliteLock(
712 lockPath: string,
713): Promise<{ readonly close: () => void } | undefined> {
714 const { DatabaseSync } = await import("node:sqlite");
715 const db = new DatabaseSync(lockPath);
716 try {
717 db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
718 return db;
719 } catch (cause) {
720 db.close();
721 if (cause instanceof Error && "errcode" in cause && cause.errcode === 5) return undefined;
722 throw cause;
723 }
725​
726export async function serviceLauncherIsRunning(lockPath: string): Promise<boolean> {
727 if (!NodeFS.existsSync(lockPath)) return false;
728 const lock = await tryAcquireSqliteLock(lockPath);
729 lock?.close();
730 return lock === undefined;
732​
733/** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */
734export async function waitForServiceLauncherExit(
735 lockPath: string,
736 signal: AbortSignal,
737): Promise<void> {
738 while (await serviceLauncherIsRunning(lockPath)) {
739 signal.throwIfAborted();
740 await new Promise((resolve) => setTimeout(resolve, 100));
741 }
743​
744// Mirror acquireServerOwnership: the lock, then a live pre-lock server-runtime.json owner;
745// leave other failures for server startup to report.
746export async function stateDirOwned(baseDir: string): Promise<boolean> {
747 const stateDir = NodePath.join(baseDir, "userdata");
748 const lock = await tryAcquireSqliteLock(NodePath.join(stateDir, "server-owner.sqlite")).catch(
749 () => null,
750 );
751 if (lock === null) return false;
752 if (lock === undefined) return true;
753 lock.close();
754 let record: unknown;
755 try {
756 record = JSON.parse(
757 await NodeFSP.readFile(NodePath.join(stateDir, "server-runtime.json"), "utf8"),
758 );
759 } catch {
760 return false;
761 }
762 if (typeof record !== "object" || record === null || "ownerId" in record) return false;
763 const pid = "pid" in record ? record.pid : undefined;
764 if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false;
765 try {
766 process.kill(pid, 0);
767 } catch (error) {
768 if (!(error instanceof Error && "code" in error && error.code === "EPERM")) return false;
769 }
770 const recordedAt = Date.parse(
771 "startedAt" in record && typeof record.startedAt === "string" ? record.startedAt : "",
772 );
773 if (!Number.isFinite(recordedAt)) return true;
774 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
775 const platform = process.platform;
776 const { command, args, env } = legacyOwnerStartTimeCommand(pid, platform);
777 const output = await new Promise<string | undefined>((resolve) =>
778 NodeChildProcess.execFile(
779 command,
780 args,
781 { env: { ...process.env, ...env }, timeout: 2_000, maxBuffer: 16_384 },
782 (error, stdout) => resolve(error === null ? stdout : undefined),
783 ),
784 );
785 return legacyOwnerStartedBeforeRecord(output, recordedAt, platform);
787​
788const stopMarkerPath = (baseDir: string) =>
789 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
790const restartPendingPath = (baseDir: string) =>
791 NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE);
792​
793export class Launcher {
794 readonly #baseDir: string;
795 readonly #statePath: string;
796 #state: ServiceState;
797 #child: ManagedChild | null = null;
798 #timer: NodeJS.Timeout | undefined;
799 #transitions: Promise<void> = Promise.resolve();
800 #stopRequested = false;
801 #stopping = false;
802 #done = false;
803 readonly #completion = Promise.withResolvers<void>();
804 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
805 readonly #serviceUnit: ServiceUnit | undefined;
806 readonly #environment: Readonly<NodeJS.ProcessEnv>;
807 readonly #stateDirOwnerPollMs: number;
808 #stateDirFreeProbes = 0;
809 readonly #output: number | undefined;
810 readonly #lockPath: string | undefined;
811 readonly #stopChildOverIpc: boolean;
812​
813 constructor(
814 baseDir: string,
815 state: ServiceState,
816 options: {
817 readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
818 readonly serviceUnit?: ServiceUnit;
819 readonly environment?: Readonly<NodeJS.ProcessEnv>;
820 readonly stateDirOwnerPollMs?: number;
821 /** Launcher and server output, for managers that cannot redirect it. */
822 readonly output?: number;
823 /** Makes this the only launcher for its home; a replacement waits for the one stopping. */
824 readonly lockPath?: string;
825 /** Windows `kill()` is TerminateProcess, which skips the server's shutdown finalizers. */
826 readonly stopChildOverIpc?: boolean;
827 } = {},
828 ) {
829 this.#baseDir = baseDir;
830 this.#stateDirOwnerPollMs = options.stateDirOwnerPollMs ?? STATE_DIR_OWNER_POLL_MS;
831 this.#backupSpaceProbe = options.backupSpaceProbe;
832 this.#serviceUnit = options.serviceUnit;
833 this.#environment = { ...(options.environment ?? process.env) };
834 this.#output = options.output;
835 this.#lockPath = options.lockPath;
836 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
837 this.#stopChildOverIpc = options.stopChildOverIpc ?? process.platform === "win32";
838 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
839 this.#state = state;
840 }
841​
842 async run(): Promise<void> {
843 const onSigterm = () => void this.stop("SIGTERM");
844 const onSigint = () => void this.stop("SIGINT");
845 process.once("SIGTERM", onSigterm);
846 process.once("SIGINT", onSigint);
847 try {
848 const lock =
849 this.#lockPath === undefined ? undefined : await this.#acquireLock(this.#lockPath);
850 try {
851 // The launcher that held the lock may have moved the state since main read it.
852 if (lock !== undefined) this.#state = await readServiceState(this.#statePath);
853 // A stop that arrived before this point leaves nothing to start or recover.
854 if (!this.#stopRequested) this.#enqueue(() => this.#recover());
855 await this.#completion.promise;
856 } finally {
857 if (lock !== undefined) {
858 // Releasing the lock says this launcher is done with the home.
859 await this.#transitions.catch(() => undefined);
860 lock.close();
861 }
862 }
863 } finally {
864 process.off("SIGTERM", onSigterm);
865 process.off("SIGINT", onSigint);
866 }
867 }
868​
869 async #acquireLock(lockPath: string): Promise<{ readonly close: () => void } | undefined> {
870 for (let attempt = 1; !this.#stopRequested; attempt++) {
871 const lock = await tryAcquireSqliteLock(lockPath);
872 if (lock !== undefined) return lock;
873 if (attempt >= LOCK_ATTEMPTS) {
874 throw new Error("Another T3 Code service launcher still runs for this T3 home.");
875 }
876 await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS));
877 }
878 return undefined;
879 }
880​
881 #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") {
882 return terminateChild(child, signal, this.#stopChildOverIpc);
883 }
884​
885 #log(line: string): void {
886 if (this.#output === undefined) process.stderr.write(line);
887 else NodeFS.writeSync(this.#output, line);
888 }
889​
890 #enqueue(transition: () => Promise<void>): void {
891 this.#transitions = this.#transitions
892 .then(transition, transition)
893 .catch((cause: unknown) =>
894 this.#fatal(cause instanceof Error ? cause : new Error(String(cause))),
895 );
896 }
897​
898 async #fatal(error: Error): Promise<void> {
899 if (this.#done) return;
900 this.#done = true;
901 this.#stopping = true;
902 this.#clearTimer();
903 const child = this.#child?.process;
904 this.#child = null;
905 if (child !== undefined) await this.#terminate(child);
906 this.#completion.reject(error);
907 }
908​
909 async stop(signal: NodeJS.Signals): Promise<void> {
910 // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer.
911 try {
912 NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 });
913 } catch {}
914 if (this.#stopRequested || this.#stopping) {
915 await this.#completion.promise.catch(() => undefined);
916 return;
917 }
918 this.#stopRequested = true;
919 this.#clearTimer();
920 this.#enqueue(async () => {
921 // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup.
922 this.#stopping = true;
923 // An exit-78 transition queued before this one may have started the owner poll.
924 this.#clearTimer();
925 const child = this.#child?.process;
926 this.#child = null;
927 if (child !== undefined) await this.#terminate(child, signal);
928 this.#done = true;
929 this.#completion.resolve();
930 });
931 await this.#completion.promise.catch(() => undefined);
932 }
933​
934 #clearTimer(): void {
935 clearTimeout(this.#timer);
936 this.#timer = undefined;
937 }
938​
939 async #selectState(next: ServiceState): Promise<void> {
940 const release = await acquireServiceSelectionLock(this.#statePath);
941 try {
942 const selected = await readServiceState(this.#statePath);
943 // Local installs replace the selection and clear its update record. An old
944 // launcher must not restore its own selection when asked to update again.
945 if (
946 selected.activeVersion !== this.#state.activeVersion ||
947 selected.update?.id !== this.#state.update?.id ||
948 selected.update?.status !== this.#state.update?.status
949 ) {
950 throw new Error("Service selection changed. Restart the service before updating.");
951 }
952 await writeServiceState(this.#statePath, next);
953 this.#state = next;
954 } finally {
955 release();
956 }
957 }
958​
959 async #recover(): Promise<void> {
960 // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
961 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
962 const restartPending = restartPendingPath(this.#baseDir);
963 try {
964 const release = await acquireServiceSelectionLock(this.#statePath);
965 try {
966 const selected = await readServiceState(this.#statePath);
967 const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(
968 () => undefined,
969 );
970 const awaited = awaitedVersion?.trim();
971 if (
972 selected.activeVersion === this.#state.activeVersion &&
973 selected.update?.id === this.#state.update?.id &&
974 selected.update?.status === this.#state.update?.status &&
975 awaited !== undefined &&
976 isExactServiceVersion(awaited) &&
977 compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0
978 ) {
979 await NodeFSP.rm(restartPending, { force: true });
980 }
981 } finally {
982 release();
983 }
984 } catch (error) {
985 process.stderr.write(
986 `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`,
987 );
988 }
989 const update = this.#state.update;
990 if (update?.status !== "pending") {
991 if (update !== undefined) {
992 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
993 }
994 try {
995 await this.#startChild(this.#state.activeVersion, "active", update);
996 } finally {
997 if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion);
998 }
999 return;
1001 if (await databaseRestorePending(this.#baseDir, update)) {
1002 await this.#returnToPrevious(update, "failed", "rollback-interrupted");
1003 return;
1005 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {
1006 await this.#returnToPrevious(update, "failed", "target-runtime-missing");
1007 return;
1009 await this.#startTrial(update);
1011​
1012 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
1013 try {
1014 await backupDatabaseOnce(this.#baseDir, pending);
1015 } catch {
1016 await this.#returnToPrevious(pending, "failed", "db-backup-failed");
1017 return;
1019 try {
1020 await this.#startChild(pending.targetVersion, "trial", pending);
1021 } catch {
1022 await this.#returnToPrevious(pending, "failed", "candidate-start-failed");
1025​
1026 async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
1027 if (this.#stopping) return;
1028 if (!(await runtimeExists(this.#baseDir, version))) {
1029 throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
1031 if (this.#stopping) return;
1032 const paths = runtimePaths(this.#baseDir, version);
1033 const context: ServiceLauncherContext = {
1034 protocol: SERVICE_LAUNCHER_PROTOCOL,
1035 childVersion: version,
1036 ...(update === undefined ? {} : { update }),
1037 };
1038 const spawnArguments = runtimeSpawnArguments(paths);
1039 const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, {
1040 env: { ...this.#environment, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) },
1041 stdio:
1042 this.#output === undefined
1043 ? ["inherit", "inherit", "inherit", "ipc"]
1044 : ["ignore", this.#output, this.#output, "ipc"],
1045 });
1046 await new Promise<void>((resolve, reject) => {
1047 const onError = (error: Error) => reject(error);
1048 child.once("error", onError);
1049 child.once("spawn", () => {
1050 child.removeListener("error", onError);
1051 child.on("error", (error) => this.#enqueue(() => Promise.reject(error)));
1052 resolve();
1053 });
1054 });
1055 if (this.#stopping) {
1056 await this.#terminate(child);
1057 return;
1059​
1060 const managed: ManagedChild = {
1061 version,
1062 role,
1063 process: child,
1064 };
1065 this.#child = managed;
1066 child.on("message", (value) => {
1067 const message = decodeServiceLauncherChildMessage(value);
1068 if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message));
1069 });
1070 child.once("exit", (code, signal) =>
1071 this.#enqueue(() => this.#handleExit(managed, code, signal)),
1072 );
1073​
1074 if (role === "trial") {
1075 this.#timer = setTimeout(
1076 () => this.#enqueue(() => this.#handlePreparedTimeout(managed)),
1077 PREPARED_TIMEOUT_MS,
1078 );
1081​
1082 async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise<void> {
1083 if (this.#child !== child || this.#stopping) return;
1084 if (message.type === "request-update") {
1085 await this.#handleUpdateRequest(child, message);
1086 return;
1088 await this.#handlePrepared(child, message.updateId);
1090​
1091 async #handleUpdateRequest(
1092 child: ManagedChild,
1093 message: Extract<ServiceLauncherChildMessage, { readonly type: "request-update" }>,
1094 ): Promise<void> {
1095 const reject = (reason: string) =>
1096 sendMessage(child.process, { type: "update-rejected", reason });
1097 if (child.role !== "active") {
1098 await reject("Only the active server can request an update.");
1099 return;
1101 if (child.version !== this.#state.activeVersion) {
1102 await reject("The requesting server is not the selected active version.");
1103 return;
1105 if (this.#state.update?.status === "pending") {
1106 await reject("Another server update is already pending.");
1107 return;
1109 if (!isExactServiceVersion(message.targetVersion)) {
1110 await reject("The requested target is not an exact version.");
1111 return;
1113 if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) {
1114 await reject("Remote updates must select a newer server version.");
1115 return;
1117 if (!NodePath.isAbsolute(message.dbPath)) {
1118 await reject("The requested database path is not absolute.");
1119 return;
1121 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {
1122 await reject("The requested target runtime is missing or incomplete.");
1123 return;
1125 // Failed checks must refuse updates rather than stop the server with unknown backup capacity.
1126 const backupBlocker = await databaseBackupBlocker(
1127 this.#baseDir,
1128 message.dbPath,
1129 this.#backupSpaceProbe,
1130 ).catch(
1131 (error: unknown) =>
1132 `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`,
1133 );
1134 if (!child.process.connected) return;
1135 if (backupBlocker !== undefined) {
1136 await reject(backupBlocker);
1137 return;
1139​
1140 const pending: PendingServiceUpdate = {
1141 id: NodeCrypto.randomUUID(),
1142 fromVersion: child.version,
1143 targetVersion: message.targetVersion,
1144 dbPath: message.dbPath,
1145 status: "pending",
1146 };
1147 const next: ServiceState = { ...this.#state, update: pending };
1148 try {
1149 await this.#selectState(next);
1150 } catch (error) {
1151 await reject(error instanceof Error ? error.message : String(error));
1152 return;
1154 await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
1155 this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
1157​
1158 async #beginTrial(child: ManagedChild): Promise<void> {
1159 const pending = this.#state.update;
1160 if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") {
1161 return;
1163 this.#timer = undefined;
1164 // Anything thrown outside its own fallback still leaves the offline backup to run.
1165 const finishOnlineBackup = await startOnlineDatabaseBackup(
1166 this.#baseDir,
1167 pending,
1168 this.#backupSpaceProbe,
1169 ).catch(() => undefined);
1170 this.#child = null;
1171 await this.#terminate(child.process);
1172 await finishOnlineBackup?.();
1173 // A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
1174 if (this.#stopRequested) return;
1175 await this.#startTrial(pending);
1177​
1178 async #handlePrepared(child: ManagedChild, updateId: string): Promise<void> {
1179 const pending = this.#state.update;
1180 if (
1181 child.role !== "trial" ||
1182 pending?.status !== "pending" ||
1183 pending.id !== updateId ||
1184 pending.targetVersion !== child.version
1185 ) {
1186 if (child.role === "trial" && pending?.status === "pending") {
1187 await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child);
1188 return;
1190 throw new Error("Trial child reported prepared for an unexpected update.");
1192 this.#clearTimer();
1193 const committed = terminalUpdate({ pending, status: "committed" });
1194 const next: ServiceState = {
1195 ...this.#state,
1196 activeVersion: pending.targetVersion,
1197 update: committed,
1198 };
1199 await this.#selectState(next);
1200 child.role = "active";
1201 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
1202 try {
1203 await sendMessage(child.process, { type: "committed", updateId: committed.id });
1204 } finally {
1205 await this.#repointServiceUnit(committed.targetVersion);
1208​
1209 // Selection is durable before this best-effort unit reconciliation.
1210 async #repointServiceUnit(version: string): Promise<void> {
1211 const unit = this.#serviceUnit;
1212 if (unit === undefined) return;
1213 try {
1214 const release = await acquireServiceSelectionLock(unit.path);
1215 try {
1216 // The installer writes state before the unit under this lease. Do not
1217 // acquire its state lease here: the installer takes that lease first.
1218 const selected = await readServiceState(this.#statePath);
1219 if (
1220 selected.activeVersion !== this.#state.activeVersion ||
1221 selected.update?.id !== this.#state.update?.id ||
1222 selected.update?.status !== this.#state.update?.status
1223 ) {
1224 return;
1226 const contents = await NodeFSP.readFile(unit.path, "utf8");
1227 const installedBaseDir = serviceUnitBaseDirOf(contents);
1228 if (
1229 installedBaseDir === undefined ||
1230 NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir)
1231 ) {
1232 return;
1234 const repointed = repointServiceUnit(contents, version);
1235 if (repointed !== contents) await replaceFileDurably(unit.path, repointed);
1236 // A previous rename may have succeeded before reload failed.
1237 await unit.reload();
1238 } finally {
1239 release();
1241 } catch (error) {
1242 process.stderr.write(
1243 `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`,
1244 );
1247​
1248 async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
1249 const pending = this.#state.update;
1250 if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") {
1251 return;
1253 this.#timer = undefined;
1254 await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child);
1256​
1257 async #handleExit(
1258 child: ManagedChild,
1259 code: number | null,
1260 signal: NodeJS.Signals | null,
1261 ): Promise<void> {
1262 if (this.#child !== child || this.#stopping) return;
1263 this.#child = null;
1264 if (code === STATE_DIR_OWNED_EXIT_CODE) {
1265 await this.#idleWhileStateDirOwned();
1266 return;
1268 if (child.role === "trial") {
1269 this.#clearTimer();
1270 const pending = this.#state.update;
1271 if (pending?.status !== "pending") {
1272 throw new Error("Trial child exited without matching pending state.");
1274 await this.#returnToPrevious(
1275 pending,
1276 "rolled-back",
1277 `candidate-exited:${String(code ?? signal ?? "unknown")}`,
1278 );
1279 return;
1281​
1282 this.#clearTimer();
1283 const pending = this.#state.update;
1284 if (pending?.status === "pending") {
1285 await this.#startTrial(pending);
1286 return;
1288 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
1290​
1291 // Stay idle until the lock frees to avoid supervisor restart loops. A pending update resumes its
1292 // trial, since the other owner may have migrated the database past fromVersion. It stays pending
1293 // only once its backup is gone, so no later trial, even after a launcher restart, restores it over
1294 // the other owner's writes; exit 78 comes before the child opens the database.
1295 async #idleWhileStateDirOwned(): Promise<void> {
1296 this.#clearTimer();
1297 const pending = this.#state.update;
1298 if (pending?.status === "pending") {
1299 const discarded = await discardDatabaseBackup(this.#baseDir, pending.id).then(
1300 () => true,
1301 () => false,
1302 );
1303 if (!discarded) {
1304 const next: ServiceState = {
1305 ...this.#state,
1306 activeVersion: pending.fromVersion,
1307 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
1308 };
1309 await this.#selectState(next);
1310 await this.#repointServiceUnit(next.activeVersion);
1313 this.#log(
1314 "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n",
1315 );
1316 this.#stateDirFreeProbes = 0;
1317 // The interval also keeps Node alive: signal listeners alone do not.
1318 this.#timer = setInterval(
1319 () => this.#enqueue(() => this.#reclaimStateDir()),
1320 this.#stateDirOwnerPollMs,
1321 );
1323​
1324 async #reclaimStateDir(): Promise<void> {
1325 if (this.#stopRequested) {
1326 this.#clearTimer();
1327 return;
1329 if (this.#timer === undefined || this.#child !== null) return;
1330 const owned = await stateDirOwned(this.#baseDir);
1331 // A stop during the probe clears the timer.
1332 if (this.#timer === undefined) return;
1333 // A relaunching desktop app frees the home for a few seconds; leave it that gap.
1334 this.#stateDirFreeProbes = owned ? 0 : this.#stateDirFreeProbes + 1;
1335 if (this.#stateDirFreeProbes < 2) return;
1336 this.#clearTimer();
1337 this.#log("[service-launcher] The T3 home is free again. Starting the server.\n");
1338 const update = this.#state.update;
1339 if (update?.status === "pending") await this.#startTrial(update);
1340 else await this.#startChild(this.#state.activeVersion, "active", update);
1342​
1343 async #returnToPrevious(
1344 pending: PendingServiceUpdate,
1345 status: "rolled-back" | "failed",
1346 reason: string,
1347 child?: ManagedChild,
1348 ): Promise<void> {
1349 if (child !== undefined) {
1350 this.#child = null;
1351 await this.#terminate(child.process);
1353 await restoreDatabaseBackup(this.#baseDir, pending);
1354 const outcome = terminalUpdate({ pending, status, reason });
1355 const next: ServiceState = {
1356 ...this.#state,
1357 activeVersion: pending.fromVersion,
1358 update: outcome,
1359 };
1360 await this.#selectState(next);
1361 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1362 try {
1363 await this.#startChild(next.activeVersion, "active", outcome);
1364 } finally {
1365 await this.#repointServiceUnit(next.activeVersion);
1369​
1370/** Task Scheduler cannot set variables or redirect output, so a Windows task passes them as flags. */
1371export interface LauncherOptions {
1372 readonly baseDir?: string | undefined;
1373 readonly logFile?: string | undefined;
1374 readonly serviceUnit?: string | undefined;
1376​
1377export async function main(options: LauncherOptions = {}): Promise<void> {
1378 const baseDir = (options.baseDir ?? process.env.T3CODE_HOME)?.trim();
1379 if (baseDir === undefined || baseDir === "") {
1380 throw new Error("T3CODE_HOME is required by the T3 Code service launcher.");
1382 const output =
1383 options.logFile === undefined ? undefined : NodeFS.openSync(options.logFile, "a", 0o600);
1384 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
1385 const windows = process.platform === "win32";
1386 let logTimer: NodeJS.Timeout | undefined;
1387 try {
1388 // What the service manager started the launcher with. It locates the unit; service.env
1389 // changes only what the server gets.
1390 const managerEnvironment = {
1391 ...process.env,
1392 T3CODE_HOME: baseDir,
1393 ...(options.serviceUnit === undefined ? {} : { [SERVICE_UNIT_ENV]: options.serviceUnit }),
1394 };
1395 const environment = await readServiceEnvironment(baseDir, managerEnvironment);
1396 const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
1397 const contents = await NodeFSP.readFile(statePath, "utf8");
1398 let state = parseServiceState(contents);
1399 if (state === undefined) {
1400 state = adoptLegacyServiceState(contents);
1401 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
1402 if (!(await runtimeExists(baseDir, state.activeVersion))) {
1403 throw new Error(`Selected t3@${state.activeVersion} runtime is missing or incomplete.`);
1406 const serviceUnit = serviceUnitFromEnvironment(managerEnvironment);
1407 const logPath = await serviceLogPath(options, serviceUnit);
1408 if (logPath !== undefined) {
1409 await boundServiceLog(logPath);
1410 logTimer = setInterval(
1411 () => void boundServiceLog(logPath),
1412 SERVICE_LOG_CHECK_INTERVAL_MS,
1413 ).unref();
1415 const launcher = new Launcher(baseDir, state, {
1416 environment,
1417 ...(serviceUnit === undefined ? {} : { serviceUnit }),
1418 ...(output === undefined ? {} : { output }),
1419 ...(windows ? { lockPath: serviceLauncherLockPath(baseDir) } : {}),
1420 });
1421 // Ending a scheduled task kills its console host without a signal; the console read fails instead.
1422 const watchConsole = windows && process.stdin.isTTY === true;
1423 const onConsoleLost = () => void launcher.stop("SIGTERM");
1424 if (watchConsole) {
1425 process.stdin.on("error", () => undefined);
1426 process.stdin.once("close", onConsoleLost);
1427 process.stdin.resume();
1429 try {
1430 await launcher.run();
1431 } finally {
1432 if (watchConsole) {
1433 process.stdin.off("close", onConsoleLost);
1434 process.stdin.destroy();
1437 } catch (cause) {
1438 if (output !== undefined) {
1439 NodeFS.writeSync(
1440 output,
1441 `[service-launcher] ${cause instanceof Error ? cause.message : String(cause)}\n`,
1442 );
1444 // Agents the server left behind keep the console host, and so the task, running; ending it
1445 // lets the next repeating trigger start the service again.
1446 if (windows && options.serviceUnit !== undefined) {
1447 const taskName = serviceTaskName(process.env.USERDOMAIN ?? "", process.env.USERNAME ?? "");
1448 NodeChildProcess.spawnSync("schtasks", ["/End", "/TN", taskName], {
1449 stdio: "ignore",
1450 windowsHide: true,
1451 });
1453 throw cause;
1454 } finally {
1455 clearInterval(logTimer);
1456 if (output !== undefined) NodeFS.closeSync(output);