1// @effect-diagnostics nodeBuiltinImport:off
2// @effect-diagnostics globalTimers:off
3// The launcher must survive unloadable server versions; use Node built-ins without Effect.
4import * as NodeChildProcess from "node:child_process";
5import * as NodeCrypto from "node:crypto";
6import * as NodeFS from "node:fs";
7import * as NodeFSP from "node:fs/promises";
8import * as NodePath from "node:path";
9import * as NodeUtil from "node:util";
10import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts";
11
12import type {
13 PendingServiceUpdate,
14 ServiceLauncherChildMessage,
15 ServiceLauncherContext,
16 ServiceLauncherParentMessage,
17 ServiceState,
18 ServiceUpdateRecord,
19} from "./cloud/serviceProtocol.ts";
20import {
21 compareExactServiceVersions,
22 decodeServiceLauncherChildMessage,
23 isExactServiceVersion,
24 parseLegacyServiceState,
25 disabledServiceTask,
26 parseServiceState,
27 serviceTaskIsDisabled,
28 serviceUnitBaseDirOf,
29 serviceUnitLogPathOf,
30 SERVICE_ENV_FILE,
31 SERVICE_LAUNCHER_LOCK_FILE,
32 SERVICE_LAUNCHER_CONTEXT_ENV,
33 SERVICE_LAUNCHER_PROTOCOL,
34 SERVICE_STATE_FILE,
35 SERVICE_RESTART_PENDING_FILE,
36 SERVICE_STOP_MARKER_FILE,
37 SERVICE_UNIT_ENV,
38 systemdUnitDirectoryOf,
39} from "./cloud/serviceProtocol.ts";
40import {
41 legacyOwnerStartedBeforeRecord,
42 legacyOwnerStartTimeCommand,
43} from "./legacyServerOwner.ts";
44
45const HANDOFF_DELAY_MS = 2_000;
46/**
47 * Migrations count toward it and can scale with the database: tens of GB on a cold page cache
48 * take minutes. A missed window rolls back the database, so the next attempt pays the same cost.
49 */
50const PREPARED_TIMEOUT_MS = 30 * 60_000;
51const TERMINATE_GRACE_MS = 5_000;
52// A replacement started by a handoff waits up to two minutes for the stopping launcher.
53const LOCK_RETRY_MS = 500;
54const LOCK_ATTEMPTS = 240;
55// Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect.
56const STATE_DIR_OWNED_EXIT_CODE = 78;
57const STATE_DIR_OWNER_POLL_MS = 30_000;
58
59type TerminalStatus = "committed" | "rolled-back" | "failed";
60type ChildRole = "active" | "trial";
61
62interface ManagedChild {
63 readonly version: string;
64 role: ChildRole;
65 readonly process: NodeChildProcess.ChildProcess;
66}
67
68const runtimePaths = (baseDir: string, version: string) => {
69 const versionDir = NodePath.join(baseDir, "runtime", "versions", version);
70 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
71 const executableName = process.platform === "win32" ? "t3.exe" : "t3";
72 return {
73 versionDir,
74 entryPath: NodePath.join(versionDir, executableName),
75 sentinelPath: NodePath.join(versionDir, ".install-complete"),
76 };
77};
78
79const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
80 command: paths.entryPath,
81 args: ["serve"],
82});
83
84/**
85 * The unit or launch agent that `t3 service install` wrote. Its program names
86 * the runtime that ran the install, and a remote update never replaces it, so
87 * after a commit the launcher points it at the committed runtime. Otherwise the
88 * next start lands on a runtime that nothing in the service state records.
89 */
90export interface ServiceUnit {
91 readonly path: string;
92 /** systemd and Task Scheduler keep a loaded copy; launchd re-reads the plist at bootstrap. */
93 readonly reload: () => Promise<void>;
94}
95
96const runSchtasks = (...args: ReadonlyArray<string>) =>
97 NodeUtil.promisify(NodeChildProcess.execFile)("schtasks", args, {
98 windowsHide: true,
99 timeout: 30_000,
100 });
101
102/**
103 * Registering replaces the task, so one the CLI disabled to keep the service stopped is
104 * registered disabled. The caller holds the unit lease, as the CLI does to disable it.
105 */
106export async function registerServiceTask(
107 unitPath: string,
108 taskName: string,
109 schtasks: (...args: ReadonlyArray<string>) => Promise<{ readonly stdout: string }> = runSchtasks,
110): Promise<void> {
111 const register = (xmlPath: string) => schtasks("/Create", "/XML", xmlPath, "/TN", taskName, "/F");
112 const registered = await schtasks("/Query", "/TN", taskName, "/XML");
113 if (!serviceTaskIsDisabled(registered.stdout)) {
114 await register(unitPath);
115 return;
116 }
117 const disabledPath = `${unitPath}.${process.pid}.disabled`;
118 try {
119 const unit = await NodeFSP.readFile(unitPath, "utf8");
120 await NodeFSP.writeFile(disabledPath, disabledServiceTask(unit));
121 await register(disabledPath);
122 } finally {
123 await NodeFSP.rm(disabledPath, { force: true });
124 }
125}
126
127// The manager's environment can differ from the one it gives services (environment.d).
128async function readSystemdUnitDirectory(): Promise<string | undefined> {
129 try {
130 const { stdout } = await NodeUtil.promisify(NodeChildProcess.execFile)(
131 "busctl",
132 [
133 "--user",
134 "--json=short",
135 "get-property",
136 "org.freedesktop.systemd1",
137 "/org/freedesktop/systemd1",
138 "org.freedesktop.systemd1.Manager",
139 "UnitPath",
140 ],
141 { timeout: 5_000 },
142 );
143 const { data } = JSON.parse(stdout) as { readonly data?: unknown };
144 return Array.isArray(data) && data.every((entry) => typeof entry === "string")
145 ? systemdUnitDirectoryOf(data)
146 : undefined;
147 } catch {
148 return undefined;
149 }
150}
151
152async function serviceUnitFromEnvironment(
153 env: NodeJS.ProcessEnv = process.env,
154): Promise<ServiceUnit | undefined> {
155 const unitName = env[SERVICE_UNIT_ENV]?.trim();
156 if (!unitName) return undefined;
157 if (unitName.endsWith(".xml")) {
158 const localAppData = env.LOCALAPPDATA?.trim();
159 if (!localAppData) return undefined;
160 const path = NodePath.win32.join(localAppData, "t3code", unitName);
161 const taskName = serviceTaskName(env.USERDOMAIN ?? "", env.USERNAME ?? "");
162 return { path, reload: () => registerServiceTask(path, taskName) };
163 }
164 const home = env.HOME?.trim();
165 if (!home) return undefined;
166 if (unitName.endsWith(".plist")) {
167 return {
168 path: NodePath.join(home, "Library", "LaunchAgents", unitName),
169 reload: () => Promise.resolve(),
170 };
171 }
172 const xdgConfigHome = env.XDG_CONFIG_HOME?.trim();
173 const unitDir =
174 (await readSystemdUnitDirectory()) ??
175 NodePath.join(
176 xdgConfigHome && NodePath.isAbsolute(xdgConfigHome)
177 ? xdgConfigHome
178 : NodePath.join(home, ".config"),
179 "systemd",
180 "user",
181 );
182 return {
183 path: NodePath.join(unitDir, unitName),
184 reload: () =>
185 new Promise((resolve, reject) => {
186 NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], {
187 stdio: "ignore",
188 timeout: 30_000,
189 })
190 .once("error", reject)
191 .once("exit", (code, signal) =>
192 code === 0
193 ? resolve()
194 : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)),
195 );
196 }),
197 };
198}
199
200// systemd's ExecStart value, or the plist string or task argument before __service-launcher.
201// Only the version segment changes, so quoted or escaped paths stay as rendered.
202// The launch agent's PATH names a versions directory without the executable; leave it.
203const SERVICE_UNIT_PROGRAMS = [
204 /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m,
205 /(<string>[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*<string>__service-launcher<\/string>)/,
206 /(<Arguments>--headless "[^"<]*?[\\/]runtime[\\/]versions[\\/])([^\\/"<]+)([\\/]t3\.exe" "__service-launcher")/,
207] as const;
208
209/**
210 * The unit contents with their program pointed at `version`'s runtime. A unit
211 * already at that version or a newer one is returned unchanged: `t3 update`
212 * may have repointed it to a later version while this commit was in flight.
213 */
214function repointServiceUnit(contents: string, version: string): string {
215 for (const program of SERVICE_UNIT_PROGRAMS) {
216 const match = program.exec(contents);
217 if (match === null) continue;
218 const [, prefix = "", current = "", suffix = ""] = match;
219 if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) {
220 return contents;
221 }
222 return contents.replace(program, () => `${prefix}${version}${suffix}`);
223 }
224 return contents;
225}
226
227const SERVICE_LOG_MAX_BYTES = 10 * 1024 * 1024;
228const SERVICE_LOG_CHECK_INTERVAL_MS = 10 * 60_000;
229
230/**
231 * The service manager holds the log open for appending, so it is copied aside and emptied in
232 * place rather than renamed. The copy is best effort: on a full disk the truncate frees space.
233 */
234async function boundServiceLog(logPath: string): Promise<void> {
235 const size = await NodeFSP.stat(logPath).then(
236 (stats) => stats.size,
237 () => 0,
238 );
239 if (size <= SERVICE_LOG_MAX_BYTES) return;
240 const archivePath = `${logPath}.1`;
241 const pendingArchivePath = `${archivePath}.tmp`;
242 await NodeFSP.copyFile(logPath, pendingArchivePath)
243 .then(() => NodeFSP.rename(pendingArchivePath, archivePath))
244 .catch(() => NodeFSP.rm(pendingArchivePath, { force: true }).catch(() => undefined));
245 await NodeFSP.truncate(logPath, 0).catch(() => undefined);
246}
247
248async function serviceLogPath(
249 options: LauncherOptions,
250 serviceUnit: ServiceUnit | undefined,
251): Promise<string | undefined> {
252 if (options.logFile !== undefined) return options.logFile;
253 if (serviceUnit === undefined) return undefined;
254 const contents = await NodeFSP.readFile(serviceUnit.path, "utf8").catch(() => undefined);
255 return contents === undefined ? undefined : serviceUnitLogPathOf(contents);
256}
257
258const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
259const RESTORE_MARKER = ".restore-pending";
260
261const databaseBackupDir = (baseDir: string, updateId: string) =>
262 NodePath.join(baseDir, "runtime", "db-backup", updateId);
263
264const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>
265 NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);
266
267async function pathExists(target: string): Promise<boolean> {
268 try {
269 await NodeFSP.access(target);
270 return true;
271 } catch (cause) {
272 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;
273 throw cause;
274 }
275}
276
277const MANAGED_SERVICE_ENV_NAMES = new Set([
278 "PATH",
279 "T3CODE_HOME",
280 "T3_BOOT_SERVICE_UNIT",
281 SERVICE_LAUNCHER_CONTEXT_ENV,
282]);
283
284export async function readServiceEnvironment(
285 baseDir: string,
286 inherited: NodeJS.ProcessEnv = process.env,
287): Promise<NodeJS.ProcessEnv> {
288 const environment = { ...inherited };
289 let contents: string;
290 try {
291 contents = await NodeFSP.readFile(NodePath.join(baseDir, SERVICE_ENV_FILE), "utf8");
292 } catch (cause) {
293 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return environment;
294 throw cause;
295 }
296 for (const [name, value] of Object.entries(NodeUtil.parseEnv(contents))) {
297 if (
298 /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
299 !MANAGED_SERVICE_ENV_NAMES.has(name.toUpperCase())
300 ) {
301 environment[name] = value;
302 }
303 }
304 return environment;
305}
306
307// Opened read-write: Windows refuses to flush a handle without write access.
308async function syncFile(filePath: string): Promise<void> {
309 const handle = await NodeFSP.open(filePath, "r+");
310 try {
311 await handle.sync();
312 } finally {
313 await handle.close();
314 }
315}
316
317// Fsync makes renames durable; Windows returns EPERM and NTFS journals them itself.
318async function syncDirectory(directory: string): Promise<void> {
319 const handle = await NodeFSP.open(directory, "r");
320 try {
321 await handle.sync();
322 } catch (error) {
323 if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error;
324 } finally {
325 await handle.close();
326 }
327}
328
329function runCp(args: ReadonlyArray<string>): Promise<boolean> {
330 return new Promise((resolve) => {
331 NodeChildProcess.execFile("cp", args, (error) => resolve(error === null));
332 });
333}
334
335// macOS fs.copyFile clone flags fall back to full copies.
336async function copyDatabaseFile(source: string, destination: string): Promise<void> {
337 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
338 const platform = process.platform;
339 // cp -c replaces files and breaks hard links; copy into hard-linked destinations instead.
340 const hardLinked =
341 platform === "darwin" &&
342 ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1;
343 const args =
344 platform === "darwin" && !hardLinked
345 ? ["-c", source, destination]
346 : platform === "linux"
347 ? ["--reflink=auto", source, destination]
348 : undefined;
349 if (args === undefined || !(await runCp(args))) await NodeFSP.copyFile(source, destination);
350}
351
352const GIB = 1024 ** 3;
353// Cloned backups still need headroom for trial writes and migrations to diverge.
354const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
355const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
356// While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
357const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
358
359async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
360 const stats: Array<NodeFS.Stats> = [];
361 for (const suffix of DB_FILE_SUFFIXES) {
362 const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => {
363 if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined;
364 throw cause;
365 });
366 if (stat !== undefined) stats.push(stat);
367 }
368 return stats;
369}
370
371// cp -c silently copies across volumes; require one device matching the APFS system volume's filesystem type.
372async function backupCanClone(dbPath: string, backupRoot: string): Promise<boolean> {
373 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
374 const platform = process.platform;
375 if (platform === "darwin") {
376 const [backupDirectory, backupFs, systemFs] = await Promise.all([
377 NodeFSP.stat(backupRoot),
378 NodeFSP.statfs(backupRoot),
379 NodeFSP.statfs("/"),
380 ]);
381 if (backupFs.type !== systemFs.type) return false;
382 return (await statDatabaseFiles(dbPath)).every(
383 (source) => source.dev === backupDirectory.dev && source.nlink === 1,
384 );
385 }
386 if (platform !== "linux") return false;
387 const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`);
388 try {
389 for (const suffix of DB_FILE_SUFFIXES) {
390 const source = `${dbPath}${suffix}`;
391 if (suffix !== "" && !(await pathExists(source))) continue;
392 if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false;
393 }
394 return true;
395 } finally {
396 for (const suffix of DB_FILE_SUFFIXES) {
397 await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined);
398 }
399 }
400}
401
402export interface BackupSpaceProbe {
403 readonly readFreeBytes: (directory: string) => Promise<number>;
404 readonly canClone: (dbPath: string, backupRoot: string) => Promise<boolean>;
405}
406
407const filesystemBackupSpaceProbe: BackupSpaceProbe = {
408 readFreeBytes: async (directory) => {
409 const stats = await NodeFSP.statfs(directory);
410 return stats.bavail * stats.bsize;
411 },
412 canClone: backupCanClone,
413};
414
415// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.
416// Update requests are refused while one is pending, so no staging directory is still in use here.
417async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {
418 for (const entry of await NodeFSP.readdir(backupRoot)) {
419 if (!entry.endsWith(".staging")) continue;
420 await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(
421 () => undefined,
422 );
423 }
424}
425
426/** Check before stopping the server so full disks refuse updates without an outage. */
427export async function databaseBackupBlocker(
428 baseDir: string,
429 dbPath: string,
430 probe: Partial<BackupSpaceProbe> = {},
431): Promise<string | undefined> {
432 const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
433 const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
434 await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
435 await removeAbandonedBackupCopies(backupRoot);
436 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
437 const size = files.reduce((total, file) => total + file.size, 0);
438 const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
439 if (free >= fullCopyBytes) return undefined;
440 const clonedBytes = Math.max(
441 CLONED_BACKUP_MIN_HEADROOM_BYTES,
442 size * CLONED_BACKUP_HEADROOM_FRACTION,
443 );
444 const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes;
445 if (free >= needed) return undefined;
446 const gb = (bytes: number) => (bytes / GIB).toFixed(1);
447 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
448}
449
450// A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
451async function copyWithoutOpening(source: string, destination: string): Promise<void> {
452 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
453 if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
454 if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
455}
456
457// Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
458// which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
459// `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
460// a write-heavy server's new writes wait, uncommitted, until the server stops.
461async function startOnlineDatabaseBackup(
462 baseDir: string,
463 pending: PendingServiceUpdate,
464 probe: Partial<BackupSpaceProbe> = {},
465): Promise<(() => Promise<void>) | undefined> {
466 const backupDir = databaseBackupDir(baseDir, pending.id);
467 if (await pathExists(backupDir)) return undefined;
468 const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
469 if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
470 return undefined;
471 const NodeSqlite = await import("node:sqlite").catch(() => undefined);
472 if (NodeSqlite === undefined) return undefined;
473
474 const stagingDir = `${backupDir}.staging`;
475 const discardStaging = () =>
476 NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
477 const fallBack = async (cause: unknown) => {
478 await discardStaging();
479 process.stderr.write(
480 `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
481 );
482 return undefined;
483 };
484 let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
485 // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
486 let dbPath = pending.dbPath;
487 const release = () => {
488 try {
489 snapshot?.close();
490 } catch {}
491 snapshot = undefined;
492 };
493 try {
494 await discardStaging();
495 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
496 dbPath = await NodeFSP.realpath(pending.dbPath);
497 snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
498 const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
499 if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
500 snapshot.exec("BEGIN");
501 snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
502 const mainCopy = databaseBackupFile(stagingDir, "");
503 await copyWithoutOpening(dbPath, mainCopy);
504 await syncFile(mainCopy);
505 } catch (cause) {
506 release();
507 return fallBack(cause);
508 }
509 return async () => {
510 try {
511 const wal = `${dbPath}-wal`;
512 if (!(await pathExists(wal)))
513 throw new Error("the WAL disappeared while the snapshot was open.");
514 const walCopy = databaseBackupFile(stagingDir, "-wal");
515 await copyWithoutOpening(wal, walCopy);
516 await syncFile(walCopy);
517 release();
518 await NodeFSP.rename(stagingDir, backupDir);
519 await syncDirectory(NodePath.dirname(backupDir));
520 } catch (cause) {
521 release();
522 await fallBack(cause);
523 }
524 };
525}
526
527// Preserve completed backups; restarted trials may already have changed the live database.
528async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
529 const backupDir = databaseBackupDir(baseDir, pending.id);
530 if (await pathExists(backupDir)) return;
531
532 const stagingDir = `${backupDir}.staging`;
533 await NodeFSP.rm(stagingDir, { recursive: true, force: true });
534 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
535 try {
536 for (const suffix of DB_FILE_SUFFIXES) {
537 const source = `${pending.dbPath}${suffix}`;
538 if (suffix !== "" && !(await pathExists(source))) continue;
539 const destination = databaseBackupFile(stagingDir, suffix);
540 await copyDatabaseFile(source, destination);
541 await syncFile(destination);
542 }
543 await NodeFSP.rename(stagingDir, backupDir);
544 await syncDirectory(NodePath.dirname(backupDir));
545 } catch (cause) {
546 await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
547 throw cause;
548 }
549}
550
551const restoreMarkerPath = (baseDir: string, updateId: string) =>
552 NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);
553
554const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>
555 pathExists(restoreMarkerPath(baseDir, pending.id));
556
557// Mark before live writes so recovery cannot boot a partial restore.
558async function markDatabaseRestorePending(backupDir: string): Promise<void> {
559 const markerPath = NodePath.join(backupDir, RESTORE_MARKER);
560 if (!(await pathExists(markerPath))) {
561 const handle = await NodeFSP.open(markerPath, "wx", 0o600);
562 try {
563 await handle.sync();
564 } finally {
565 await handle.close();
566 }
567 await syncDirectory(backupDir);
568 }
569}
570
571// Resolve symlink targets even if a crashed cp -c left the final target missing.
572async function restoreTarget(path: string): Promise<string> {
573 let current = path;
574 for (let hops = 0; hops < 40; hops++) {
575 try {
576 return await NodeFSP.realpath(current);
577 } catch (cause) {
578 if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
579 }
580 const link = await NodeFSP.readlink(current).catch((cause: unknown) => {
581 const code = (cause as NodeJS.ErrnoException).code;
582 if (code === "ENOENT" || code === "EINVAL") return undefined;
583 throw cause;
584 });
585 if (link === undefined) return current;
586 // Do not normalize .. before following directory symlinks; match filesystem resolution.
587 current = NodePath.isAbsolute(link)
588 ? link
589 : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`;
590 }
591 throw new Error(`Too many symbolic links resolving ${path}.`);
592}
593
594async function restoreDatabaseBackup(
595 baseDir: string,
596 pending: PendingServiceUpdate,
597): Promise<void> {
598 const backupDir = databaseBackupDir(baseDir, pending.id);
599 if (!(await pathExists(backupDir))) return;
600
601 await markDatabaseRestorePending(backupDir);
602 const directories = new Set([NodePath.dirname(pending.dbPath)]);
603 // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
604 const mainTarget = await restoreTarget(pending.dbPath);
605 for (const suffix of DB_FILE_SUFFIXES) {
606 const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
607 if (livePath !== `${pending.dbPath}${suffix}`) {
608 await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
609 }
610 const source = databaseBackupFile(backupDir, suffix);
611 if (await pathExists(source)) {
612 // Restore into the symlink target; cloning would replace the link.
613 const target = await restoreTarget(livePath);
614 await copyDatabaseFile(source, target);
615 await syncFile(target);
616 directories.add(NodePath.dirname(target));
617 } else {
618 await NodeFSP.rm(livePath, { force: true });
619 }
620 }
621 for (const directory of directories) await syncDirectory(directory);
622}
623
624async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
625 const backupDir = databaseBackupDir(baseDir, updateId);
626 if (!(await pathExists(backupDir))) return;
627 await NodeFSP.rm(backupDir, { recursive: true, force: true });
628 await syncDirectory(NodePath.dirname(backupDir));
629}
630
631export async function readServiceState(filePath: string): Promise<ServiceState> {
632 const contents = await NodeFSP.readFile(filePath, "utf8");
633 const state = parseServiceState(contents) ?? adoptLegacyServiceState(contents);
634 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
635 return state;
636}
637
638// An older coordinator can write protocol 2 beside a newer runtime. Adopt
639// only its install record, in memory: rewriting it outside the selection lock
640// could replace state a concurrent install wrote after this read.
641function adoptLegacyServiceState(contents: string): ServiceState | undefined {
642 const legacy = parseLegacyServiceState(contents);
643 return legacy && { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: legacy.activeVersion };
644}
645
646export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
647 await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 });
648 await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`);
649}
650
651async function replaceFileDurably(filePath: string, contents: string): Promise<void> {
652 const directory = NodePath.dirname(filePath);
653 const tempPath = NodePath.join(
654 directory,
655 `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
656 );
657 let handle: NodeFSP.FileHandle | undefined;
658 try {
659 handle = await NodeFSP.open(tempPath, "wx", 0o600);
660 await handle.writeFile(contents, "utf8");
661 await handle.sync();
662 await handle.close();
663 handle = undefined;
664 await NodeFSP.rename(tempPath, filePath);
665 await syncDirectory(directory);
666 } finally {
667 await handle?.close().catch(() => undefined);
668 await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined);
669 }
670}
671
672async function runtimeExists(baseDir: string, version: string): Promise<boolean> {
673 const paths = runtimePaths(baseDir, version);
674 try {
675 const [entry, sentinel] = await Promise.all([
676 NodeFSP.stat(paths.entryPath),
677 NodeFSP.readFile(paths.sentinelPath, "utf8"),
678 ]);
679 return entry.isFile() && sentinel.trim() === version;
680 } catch {
681 return false;
682 }
683}
684
685function terminalUpdate<S extends TerminalStatus>(input: {
686 readonly pending: PendingServiceUpdate;
687 readonly status: S;
688 readonly reason?: string;
689}): Exclude<ServiceUpdateRecord, PendingServiceUpdate> & { readonly status: S } {
690 return {
691 id: input.pending.id,
692 fromVersion: input.pending.fromVersion,
693 targetVersion: input.pending.targetVersion,
694 status: input.status,
695 ...(input.reason === undefined ? {} : { reason: input.reason }),
696 };
697}
698
699function sendMessage(
700 child: NodeChildProcess.ChildProcess,
701 message: ServiceLauncherParentMessage,
702): Promise<void> {
703 return new Promise((resolve, reject) => {
704 if (!child.connected || child.send === undefined) {
705 reject(new Error("service child IPC is disconnected."));
706 return;
707 }
708 child.send(message, (error) => (error === null ? resolve() : reject(error)));
709 });
710}
711
712function waitForExit(child: NodeChildProcess.ChildProcess): Promise<void> {
713 if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
714 return new Promise((resolve) => child.once("exit", () => resolve()));
715}
716
717async function terminateChild(
718 child: NodeChildProcess.ChildProcess,
719 signal: NodeJS.Signals,
720 overIpc: boolean,
721): Promise<void> {
722 if (child.exitCode !== null || child.signalCode !== null) return;
723 if (overIpc) {
724 await sendMessage(child, { type: "stop" }).catch(() => child.kill(signal));
725 } else {
726 child.kill(signal);
727 }
728 const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS);
729 try {
730 await waitForExit(child);
731 } finally {
732 clearTimeout(force);
733 }
734}
735
736/** Task Scheduler names are machine-wide, so each Windows account runs its own task. */
737export const serviceTaskName = (domain: string, userName: string) => `t3code-${domain}-${userName}`;
738
739export const serviceLauncherLockPath = (baseDir: string) =>
740 NodePath.join(baseDir, "runtime", SERVICE_LAUNCHER_LOCK_FILE);
741
742// An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies.
743async function tryAcquireSqliteLock(
744 lockPath: string,
745): Promise<{ readonly close: () => void } | undefined> {
746 const { DatabaseSync } = await import("node:sqlite");
747 const db = new DatabaseSync(lockPath);
748 try {
749 db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
750 return db;
751 } catch (cause) {
752 db.close();
753 if (cause instanceof Error && "errcode" in cause && cause.errcode === 5) return undefined;
754 throw cause;
755 }
756}
757
758export async function serviceLauncherIsRunning(lockPath: string): Promise<boolean> {
759 if (!NodeFS.existsSync(lockPath)) return false;
760 const lock = await tryAcquireSqliteLock(lockPath);
761 lock?.close();
762 return lock === undefined;
763}
764
765/** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */
766export async function waitForServiceLauncherExit(
767 lockPath: string,
768 signal: AbortSignal,
769): Promise<void> {
770 while (await serviceLauncherIsRunning(lockPath)) {
771 signal.throwIfAborted();
772 await new Promise((resolve) => setTimeout(resolve, 100));
773 }
774}
775
776// Mirror acquireServerOwnership: the lock, then a live pre-lock server-runtime.json owner;
777// leave other failures for server startup to report.
778export async function stateDirOwned(baseDir: string): Promise<boolean> {
779 const stateDir = NodePath.join(baseDir, "userdata");
780 const lock = await tryAcquireSqliteLock(NodePath.join(stateDir, "server-owner.sqlite")).catch(
781 () => null,
782 );
783 if (lock === null) return false;
784 if (lock === undefined) return true;
785 lock.close();
786 let record: unknown;
787 try {
788 record = JSON.parse(
789 await NodeFSP.readFile(NodePath.join(stateDir, "server-runtime.json"), "utf8"),
790 );
791 } catch {
792 return false;
793 }
794 if (typeof record !== "object" || record === null || "ownerId" in record) return false;
795 const pid = "pid" in record ? record.pid : undefined;
796 if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false;
797 try {
798 process.kill(pid, 0);
799 } catch (error) {
800 if (!(error instanceof Error && "code" in error && error.code === "EPERM")) return false;
801 }
802 const recordedAt = Date.parse(
803 "startedAt" in record && typeof record.startedAt === "string" ? record.startedAt : "",
804 );
805 if (!Number.isFinite(recordedAt)) return true;
806 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
807 const platform = process.platform;
808 const { command, args, env } = legacyOwnerStartTimeCommand(pid, platform);
809 const output = await new Promise<string | undefined>((resolve) =>
810 NodeChildProcess.execFile(
811 command,
812 args,
813 { env: { ...process.env, ...env }, timeout: 2_000, maxBuffer: 16_384 },
814 (error, stdout) => resolve(error === null ? stdout : undefined),
815 ),
816 );
817 return legacyOwnerStartedBeforeRecord(output, recordedAt, platform);
818}
819
820const stopMarkerPath = (baseDir: string) =>
821 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
822const restartPendingPath = (baseDir: string) =>
823 NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE);
824
825export class Launcher {
826 readonly #baseDir: string;
827 readonly #statePath: string;
828 #state: ServiceState;
829 #child: ManagedChild | null = null;
830 #timer: NodeJS.Timeout | undefined;
831 #transitions: Promise<void> = Promise.resolve();
832 #stopRequested = false;
833 #stopping = false;
834 #done = false;
835 readonly #completion = Promise.withResolvers<void>();
836 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
837 readonly #serviceUnit: ServiceUnit | undefined;
838 readonly #environment: Readonly<NodeJS.ProcessEnv>;
839 readonly #stateDirOwnerPollMs: number;
840 #stateDirFreeProbes = 0;
841 readonly #output: number | undefined;
842 readonly #lockPath: string | undefined;
843 readonly #stopChildOverIpc: boolean;
844
845 constructor(
846 baseDir: string,
847 state: ServiceState,
848 options: {
849 readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
850 readonly serviceUnit?: ServiceUnit;
851 readonly environment?: Readonly<NodeJS.ProcessEnv>;
852 readonly stateDirOwnerPollMs?: number;
853 /** Launcher and server output, for managers that cannot redirect it. */
854 readonly output?: number;
855 /** Makes this the only launcher for its home; a replacement waits for the one stopping. */
856 readonly lockPath?: string;
857 /** Windows `kill()` is TerminateProcess, which skips the server's shutdown finalizers. */
858 readonly stopChildOverIpc?: boolean;
859 } = {},
860 ) {
861 this.#baseDir = baseDir;
862 this.#stateDirOwnerPollMs = options.stateDirOwnerPollMs ?? STATE_DIR_OWNER_POLL_MS;
863 this.#backupSpaceProbe = options.backupSpaceProbe;
864 this.#serviceUnit = options.serviceUnit;
865 this.#environment = { ...(options.environment ?? process.env) };
866 this.#output = options.output;
867 this.#lockPath = options.lockPath;
868 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
869 this.#stopChildOverIpc = options.stopChildOverIpc ?? process.platform === "win32";
870 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
871 this.#state = state;
872 }
873
874 async run(): Promise<void> {
875 const onSigterm = () => void this.stop("SIGTERM");
876 const onSigint = () => void this.stop("SIGINT");
877 process.once("SIGTERM", onSigterm);
878 process.once("SIGINT", onSigint);
879 try {
880 const lock =
881 this.#lockPath === undefined ? undefined : await this.#acquireLock(this.#lockPath);
882 try {
883 // The launcher that held the lock may have moved the state since main read it.
884 if (lock !== undefined) this.#state = await readServiceState(this.#statePath);
885 // A stop that arrived before this point leaves nothing to start or recover.
886 if (!this.#stopRequested) this.#enqueue(() => this.#recover());
887 await this.#completion.promise;
888 } finally {
889 if (lock !== undefined) {
890 // Releasing the lock says this launcher is done with the home.
891 await this.#transitions.catch(() => undefined);
892 lock.close();
893 }
894 }
895 } finally {
896 process.off("SIGTERM", onSigterm);
897 process.off("SIGINT", onSigint);
898 }
899 }
900
901 async #acquireLock(lockPath: string): Promise<{ readonly close: () => void } | undefined> {
902 for (let attempt = 1; !this.#stopRequested; attempt++) {
903 const lock = await tryAcquireSqliteLock(lockPath);
904 if (lock !== undefined) return lock;
905 if (attempt >= LOCK_ATTEMPTS) {
906 throw new Error("Another T3 Code service launcher still runs for this T3 home.");
907 }
908 await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS));
909 }
910 return undefined;
911 }
912
913 #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") {
914 return terminateChild(child, signal, this.#stopChildOverIpc);
915 }
916
917 #log(line: string): void {
918 if (this.#output === undefined) process.stderr.write(line);
919 else NodeFS.writeSync(this.#output, line);
920 }
921
922 #enqueue(transition: () => Promise<void>): void {
923 this.#transitions = this.#transitions
924 .then(transition, transition)
925 .catch((cause: unknown) =>
926 this.#fatal(cause instanceof Error ? cause : new Error(String(cause))),
927 );
928 }
929
930 async #fatal(error: Error): Promise<void> {
931 if (this.#done) return;
932 this.#done = true;
933 this.#stopping = true;
934 this.#clearTimer();
935 const child = this.#child?.process;
936 this.#child = null;
937 if (child !== undefined) await this.#terminate(child);
938 this.#completion.reject(error);
939 }
940
941 async stop(signal: NodeJS.Signals): Promise<void> {
942 // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer.
943 try {
944 NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 });
945 } catch {}
946 if (this.#stopRequested || this.#stopping) {
947 await this.#completion.promise.catch(() => undefined);
948 return;
949 }
950 this.#stopRequested = true;
951 this.#clearTimer();
952 this.#enqueue(async () => {
953 // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup.
954 this.#stopping = true;
955 // An exit-78 transition queued before this one may have started the owner poll.
956 this.#clearTimer();
957 const child = this.#child?.process;
958 this.#child = null;
959 if (child !== undefined) await this.#terminate(child, signal);
960 this.#done = true;
961 this.#completion.resolve();
962 });
963 await this.#completion.promise.catch(() => undefined);
964 }
965
966 #clearTimer(): void {
967 clearTimeout(this.#timer);
968 this.#timer = undefined;
969 }
970
971 async #selectState(next: ServiceState): Promise<void> {
972 const release = await acquireServiceSelectionLock(this.#statePath);
973 try {
974 const selected = await readServiceState(this.#statePath);
975 // Local installs replace the selection and clear its update record. An old
976 // launcher must not restore its own selection when asked to update again.
977 if (
978 selected.activeVersion !== this.#state.activeVersion ||
979 selected.update?.id !== this.#state.update?.id ||
980 selected.update?.status !== this.#state.update?.status
981 ) {
982 throw new Error("Service selection changed. Restart the service before updating.");
983 }
984 await writeServiceState(this.#statePath, next);
985 this.#state = next;
986 } finally {
987 release();
988 }
989 }
990
991 async #recover(): Promise<void> {
992 // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
993 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
994 const restartPending = restartPendingPath(this.#baseDir);
995 try {
996 const release = await acquireServiceSelectionLock(this.#statePath);
997 try {
998 const selected = await readServiceState(this.#statePath);
999 const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(
1000 () => undefined,
1001 );
1002 const awaited = awaitedVersion?.trim();
1003 if (
1004 selected.activeVersion === this.#state.activeVersion &&
1005 selected.update?.id === this.#state.update?.id &&
1006 selected.update?.status === this.#state.update?.status &&
1007 awaited !== undefined &&
1008 isExactServiceVersion(awaited) &&
1009 compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0
1010 ) {
1011 await NodeFSP.rm(restartPending, { force: true });
1012 }
1013 } finally {
1014 release();
1015 }
1016 } catch (error) {
1017 process.stderr.write(
1018 `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`,
1019 );
1020 }
1021 const update = this.#state.update;
1022 if (update?.status !== "pending") {
1023 if (update !== undefined) {
1024 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
1025 }
1026 try {
1027 await this.#startChild(this.#state.activeVersion, "active", update);
1028 } finally {
1029 if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion);
1030 }
1031 return;
1032 }
1033 if (await databaseRestorePending(this.#baseDir, update)) {
1034 await this.#returnToPrevious(update, "failed", "rollback-interrupted");
1035 return;
1036 }
1037 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {
1038 await this.#returnToPrevious(update, "failed", "target-runtime-missing");
1039 return;
1040 }
1041 await this.#startTrial(update);
1042 }
1043
1044 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
1045 try {
1046 await backupDatabaseOnce(this.#baseDir, pending);
1047 } catch {
1048 await this.#returnToPrevious(pending, "failed", "db-backup-failed");
1049 return;
1050 }
1051 try {
1052 await this.#startChild(pending.targetVersion, "trial", pending);
1053 } catch {
1054 await this.#returnToPrevious(pending, "failed", "candidate-start-failed");
1055 }
1056 }
1057
1058 async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
1059 if (this.#stopping) return;
1060 if (!(await runtimeExists(this.#baseDir, version))) {
1061 throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
1062 }
1063 if (this.#stopping) return;
1064 const paths = runtimePaths(this.#baseDir, version);
1065 const context: ServiceLauncherContext = {
1066 protocol: SERVICE_LAUNCHER_PROTOCOL,
1067 childVersion: version,
1068 ...(update === undefined ? {} : { update }),
1069 };
1070 const spawnArguments = runtimeSpawnArguments(paths);
1071 const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, {
1072 env: { ...this.#environment, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) },
1073 stdio:
1074 this.#output === undefined
1075 ? ["inherit", "inherit", "inherit", "ipc"]
1076 : ["ignore", this.#output, this.#output, "ipc"],
1077 });
1078 await new Promise<void>((resolve, reject) => {
1079 const onError = (error: Error) => reject(error);
1080 child.once("error", onError);
1081 child.once("spawn", () => {
1082 child.removeListener("error", onError);
1083 child.on("error", (error) => this.#enqueue(() => Promise.reject(error)));
1084 resolve();
1085 });
1086 });
1087 if (this.#stopping) {
1088 await this.#terminate(child);
1089 return;
1090 }
1091
1092 const managed: ManagedChild = {
1093 version,
1094 role,
1095 process: child,
1096 };
1097 this.#child = managed;
1098 child.on("message", (value) => {
1099 const message = decodeServiceLauncherChildMessage(value);
1100 if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message));
1101 });
1102 child.once("exit", (code, signal) =>
1103 this.#enqueue(() => this.#handleExit(managed, code, signal)),
1104 );
1105
1106 if (role === "trial") {
1107 this.#timer = setTimeout(
1108 () => this.#enqueue(() => this.#handlePreparedTimeout(managed)),
1109 PREPARED_TIMEOUT_MS,
1110 );
1111 }
1112 }
1113
1114 async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise<void> {
1115 if (this.#child !== child || this.#stopping) return;
1116 if (message.type === "request-update") {
1117 await this.#handleUpdateRequest(child, message);
1118 return;
1119 }
1120 await this.#handlePrepared(child, message.updateId);
1121 }
1122
1123 async #handleUpdateRequest(
1124 child: ManagedChild,
1125 message: Extract<ServiceLauncherChildMessage, { readonly type: "request-update" }>,
1126 ): Promise<void> {
1127 const reject = (reason: string) =>
1128 sendMessage(child.process, { type: "update-rejected", reason });
1129 if (child.role !== "active") {
1130 await reject("Only the active server can request an update.");
1131 return;
1132 }
1133 if (child.version !== this.#state.activeVersion) {
1134 await reject("The requesting server is not the selected active version.");
1135 return;
1136 }
1137 if (this.#state.update?.status === "pending") {
1138 await reject("Another server update is already pending.");
1139 return;
1140 }
1141 if (!isExactServiceVersion(message.targetVersion)) {
1142 await reject("The requested target is not an exact version.");
1143 return;
1144 }
1145 if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) {
1146 await reject("Remote updates must select a newer server version.");
1147 return;
1148 }
1149 if (!NodePath.isAbsolute(message.dbPath)) {
1150 await reject("The requested database path is not absolute.");
1151 return;
1152 }
1153 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {
1154 await reject("The requested target runtime is missing or incomplete.");
1155 return;
1156 }
1157 // Failed checks must refuse updates rather than stop the server with unknown backup capacity.
1158 const backupBlocker = await databaseBackupBlocker(
1159 this.#baseDir,
1160 message.dbPath,
1161 this.#backupSpaceProbe,
1162 ).catch(
1163 (error: unknown) =>
1164 `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`,
1165 );
1166 if (!child.process.connected) return;
1167 if (backupBlocker !== undefined) {
1168 await reject(backupBlocker);
1169 return;
1170 }
1171
1172 const pending: PendingServiceUpdate = {
1173 id: NodeCrypto.randomUUID(),
1174 fromVersion: child.version,
1175 targetVersion: message.targetVersion,
1176 dbPath: message.dbPath,
1177 status: "pending",
1178 };
1179 const next: ServiceState = { ...this.#state, update: pending };
1180 try {
1181 await this.#selectState(next);
1182 } catch (error) {
1183 await reject(error instanceof Error ? error.message : String(error));
1184 return;
1185 }
1186 await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
1187 this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
1188 }
1189
1190 async #beginTrial(child: ManagedChild): Promise<void> {
1191 const pending = this.#state.update;
1192 if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") {
1193 return;
1194 }
1195 this.#timer = undefined;
1196 // Anything thrown outside its own fallback still leaves the offline backup to run.
1197 const finishOnlineBackup = await startOnlineDatabaseBackup(
1198 this.#baseDir,
1199 pending,
1200 this.#backupSpaceProbe,
1201 ).catch(() => undefined);
1202 this.#child = null;
1203 await this.#terminate(child.process);
1204 await finishOnlineBackup?.();
1205 // A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
1206 if (this.#stopRequested) return;
1207 await this.#startTrial(pending);
1208 }
1209
1210 async #handlePrepared(child: ManagedChild, updateId: string): Promise<void> {
1211 const pending = this.#state.update;
1212 if (
1213 child.role !== "trial" ||
1214 pending?.status !== "pending" ||
1215 pending.id !== updateId ||
1216 pending.targetVersion !== child.version
1217 ) {
1218 if (child.role === "trial" && pending?.status === "pending") {
1219 await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child);
1220 return;
1221 }
1222 throw new Error("Trial child reported prepared for an unexpected update.");
1223 }
1224 this.#clearTimer();
1225 const committed = terminalUpdate({ pending, status: "committed" });
1226 const next: ServiceState = {
1227 ...this.#state,
1228 activeVersion: pending.targetVersion,
1229 update: committed,
1230 };
1231 await this.#selectState(next);
1232 child.role = "active";
1233 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
1234 try {
1235 await sendMessage(child.process, { type: "committed", updateId: committed.id });
1236 } finally {
1237 await this.#repointServiceUnit(committed.targetVersion);
1238 }
1239 }
1240
1241 // Selection is durable before this best-effort unit reconciliation.
1242 async #repointServiceUnit(version: string): Promise<void> {
1243 const unit = this.#serviceUnit;
1244 if (unit === undefined) return;
1245 try {
1246 const release = await acquireServiceSelectionLock(unit.path);
1247 try {
1248 // The installer writes state before the unit under this lease. Do not
1249 // acquire its state lease here: the installer takes that lease first.
1250 const selected = await readServiceState(this.#statePath);
1251 if (
1252 selected.activeVersion !== this.#state.activeVersion ||
1253 selected.update?.id !== this.#state.update?.id ||
1254 selected.update?.status !== this.#state.update?.status
1255 ) {
1256 return;
1257 }
1258 const contents = await NodeFSP.readFile(unit.path, "utf8");
1259 const installedBaseDir = serviceUnitBaseDirOf(contents);
1260 if (
1261 installedBaseDir === undefined ||
1262 NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir)
1263 ) {
1264 return;
1265 }
1266 const repointed = repointServiceUnit(contents, version);
1267 if (repointed !== contents) await replaceFileDurably(unit.path, repointed);
1268 // A previous rename may have succeeded before reload failed.
1269 await unit.reload();
1270 } finally {
1271 release();
1272 }
1273 } catch (error) {
1274 process.stderr.write(
1275 `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`,
1276 );
1277 }
1278 }
1279
1280 async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
1281 const pending = this.#state.update;
1282 if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") {
1283 return;
1284 }
1285 this.#timer = undefined;
1286 await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child);
1287 }
1288
1289 async #handleExit(
1290 child: ManagedChild,
1291 code: number | null,
1292 signal: NodeJS.Signals | null,
1293 ): Promise<void> {
1294 if (this.#child !== child || this.#stopping) return;
1295 this.#child = null;
1296 if (code === STATE_DIR_OWNED_EXIT_CODE) {
1297 await this.#idleWhileStateDirOwned();
1298 return;
1299 }
1300 if (child.role === "trial") {
1301 this.#clearTimer();
1302 const pending = this.#state.update;
1303 if (pending?.status !== "pending") {
1304 throw new Error("Trial child exited without matching pending state.");
1305 }
1306 await this.#returnToPrevious(
1307 pending,
1308 "rolled-back",
1309 `candidate-exited:${String(code ?? signal ?? "unknown")}`,
1310 );
1311 return;
1312 }
1313
1314 this.#clearTimer();
1315 const pending = this.#state.update;
1316 if (pending?.status === "pending") {
1317 await this.#startTrial(pending);
1318 return;
1319 }
1320 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
1321 }
1322
1323 // Stay idle until the lock frees to avoid supervisor restart loops. A pending update resumes its
1324 // trial, since the other owner may have migrated the database past fromVersion. It stays pending
1325 // only once its backup is gone, so no later trial, even after a launcher restart, restores it over
1326 // the other owner's writes; exit 78 comes before the child opens the database.
1327 async #idleWhileStateDirOwned(): Promise<void> {
1328 this.#clearTimer();
1329 const pending = this.#state.update;
1330 if (pending?.status === "pending") {
1331 const discarded = await discardDatabaseBackup(this.#baseDir, pending.id).then(
1332 () => true,
1333 () => false,
1334 );
1335 if (!discarded) {
1336 const next: ServiceState = {
1337 ...this.#state,
1338 activeVersion: pending.fromVersion,
1339 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
1340 };
1341 await this.#selectState(next);
1342 await this.#repointServiceUnit(next.activeVersion);
1343 }
1344 }
1345 this.#log(
1346 "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n",
1347 );
1348 this.#stateDirFreeProbes = 0;
1349 // The interval also keeps Node alive: signal listeners alone do not.
1350 this.#timer = setInterval(
1351 () => this.#enqueue(() => this.#reclaimStateDir()),
1352 this.#stateDirOwnerPollMs,
1353 );
1354 }
1355
1356 async #reclaimStateDir(): Promise<void> {
1357 if (this.#stopRequested) {
1358 this.#clearTimer();
1359 return;
1360 }
1361 if (this.#timer === undefined || this.#child !== null) return;
1362 const owned = await stateDirOwned(this.#baseDir);
1363 // A stop during the probe clears the timer.
1364 if (this.#timer === undefined) return;
1365 // A relaunching desktop app frees the home for a few seconds; leave it that gap.
1366 this.#stateDirFreeProbes = owned ? 0 : this.#stateDirFreeProbes + 1;
1367 if (this.#stateDirFreeProbes < 2) return;
1368 this.#clearTimer();
1369 this.#log("[service-launcher] The T3 home is free again. Starting the server.\n");
1370 const update = this.#state.update;
1371 if (update?.status === "pending") await this.#startTrial(update);
1372 else await this.#startChild(this.#state.activeVersion, "active", update);
1373 }
1374
1375 async #returnToPrevious(
1376 pending: PendingServiceUpdate,
1377 status: "rolled-back" | "failed",
1378 reason: string,
1379 child?: ManagedChild,
1380 ): Promise<void> {
1381 if (child !== undefined) {
1382 this.#child = null;
1383 await this.#terminate(child.process);
1384 }
1385 await restoreDatabaseBackup(this.#baseDir, pending);
1386 const outcome = terminalUpdate({ pending, status, reason });
1387 const next: ServiceState = {
1388 ...this.#state,
1389 activeVersion: pending.fromVersion,
1390 update: outcome,
1391 };
1392 await this.#selectState(next);
1393 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1394 try {
1395 await this.#startChild(next.activeVersion, "active", outcome);
1396 } finally {
1397 await this.#repointServiceUnit(next.activeVersion);
1398 }
1399 }
1400}
1401
1402/** Task Scheduler cannot set variables or redirect output, so a Windows task passes them as flags. */
1403export interface LauncherOptions {
1404 readonly baseDir?: string | undefined;
1405 readonly logFile?: string | undefined;
1406 readonly serviceUnit?: string | undefined;
1407}
1408
1409export async function main(options: LauncherOptions = {}): Promise<void> {
1410 const baseDir = options.baseDir ?? process.env.T3CODE_HOME;
1411 if (baseDir === undefined || baseDir.trim() === "") {
1412 throw new Error("T3CODE_HOME is required by the T3 Code service launcher.");
1413 }
1414 const output =
1415 options.logFile === undefined ? undefined : NodeFS.openSync(options.logFile, "a", 0o600);
1416 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
1417 const windows = process.platform === "win32";
1418 let logTimer: NodeJS.Timeout | undefined;
1419 try {
1420 // What the service manager started the launcher with. It locates the unit; service.env
1421 // changes only what the server gets.
1422 const managerEnvironment = {
1423 ...process.env,
1424 T3CODE_HOME: baseDir,
1425 ...(options.serviceUnit === undefined ? {} : { [SERVICE_UNIT_ENV]: options.serviceUnit }),
1426 };
1427 const environment = await readServiceEnvironment(baseDir, managerEnvironment);
1428 const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
1429 const contents = await NodeFSP.readFile(statePath, "utf8");
1430 let state = parseServiceState(contents);
1431 if (state === undefined) {
1432 state = adoptLegacyServiceState(contents);
1433 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
1434 if (!(await runtimeExists(baseDir, state.activeVersion))) {
1435 throw new Error(`Selected t3@${state.activeVersion} runtime is missing or incomplete.`);
1436 }
1437 }
1438 const serviceUnit = await serviceUnitFromEnvironment(managerEnvironment);
1439 const logPath = await serviceLogPath(options, serviceUnit);
1440 if (logPath !== undefined) {
1441 await boundServiceLog(logPath);
1442 logTimer = setInterval(
1443 () => void boundServiceLog(logPath),
1444 SERVICE_LOG_CHECK_INTERVAL_MS,
1445 ).unref();
1446 }
1447 const launcher = new Launcher(baseDir, state, {
1448 environment,
1449 ...(serviceUnit === undefined ? {} : { serviceUnit }),
1450 ...(output === undefined ? {} : { output }),
1451 ...(windows ? { lockPath: serviceLauncherLockPath(baseDir) } : {}),
1452 });
1453 // Ending a scheduled task kills its console host without a signal; the console read fails instead.
1454 const watchConsole = windows && process.stdin.isTTY === true;
1455 const onConsoleLost = () => void launcher.stop("SIGTERM");
1456 if (watchConsole) {
1457 process.stdin.on("error", () => undefined);
1458 process.stdin.once("close", onConsoleLost);
1459 process.stdin.resume();
1460 }
1461 try {
1462 await launcher.run();
1463 } finally {
1464 if (watchConsole) {
1465 process.stdin.off("close", onConsoleLost);
1466 process.stdin.destroy();
1467 }
1468 }
1469 } catch (cause) {
1470 if (output !== undefined) {
1471 NodeFS.writeSync(
1472 output,
1473 `[service-launcher] ${cause instanceof Error ? cause.message : String(cause)}\n`,
1474 );
1475 }
1476 // Agents the server left behind keep the console host, and so the task, running; ending it
1477 // lets the next repeating trigger start the service again.
1478 if (windows && options.serviceUnit !== undefined) {
1479 const taskName = serviceTaskName(process.env.USERDOMAIN ?? "", process.env.USERNAME ?? "");
1480 NodeChildProcess.spawnSync("schtasks", ["/End", "/TN", taskName], {
1481 stdio: "ignore",
1482 windowsHide: true,
1483 });
1484 }
1485 throw cause;
1486 } finally {
1487 clearInterval(logTimer);
1488 if (output !== undefined) NodeFS.closeSync(output);
1489 }
1490}