Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

serverOwnership.ts

297 lines12.2 KB

1// @effect-diagnostics nodeBuiltinImport:off - The ownership lock is a native SQLite handle held for the server's lifetime.
2import * as NodeFS from "node:fs";
3import * as NodePath from "node:path";
4import * as NodeSqlite from "node:sqlite";
5​
6import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts";
7import * as HostProcess from "@t3tools/shared/HostProcess";
8import * as SecureRandom from "effect/crypto/SecureRandom";
9import * as Duration from "effect/Duration";
10import * as Effect from "effect/Effect";
11import * as FileSystem from "effect/FileSystem";
12import * as Option from "effect/Option";
13import * as Runtime from "effect/Runtime";
14import * as Schema from "effect/Schema";
15​
16import {
17 BOOT_SERVICE_LAUNCHD_LABEL,
18 BOOT_SERVICE_PLIST_FILE,
19 BOOT_SERVICE_UNIT_FILE,
20 bootServiceBaseDirOf,
21} from "./cloud/bootService.ts";
22import { ServiceLauncherHostProcess } from "./cloud/serviceLauncherClient.ts";
23import {
24 SERVICE_LAUNCHER_CONTEXT_ENV,
25 SERVICE_RESTART_PENDING_FILE,
26 SERVICE_STATE_FILE,
27 compareExactServiceVersions,
28 serviceStateActiveVersion,
29} from "./cloud/serviceProtocol.ts";
30import * as ProcessRunner from "./processRunner.ts";
31import {
32 clearPersistedServerRuntimeState,
33 isProcessAlive,
34 persistServerRuntimeState,
35 readPersistedServerRuntimeState,
36 type PersistedServerRuntimeState,
37} from "./serverRuntimeState.ts";
38​
39export class ServerAlreadyRunningError extends Schema.TaggedError<ServerAlreadyRunningError>()(
40 "ServerAlreadyRunningError",
41 { stateDir: Schema.String },
42) {
43 // Distinct exit codes let supervisors stop restarting an already-owned server.
44 override readonly [Runtime.errorExitCode] = SERVER_EXIT_CODE_STATE_DIR_OWNED;
45​
46 override get message(): string {
47 return `A T3 Code server already owns ${this.stateDir}. Stop that server, or use a separate T3 home and pair with it, then retry. No server was stopped. server-owner.sqlite is the ownership lock; do not delete it while a server is running.`;
48 }
50​
51export class OlderBootServiceError extends Schema.TaggedError<OlderBootServiceError>()(
52 "OlderBootServiceError",
53 { baseDir: Schema.String },
54) {
55 override readonly [Runtime.errorExitCode] = SERVER_EXIT_CODE_STATE_DIR_OWNED;
56​
57 override get message(): string {
58 return `A T3 Code background service set up for ${this.baseDir} may still run a release without the ownership lock. On that machine, try \`t3 update\` and let it restart (or run \`t3 service restart\` if the update is already installed). If the older launcher cannot update, run \`t3 service uninstall\`, then reinstall with \`t3 service install\` from a current CLI. You can also leave the service uninstalled. Pass \`--base-dir ${this.baseDir}\` to each command, and use \`npx t3@latest\` if \`t3\` is not installed or is outdated. Then retry. No server was stopped.`;
59 }
61​
62export class ServerOwnershipError extends Schema.TaggedError<ServerOwnershipError>()(
63 "ServerOwnershipError",
64 { stateDir: Schema.String, cause: Schema.Defect() },
65) {
66 override get message(): string {
67 return `Could not acquire server ownership of ${this.stateDir}.`;
68 }
70​
71const isSqliteBusy = (cause: unknown) =>
72 cause instanceof Error &&
73 (("errcode" in cause && cause.errcode === 5) ||
74 ("code" in cause && cause.code === "SQLITE_BUSY"));
75​
76// Pre-lock servers publish only runtime state; require start-time proof of PID reuse before treating it as stale.
77const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* (
78 state: PersistedServerRuntimeState,
79) {
80 if (!isProcessAlive(state.pid)) return false;
81 const recordedAt = Date.parse(state.startedAt);
82 if (!Number.isFinite(recordedAt)) return true;
83 const windows = (yield* HostProcess.Platform) === "win32";
84 const runner = yield* ProcessRunner.ProcessRunner;
85 const result = yield* runner
86 .run({
87 command: windows ? "powershell.exe" : "ps",
88 args: windows
89 ? [
90 "-NoProfile",
91 "-NonInteractive",
92 "-Command",
93 `(Get-Process -Id ${state.pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
94 ]
95 : ["-p", String(state.pid), "-o", "lstart="],
96 env: { LC_ALL: "C", TZ: "UTC" },
97 timeout: Duration.seconds(2),
98 maxOutputBytes: 16_384,
99 })
100 .pipe(Effect.option);
101 if (Option.isNone(result) || result.value.code !== 0) return true;
102 const output = result.value.stdout.trim();
103 const startedAt = Date.parse(windows ? output : `${output} UTC`);
104 // ps reports whole seconds. Unknown identity stays conservative.
105 return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000;
106});
107​
108// Pre-lock services become invisible when another server replaces their runtime
109// record. Unreadable configuration or an unavailable manager allows startup.
110const olderBootServiceHome = Effect.fn("olderBootServiceHome")(
111 function* (stateDir: string) {
112 if (
113 (yield* ServiceLauncherHostProcess).connected &&
114 (yield* HostProcess.Environment)[SERVICE_LAUNCHER_CONTEXT_ENV] !== undefined
115 ) {
116 return undefined;
117 }
118 const platform = yield* HostProcess.Platform;
119 const uid = yield* HostProcess.UserId;
120 if (platform !== "linux" && (platform !== "darwin" || uid === undefined)) return undefined;
121​
122 const homeDir = yield* HostProcess.HomeDirectory;
123 const fs = yield* FileSystem.FileSystem;
124 const runner = yield* ProcessRunner.ProcessRunner;
125 const unitPath =
126 platform === "darwin"
127 ? NodePath.join(homeDir, "Library", "LaunchAgents", BOOT_SERVICE_PLIST_FILE)
128 : NodePath.join(homeDir, ".config", "systemd", "user", BOOT_SERVICE_UNIT_FILE);
129 const baseDir = bootServiceBaseDirOf(yield* fs.readFileString(unitPath));
130 if (
131 baseDir === undefined ||
132 (yield* fs.realPath(NodePath.join(baseDir, "userdata"))) !== (yield* fs.realPath(stateDir))
133 ) {
134 return undefined;
135 }
136 const runtimeDir = NodePath.join(baseDir, "runtime");
137 const version = serviceStateActiveVersion(
138 yield* fs.readFileString(NodePath.join(runtimeDir, SERVICE_STATE_FILE)),
139 );
140 if (version === undefined) return undefined;
141 const run = (command: string, args: ReadonlyArray<string>) =>
142 runner.run({ command, args, timeout: Duration.seconds(5), maxOutputBytes: 65_536 });
143 let loaded: boolean;
144 if (platform === "darwin") {
145 loaded =
146 (yield* run("launchctl", ["print", `gui/${uid}/${BOOT_SERVICE_LAUNCHD_LABEL}`])).code === 0;
147 if (!loaded) {
148 const disabled = yield* run("launchctl", ["print-disabled", `gui/${uid}`]);
149 // Older macOS versions print `true` instead of `disabled`.
150 if (
151 disabled.code !== 0 ||
152 disabled.stdout.includes(`"${BOOT_SERVICE_LAUNCHD_LABEL}" => disabled`) ||
153 disabled.stdout.includes(`"${BOOT_SERVICE_LAUNCHD_LABEL}" => true`)
154 )
155 return undefined;
156 }
157 } else {
158 const active = yield* run("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]);
159 // A unit waiting out its Restart= delay is activating, with exit code 3.
160 loaded = active.code === 0 || (active.code === 3 && active.stdout.trim() === "activating");
161 if (!loaded) {
162 const enabled = yield* run("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]);
163 if (enabled.code !== 0 || enabled.stdout.trim() !== "enabled") return undefined;
164 }
165 }
166 // A deferred update names the staged runtime, not the loaded service's runtime.
167 if (loaded && (yield* fs.exists(NodePath.join(runtimeDir, SERVICE_RESTART_PENDING_FILE)))) {
168 return baseDir;
169 }
170 // Stable 0.1.0 shipped the ownership lock. Earlier lock-aware prereleases
171 // are conservatively refused because older runtimes expose no capability marker.
172 return compareExactServiceVersions(version, "0.1.0") < 0 ? baseDir : undefined;
173 },
174 Effect.orElseSucceed(() => undefined),
175);
176​
177const ownershipLockPath = (stateDir: string) => NodePath.join(stateDir, "server-owner.sqlite");
178​
179/**
180 * The lock is an exclusive transaction on `server-owner.sqlite`, which never
181 * holds data and must never be unlinked: the OS releases it when the process
182 * exits, even on SIGKILL, so a crashed owner never blocks the next start.
183 */
184const openOwnershipLock = (stateDir: string) =>
185 Effect.try({
186 try: () => {
187 const lockPath = ownershipLockPath(stateDir);
188 NodeFS.mkdirSync(stateDir, { recursive: true });
189 let before = NodeFS.statSync(lockPath, { bigint: true, throwIfNoEntry: false });
190 if (before === undefined) {
191 NodeFS.closeSync(NodeFS.openSync(lockPath, "a", 0o644));
192 before = NodeFS.statSync(lockPath, { bigint: true });
193 }
194 const db = new NodeSqlite.DatabaseSync(lockPath);
195 try {
196 db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
197 const identity = NodeFS.statSync(lockPath, { bigint: true });
198 return {
199 db,
200 identity,
201 replacedDuringAcquire: before.dev !== identity.dev || before.ino !== identity.ino,
202 };
203 } catch (cause) {
204 db.close();
205 throw cause;
206 }
207 },
208 catch: (cause) =>
209 isSqliteBusy(cause)
210 ? new ServerAlreadyRunningError({ stateDir })
211 : new ServerOwnershipError({ stateDir, cause }),
212 });
213​
214/** Hold only the ownership lock until the scope closes, without claiming the runtime state. */
215export const holdServerOwnershipLock = Effect.fn("holdServerOwnershipLock")(function* (
216 statePath: string,
217) {
218 yield* Effect.acquireRelease(openOwnershipLock(NodePath.dirname(statePath)), ({ db }) =>
219 Effect.sync(() => db.close()),
220 );
221});
222​
223/** Hold an exclusive lock on the state directory until the scope closes. */
224export const acquireServerOwnership = Effect.fn("acquireServerOwnership")(function* (
225 statePath: string,
226) {
227 const stateDir = NodePath.dirname(statePath);
228 const random = yield* SecureRandom.SecureRandom;
229 const ownerId = yield* random
230 .uuidV4()
231 .pipe(Effect.mapError((cause) => new ServerOwnershipError({ stateDir, cause })));
232 const lockPath = ownershipLockPath(stateDir);
233 const lock = yield* Effect.acquireRelease(openOwnershipLock(stateDir), ({ db }) =>
234 readPersistedServerRuntimeState(statePath).pipe(
235 Effect.flatMap((state) =>
236 Option.isSome(state) && state.value.ownerId === ownerId
237 ? clearPersistedServerRuntimeState(statePath)
238 : Effect.void,
239 ),
240 Effect.ignore({ log: true }),
241 Effect.ensuring(Effect.sync(() => db.close())),
242 ),
243 );
244​
245 const previous = yield* readPersistedServerRuntimeState(statePath).pipe(
246 Effect.orElseSucceed(() => Option.none<PersistedServerRuntimeState>()),
247 );
248 if (
249 Option.isSome(previous) &&
250 previous.value.ownerId === undefined &&
251 (yield* legacyOwnerIsLive(previous.value))
252 ) {
253 return yield* new ServerAlreadyRunningError({ stateDir });
254 }
255 const olderServiceHome = yield* olderBootServiceHome(stateDir);
256 if (olderServiceHome !== undefined) {
257 return yield* new OlderBootServiceError({ baseDir: olderServiceHome });
258 }
259​
260 yield* Effect.gen(function* () {
261 let lost = lock.replacedDuringAcquire;
262 let statWarningLogged = false;
263 while (!lost) {
264 yield* Effect.sleep("5 seconds");
265 // Closing another descriptor for this inode releases SQLite's POSIX locks. Stat by path only.
266 const current = yield* Effect.sync(() => {
267 try {
268 return NodeFS.statSync(lockPath, { bigint: true, throwIfNoEntry: false });
269 } catch (cause) {
270 return cause instanceof Error && "code" in cause ? String(cause.code) : "unknown";
271 }
272 });
273 if (typeof current === "string") {
274 if (!statWarningLogged) {
275 yield* Effect.logWarning(
276 `Cannot check server ownership lock at ${lockPath} (${current}).`,
277 );
278 statWarningLogged = true;
279 }
280 continue;
281 }
282 statWarningLogged = false;
283 lost =
284 current === undefined ||
285 current.dev !== lock.identity.dev ||
286 current.ino !== lock.identity.ino;
287 }
288 yield* Effect.logError(
289 `Server ownership lock at ${lockPath} was deleted or replaced. Another server may now share this T3 home. Stop any other server using this home, then restart this server. Do not delete server-owner.sqlite while a server is running.`,
290 );
291 }).pipe(Effect.forkScoped);
292​
293 return {
294 publish: (state: PersistedServerRuntimeState) =>
295 persistServerRuntimeState({ path: statePath, state: { ...state, ownerId } }),
296 };
297});