Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

serverRuntimeState.test.ts

515 lines20.6 KB

1// @effect-diagnostics nodeBuiltinImport:off - Injects a real file replacement during the native SQLite transaction.
2import * as NodeFS from "node:fs";
3import * as NodeOS from "node:os";
4import * as NodeSqlite from "node:sqlite";
5​
6import { assert, describe, it } from "@effect/vitest";
7import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
8import * as NodeServices from "@effect/platform-node/NodeServices";
9import * as HostProcess from "@t3tools/shared/HostProcess";
10import * as Effect from "effect/Effect";
11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";
13import * as Logger from "effect/Logger";
14import * as Option from "effect/Option";
15import * as Path from "effect/Path";
16import * as Runtime from "effect/Runtime";
17import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
18import * as TestClock from "effect/testing/TestClock";
19import { vi } from "vite-plus/test";
20​
21import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts";
22import { ServiceLauncherHostProcess } from "./cloud/serviceLauncherClient.ts";
23import { SERVICE_LAUNCHER_CONTEXT_ENV } from "./cloud/serviceProtocol.ts";
24import * as ProcessRunner from "./processRunner.ts";
25import * as ServerOwnership from "./serverOwnership.ts";
26import * as ServerRuntimeState from "./serverRuntimeState.ts";
27​
28const startBesideBootService = Effect.fn("test.startBesideBootService")(function* (input: {
29 readonly platform: "darwin" | "linux";
30 readonly outputs: Readonly<Record<string, { readonly code: number; readonly stdout?: string }>>;
31 readonly unitServesOtherHome?: boolean;
32 readonly restartPending?: boolean;
33 readonly activeVersion?: string;
34 readonly npmRuntime?: boolean;
35 readonly missingServiceState?: boolean;
36 readonly environment?: NodeJS.ProcessEnv;
37 readonly connected?: boolean;
38}) {
39 const fs = yield* FileSystem.FileSystem;
40 const path = yield* Path.Path;
41 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-boot-service-" });
42 const t3Home = path.join(root, ".t3");
43 const serviceHome = input.unitServesOtherHome ? path.join(root, "other") : t3Home;
44 for (const home of [t3Home, serviceHome]) {
45 yield* fs.makeDirectory(path.join(home, "userdata"), { recursive: true });
46 }
47 yield* fs.makeDirectory(path.join(serviceHome, "runtime"));
48 const activeVersion = input.activeVersion ?? "0.0.42";
49 if (!input.missingServiceState) {
50 yield* fs.writeFileString(
51 path.join(serviceHome, "runtime", "service-state.json"),
52 JSON.stringify({ protocol: input.npmRuntime ? 2 : 3, activeVersion }),
53 );
54 }
55 if (input.npmRuntime) {
56 const versionDir = path.join(serviceHome, "runtime", "versions", activeVersion);
57 const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs");
58 yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true });
59 yield* fs.writeFileString(entryPath, "");
60 }
61​
62 if (input.restartPending) {
63 yield* fs.writeFileString(
64 path.join(serviceHome, "runtime", ".restart-pending"),
65 `${activeVersion}\n`,
66 );
67 }
68 const plan = {
69 program: input.npmRuntime
70 ? ["/usr/bin/node", path.join(serviceHome, "runtime", "service-launcher.mjs")]
71 : [],
72 baseDir: serviceHome,
73 logPath: "",
74 unitPath: "",
75 };
76 const [unitPath, unit] =
77 input.platform === "darwin"
78 ? [
79 path.join(root, "Library", "LaunchAgents", "com.t3tools.t3code.service.plist"),
80 renderBootServicePlist(plan, { homeDir: root, environmentPath: "" }),
81 ]
82 : [
83 path.join(root, ".config", "systemd", "user", "t3code.service"),
84 renderBootServiceUnit(plan),
85 ];
86 yield* fs.makeDirectory(path.dirname(unitPath), { recursive: true });
87 yield* fs.writeFileString(unitPath, unit);
88​
89 const probes: string[] = [];
90 const runner = ProcessRunner.ProcessRunner.of({
91 run: (run) =>
92 Effect.gen(function* () {
93 const probe = `${path.basename(run.command)} ${run.args.find((arg) => !arg.startsWith("--"))}`;
94 probes.push(probe);
95 const output = input.outputs[probe];
96 return yield* output === undefined
97 ? Effect.fail(
98 new ProcessRunner.ProcessSpawnError({
99 command: run.command,
100 argumentCount: run.args.length,
101 cause: new Error("not found"),
102 }),
103 )
104 : Effect.succeed({
105 stdout: output.stdout ?? "",
106 stderr: "",
107 code: ChildProcessSpawner.ExitCode(output.code),
108 timedOut: false,
109 stdoutTruncated: false,
110 stderrTruncated: false,
111 stdoutInvalidUtf8: false,
112 stderrInvalidUtf8: false,
113 });
114 }),
115 });
116 const result = yield* Effect.scoped(
117 ServerOwnership.acquireServerOwnership(path.join(t3Home, "userdata", "server-runtime.json")),
118 ).pipe(
119 Effect.provideService(ProcessRunner.ProcessRunner, runner),
120 Effect.provideService(HostProcess.Platform, input.platform),
121 Effect.provideService(HostProcess.UserId, 501),
122 Effect.provideService(HostProcess.HomeDirectory, root),
123 Effect.provideService(HostProcess.Environment, input.environment ?? {}),
124 Effect.provideService(ServiceLauncherHostProcess, {
125 connected: input.connected ?? false,
126 send: () => false,
127 on: () => undefined,
128 off: () => undefined,
129 }),
130 Effect.result,
131 );
132 assert.includeMembers(
133 ["launchctl print", "launchctl print-disabled", "systemctl is-active", "systemctl is-enabled"],
134 probes,
135 );
136 return { result, t3Home };
137});
138​
139describe("server runtime state machine identity", () => {
140 it.effect("stamps the writing machine and recognises records from other machines", () =>
141 Effect.gen(function* () {
142 const state = yield* ServerRuntimeState.makePersistedServerRuntimeState({
143 config: { host: undefined, devUrl: undefined },
144 port: 3773,
145 });
146 assert.equal(state.hostname, NodeOS.hostname());
147 assert.isFalse(ServerRuntimeState.isRuntimeStateFromAnotherMachine(state));
148 assert.isTrue(
149 ServerRuntimeState.isRuntimeStateFromAnotherMachine({
150 ...state,
151 hostname: `${NodeOS.hostname()}-peer`,
152 }),
153 );
154 // Records written before the field existed carry no hostname and stay local.
155 assert.isFalse(
156 ServerRuntimeState.isRuntimeStateFromAnotherMachine({ ...state, hostname: undefined }),
157 );
158 }),
159 );
160});
161​
162describe("server ownership", () => {
163 // An empty home, so no boot service unit installed on this machine is read.
164 const layerOwnership = Layer.mergeAll(
165 ProcessRunner.layer,
166 Layer.effect(
167 HostProcess.HomeDirectory,
168 Effect.gen(function* () {
169 return yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped();
170 }),
171 ),
172 Layer.succeed(HostProcess.Environment, {}),
173 ).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)));
174 const makeState = ServerRuntimeState.makePersistedServerRuntimeState({
175 config: { host: undefined, devUrl: undefined },
176 port: 3773,
177 });
178​
179 it.effect("lets only one of two racing starts own a state directory", () =>
180 Effect.gen(function* () {
181 const fs = yield* FileSystem.FileSystem;
182 const path = yield* Path.Path;
183 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-race-" });
184 const statePath = path.join(root, "server-runtime.json");
185 const attempts = yield* Effect.all(
186 [
187 ServerOwnership.acquireServerOwnership(statePath).pipe(Effect.result),
188 ServerOwnership.acquireServerOwnership(statePath).pipe(Effect.result),
189 ],
190 { concurrency: "unbounded" },
191 );
192 assert.equal(attempts.filter((result) => result._tag === "Success").length, 1);
193 const refused = attempts.find((result) => result._tag === "Failure");
194 assert.equal(refused?.failure._tag, "ServerAlreadyRunningError");
195 }).pipe(Effect.provide(layerOwnership)),
196 );
197​
198 it.effect("refuses a second owner without touching its record, then allows a restart", () =>
199 Effect.gen(function* () {
200 const fs = yield* FileSystem.FileSystem;
201 const path = yield* Path.Path;
202 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-" });
203 const statePath = path.join(root, "server-runtime.json");
204 const state = yield* makeState;
205 yield* Effect.scoped(
206 Effect.gen(function* () {
207 const owner = yield* ServerOwnership.acquireServerOwnership(statePath);
208 yield* owner.publish(state);
209 const before = yield* fs.readFileString(statePath);
210 const refused = yield* ServerOwnership.acquireServerOwnership(statePath).pipe(
211 Effect.flip,
212 );
213 assert.equal(refused._tag, "ServerAlreadyRunningError");
214 assert.include(refused.message, "do not delete it while a server is running");
215 assert.equal(yield* fs.readFileString(statePath), before);
216 }),
217 );
218 assert.isFalse(yield* fs.exists(statePath));
219 yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath));
220 }).pipe(Effect.provide(layerOwnership)),
221 );
222​
223 describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("lock file checks", () => {
224 it.effect.each([
225 "unchanged",
226 "unlink",
227 "replace",
228 "during acquisition",
229 "first acquisition",
230 // Root bypasses directory permissions, so chmod cannot force EACCES.
231 ...(process.getuid?.() === 0 ? [] : (["stat error"] as const)),
232 ] as const)("monitors ownership after %s", (operation) => {
233 const messages: unknown[] = [];
234 const warnings: unknown[] = [];
235 const logger = Logger.make(({ message, logLevel }) =>
236 (logLevel === "Warn" ? warnings : messages).push(message),
237 );
238 return Effect.gen(function* () {
239 const fs = yield* FileSystem.FileSystem;
240 const path = yield* Path.Path;
241 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-lost-" });
242 const statePath = path.join(root, "server-runtime.json");
243 const lockPath = path.join(root, "server-owner.sqlite");
244 const replacement = path.join(root, "replacement.sqlite");
245 if (operation === "during acquisition" || operation === "first acquisition") {
246 if (operation === "during acquisition") yield* fs.writeFileString(lockPath, "");
247 yield* fs.writeFileString(replacement, "");
248 const exec = NodeSqlite.DatabaseSync.prototype.exec;
249 yield* Effect.acquireRelease(
250 Effect.sync(() =>
251 vi.spyOn(NodeSqlite.DatabaseSync.prototype, "exec").mockImplementationOnce(function (
252 this: NodeSqlite.DatabaseSync,
253 sql: string,
254 ) {
255 exec.call(this, sql);
256 NodeFS.renameSync(replacement, lockPath);
257 }),
258 ),
259 (spy) => Effect.sync(() => spy.mockRestore()),
260 );
261 }
262 yield* ServerOwnership.acquireServerOwnership(statePath);
263 if (operation === "unlink") {
264 yield* fs.remove(lockPath);
265 } else if (operation === "replace") {
266 yield* fs.writeFileString(replacement, "");
267 yield* fs.rename(replacement, lockPath);
268 }
269 if (operation === "stat error") {
270 yield* fs.chmod(root, 0);
271 yield* TestClock.adjust("15 seconds").pipe(
272 Effect.ensuring(fs.chmod(root, 0o700).pipe(Effect.orDie)),
273 );
274 assert.deepEqual(warnings, [
275 [`Cannot check server ownership lock at ${lockPath} (EACCES).`],
276 ]);
277 yield* TestClock.adjust("5 seconds");
278 assert.deepEqual(messages, []);
279 yield* fs.remove(lockPath);
280 }
281 yield* TestClock.adjust("5 seconds");
282 const expected =
283 operation === "unchanged"
284 ? []
285 : [
286 [
287 `Server ownership lock at ${lockPath} was deleted or replaced. Another server may now share this T3 home. Stop any other server using this home, then restart this server. Do not delete server-owner.sqlite while a server is running.`,
288 ],
289 ];
290 assert.deepEqual(messages, expected);
291 if (operation !== "unchanged") {
292 yield* fs.remove(`${lockPath}-journal`, { force: true });
293 yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath));
294 }
295 yield* TestClock.adjust("15 seconds");
296 assert.deepEqual(messages, expected);
297 assert.lengthOf(warnings, operation === "stat error" ? 1 : 0);
298 }).pipe(
299 Effect.provide(
300 Layer.merge(layerOwnership, Logger.layer([logger], { mergeWithExisting: false })),
301 ),
302 );
303 });
304 });
305​
306 it.effect("refuses beside a live pre-lock server and ignores a stale record", () =>
307 Effect.gen(function* () {
308 const fs = yield* FileSystem.FileSystem;
309 const path = yield* Path.Path;
310 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-legacy-" });
311 const statePath = path.join(root, "server-runtime.json");
312 const state = yield* makeState;
313 yield* ServerRuntimeState.persistServerRuntimeState({
314 path: statePath,
315 // Any record time after this process started proves the PID was not reused.
316 state: { ...state, startedAt: "2100-01-01T00:00:00.000Z" },
317 });
318 const refused = yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)).pipe(
319 Effect.flip,
320 );
321 assert.equal(refused._tag, "ServerAlreadyRunningError");
322 yield* ServerRuntimeState.persistServerRuntimeState({
323 path: statePath,
324 state: { ...state, pid: 0 },
325 });
326 yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath));
327 }).pipe(Effect.provide(layerOwnership)),
328 );
329​
330 it.effect("never removes a runtime record another server published", () =>
331 Effect.gen(function* () {
332 const fs = yield* FileSystem.FileSystem;
333 const path = yield* Path.Path;
334 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-cleanup-" });
335 const statePath = path.join(root, "server-runtime.json");
336 const state = yield* makeState;
337 yield* Effect.scoped(
338 Effect.gen(function* () {
339 const owner = yield* ServerOwnership.acquireServerOwnership(statePath);
340 yield* owner.publish(state);
341 yield* ServerRuntimeState.persistServerRuntimeState({
342 path: statePath,
343 state: { ...state, ownerId: "other-owner" },
344 });
345 }),
346 );
347 const remaining = yield* ServerRuntimeState.readPersistedServerRuntimeState(statePath);
348 assert.equal(Option.getOrThrow(remaining).ownerId, "other-owner");
349 }).pipe(Effect.provide(layerOwnership)),
350 );
351​
352 describe("beside an installed boot service", () => {
353 it.effect("refuses an old active or scheduled service even without its runtime record", () =>
354 Effect.gen(function* () {
355 for (const fixture of [
356 {
357 platform: "darwin",
358 outputs: { "launchctl print": { code: 0 } },
359 },
360 {
361 platform: "darwin",
362 outputs: {
363 "launchctl print": { code: 113 },
364 "launchctl print-disabled": {
365 code: 0,
366 stdout: '"com.t3tools.t3code.service" => false',
367 },
368 },
369 },
370 {
371 platform: "linux",
372 outputs: {
373 // Restart= keeps an otherwise disabled service alive during its restart delay.
374 "systemctl is-active": { code: 3, stdout: "activating\n" },
375 "systemctl is-enabled": { code: 1, stdout: "disabled\n" },
376 },
377 },
378 {
379 platform: "linux",
380 outputs: {
381 "systemctl is-active": { code: 3, stdout: "inactive\n" },
382 "systemctl is-enabled": { code: 0, stdout: "enabled\n" },
383 },
384 },
385 {
386 platform: "darwin",
387 outputs: { "launchctl print": { code: 0 } },
388 environment: { [SERVICE_LAUNCHER_CONTEXT_ENV]: "{}" },
389 },
390 ] as const) {
391 const { result, t3Home } = yield* startBesideBootService(fixture);
392 assert.equal(result._tag, "Failure");
393 if (result._tag !== "Failure") return;
394 assert.equal(result.failure._tag, "OlderBootServiceError");
395 assert.equal(Runtime.getErrorExitCode(result.failure), 78);
396 assert.include(result.failure.message, t3Home);
397 assert.include(result.failure.message, "t3 update");
398 assert.include(result.failure.message, "t3 service restart");
399 assert.include(result.failure.message, "No server was stopped");
400 }
401 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
402 );
403​
404 it.effect(
405 "refuses an active npm-installed service with no standalone executable or runtime record",
406 () =>
407 Effect.gen(function* () {
408 const { result } = yield* startBesideBootService({
409 platform: "linux",
410 activeVersion: "0.0.40",
411 npmRuntime: true,
412 outputs: {
413 "systemctl is-active": { code: 0, stdout: "active\n" },
414 },
415 });
416 assert.equal(result._tag, "Failure");
417 if (result._tag !== "Failure") return;
418 assert.equal(result.failure._tag, "OlderBootServiceError");
419 assert.equal(Runtime.getErrorExitCode(result.failure), 78);
420 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
421 );
422​
423 it.effect("refuses while the loaded service has not restarted into its lock-aware update", () =>
424 Effect.gen(function* () {
425 const { result } = yield* startBesideBootService({
426 platform: "darwin",
427 restartPending: true,
428 activeVersion: "0.1.0",
429 outputs: {
430 "launchctl print": { code: 0 },
431 },
432 });
433 assert.equal(result._tag === "Failure" && result.failure._tag, "OlderBootServiceError");
434 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
435 );
436​
437 it.effect("allows lock-aware stable services that are loaded or stopped and enabled", () =>
438 Effect.gen(function* () {
439 for (const fixture of [
440 {
441 platform: "darwin",
442 outputs: {
443 "launchctl print": { code: 0 },
444 },
445 },
446 {
447 platform: "linux",
448 outputs: {
449 "systemctl is-active": { code: 3, stdout: "inactive\n" },
450 "systemctl is-enabled": { code: 0, stdout: "enabled\n" },
451 },
452 },
453 ] as const) {
454 const { result } = yield* startBesideBootService({ ...fixture, activeVersion: "0.1.0" });
455 assert.equal(result._tag, "Success");
456 }
457 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
458 );
459​
460 it.effect("allows services that cannot compete for this home", () =>
461 Effect.gen(function* () {
462 const outputs = {
463 "launchctl print": { code: 0 },
464 };
465 for (const fixture of [
466 { platform: "darwin", outputs, unitServesOtherHome: true },
467 {
468 platform: "darwin",
469 outputs,
470 environment: { [SERVICE_LAUNCHER_CONTEXT_ENV]: "{}" },
471 connected: true,
472 },
473 {
474 platform: "darwin",
475 outputs: {
476 "launchctl print": { code: 113 },
477 "launchctl print-disabled": {
478 code: 0,
479 stdout: '"com.t3tools.t3code.service" => disabled',
480 },
481 },
482 },
483 {
484 platform: "linux",
485 outputs: {
486 "systemctl is-active": { code: 3, stdout: "inactive\n" },
487 "systemctl is-enabled": { code: 1, stdout: "disabled\n" },
488 },
489 },
490 ] as const) {
491 const { result } = yield* startBesideBootService(fixture);
492 assert.equal(result._tag, "Success");
493 }
494 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
495 );
496​
497 it.effect("allows startup when the service configuration or manager is unavailable", () =>
498 Effect.gen(function* () {
499 for (const fixture of [
500 { outputs: {} },
501 {
502 outputs: {
503 "launchctl print": { code: 113 },
504 "launchctl print-disabled": { code: 1 },
505 },
506 },
507 { outputs: { "launchctl print": { code: 0 } }, missingServiceState: true },
508 ]) {
509 const { result } = yield* startBesideBootService({ platform: "darwin", ...fixture });
510 assert.equal(result._tag, "Success");
511 }
512 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
513 );
514 });
515});