Open History to see the last commit
1import * as Brand from "effect/Brand";
2import * as Cause from "effect/Cause";
3import * as Duration from "effect/Duration";
4import * as Effect from "effect/Effect";
5import * as Exit from "effect/Exit";
6import * as Fiber from "effect/Fiber";
7import * as FileSystem from "effect/FileSystem";
8import * as Option from "effect/Option";
9import * as PlatformError from "effect/PlatformError";
10import * as Ref from "effect/Ref";
11import * as SubscriptionRef from "effect/SubscriptionRef";
12import * as Schema from "effect/Schema";
13import * as Semaphore from "effect/Semaphore";
14import * as Scope from "effect/Scope";
15import * as Stream from "effect/Stream";
16import { HttpClient } from "effect/http";
17import { ChildProcess, ChildProcessSpawner } from "effect/process";
18
19import {
20 SERVER_EXIT_CODE_MANAGED_POLICY,
21 SERVER_EXIT_CODE_STATE_DIR_OWNED,
22 DesktopBackendBootstrap,
23 type DesktopBackendBootstrap as DesktopBackendBootstrapValue,
24 PRIMARY_LOCAL_ENVIRONMENT_ID,
25 DesktopTelemetryControlMessage,
26 type DesktopTelemetryControlMessage as DesktopTelemetryControlMessageValue,
27} from "@t3tools/contracts";
28import { isRunningDatabaseMigrations } from "@t3tools/shared/databaseMigrationLog";
29import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness";
30
31import * as DesktopObservability from "../app/DesktopObservability.ts";
32import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublisher.ts";
33import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts";
34import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts";
35
36const INITIAL_RESTART_DELAY = Duration.millis(500);
37const MAX_RESTART_DELAY = Duration.seconds(10);
38const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5;
39// Large state databases can take over a minute to become ready.
40const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(3);
41const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100);
42const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1);
43// Match the service launcher's grace period so backends can stop provider sessions.
44const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5);
45// Allow SIGKILL and the spawner exit wait after the SIGTERM grace so quit cannot
46// orphan a backend whose shutdown hangs.
47export const BACKEND_QUIT_STOP_TIMEOUT = Duration.sum(
48 DEFAULT_BACKEND_TERMINATE_GRACE,
49 Duration.seconds(2),
50);
51const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5);
52const BACKEND_READINESS_PATH = "/.well-known/t3/environment";
53// Three restarts without readiness is about 2 s of backoff: past a blip, early enough to help.
54const STARTUP_FAILURE_RESTART_ATTEMPTS = 3;
55const STARTUP_FAILURE_OUTPUT_CHARS = 4_000;
56const { logWarning: logBackendProcessWarning } =
57 DesktopObservability.makeComponentLogger("desktop-backend-process");
58
59type BackendProcessLayerServices = ChildProcessSpawner.ChildProcessSpawner | HttpClient.HttpClient;
60
61type BackendProcessRunRequirements = BackendProcessLayerServices | Scope.Scope;
62
63export type BackendProcessOutputStream = "stdout" | "stderr";
64
65export interface BackendProcessContext {
66 readonly executablePath: string;
67 readonly entryPath: string;
68 readonly cwd: string;
69 readonly httpBaseUrl: URL;
70}
71
72export type DesktopBackendBootstrapDelivery = "fd3" | "stdin";
73
74export interface DesktopBackendStartConfig extends BackendProcessContext {
75 readonly args: ReadonlyArray<string>;
76 readonly env: Record<string, string | undefined>;
77 /** False passes env verbatim, preventing the Windows T3CODE_HOME from leaking into WSL. */
78 readonly extendEnv: boolean;
79 readonly bootstrap: DesktopBackendBootstrapValue;
80 readonly bootstrapDelivery: DesktopBackendBootstrapDelivery;
81 readonly httpBaseUrl: URL;
82 readonly captureOutput: boolean;
83 readonly preflightFailure: Option.Option<PreflightFailure>;
84 readonly runningDistro?: string;
85 readonly wslRuntimeId?: string;
86}
87
88export interface PreflightFailure {
89 readonly reason: string;
90 readonly fatal: boolean;
91 readonly retryLimit?: number;
92}
93
94interface BackendProcessExit {
95 readonly code: Option.Option<number>;
96 readonly reason: string;
97}
98
99const backendProcessContextSchema = {
100 executablePath: Schema.String,
101 entryPath: Schema.String,
102 cwd: Schema.String,
103 httpBaseUrl: Schema.URL,
104};
105
106export class BackendReadinessTimeoutError extends Schema.TaggedError<BackendReadinessTimeoutError>()(
107 "BackendReadinessTimeoutError",
108 {
109 ...backendProcessContextSchema,
110 readinessUrl: Schema.URL,
111 timeoutMs: Schema.Number,
112 cause: Schema.Defect(),
113 },
114) {
115 override get message(): string {
116 return `Timed out after ${this.timeoutMs}ms waiting for desktop backend readiness at ${this.readinessUrl.href}.`;
117 }
118}
119
120export class BackendProcessBootstrapEncodeError extends Schema.TaggedError<BackendProcessBootstrapEncodeError>()(
121 "BackendProcessBootstrapEncodeError",
122 {
123 ...backendProcessContextSchema,
124 cause: Schema.Defect(),
125 },
126) {
127 override get message(): string {
128 return `Failed to encode the desktop backend bootstrap payload for ${this.entryPath}.`;
129 }
130}
131
132export class BackendProcessSpawnError extends Schema.TaggedError<BackendProcessSpawnError>()(
133 "BackendProcessSpawnError",
134 {
135 ...backendProcessContextSchema,
136 cause: Schema.Defect(),
137 },
138) {
139 override get message(): string {
140 return `Failed to spawn desktop backend entry ${this.entryPath} with ${this.executablePath}.`;
141 }
142}
143
144export class BackendProcessOutputReadError extends Schema.TaggedError<BackendProcessOutputReadError>()(
145 "BackendProcessOutputReadError",
146 {
147 ...backendProcessContextSchema,
148 pid: Schema.Number,
149 streamName: Schema.Literals(["stdout", "stderr"]),
150 cause: Schema.Defect(),
151 },
152) {
153 override get message(): string {
154 return `Failed to read ${this.streamName} from desktop backend process ${this.pid}.`;
155 }
156}
157
158export class BackendProcessOutputHandlingError extends Schema.TaggedError<BackendProcessOutputHandlingError>()(
159 "BackendProcessOutputHandlingError",
160 {
161 ...backendProcessContextSchema,
162 pid: Schema.Number,
163 streamName: Schema.Literals(["stdout", "stderr"]),
164 chunkByteLength: Schema.Number,
165 cause: Schema.Defect(),
166 },
167) {
168 override get message(): string {
169 return `Failed to handle ${this.chunkByteLength} bytes from ${this.streamName} of desktop backend process ${this.pid}.`;
170 }
171}
172
173export type BackendProcessOutputError =
174 | BackendProcessOutputReadError
175 | BackendProcessOutputHandlingError;
176
177export class BackendProcessExitStatusError extends Schema.TaggedError<BackendProcessExitStatusError>()(
178 "BackendProcessExitStatusError",
179 {
180 ...backendProcessContextSchema,
181 pid: Schema.Number,
182 cause: Schema.Defect(),
183 },
184) {
185 override get message(): string {
186 return `Failed to read the exit status of desktop backend process ${this.pid}.`;
187 }
188}
189
190export const BackendProcessError = Schema.Union([
191 BackendProcessBootstrapEncodeError,
192 BackendProcessSpawnError,
193 BackendProcessExitStatusError,
194]);
195export type BackendProcessError = typeof BackendProcessError.Type;
196
197interface RunBackendProcessOptions extends DesktopBackendStartConfig {
198 readonly desktopTelemetryStream: Stream.Stream<Uint8Array>;
199 readonly desktopBrowserStream?: Stream.Stream<Uint8Array>;
200 readonly onDesktopBrowserCommand?: (line: string) => Effect.Effect<void>;
201 readonly onDesktopTelemetryControl?: (
202 message: DesktopTelemetryControlMessageValue,
203 ) => Effect.Effect<void>;
204 readonly readinessTimeout?: Duration.Duration;
205 readonly outputDrainTimeout?: Duration.Duration;
206 readonly onStarted?: (pid: number) => Effect.Effect<void>;
207 readonly onExitObserved?: () => Effect.Effect<void>;
208 readonly onReady?: (pid: number) => Effect.Effect<void>;
209 readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect<void>;
210 readonly onOutput?: (
211 streamName: BackendProcessOutputStream,
212 chunk: Uint8Array,
213 ) => Effect.Effect<void, Error>;
214 readonly onOutputFailure?: (error: BackendProcessOutputError) => Effect.Effect<void>;
215}
216
217export interface DesktopBackendSnapshot {
218 readonly desiredRunning: boolean;
219 readonly ready: boolean;
220 readonly activePid: Option.Option<number>;
221 readonly restartAttempt: number;
222 readonly restartScheduled: boolean;
223}
224
225export type BackendInstanceId = string & Brand.Brand<"BackendInstanceId">;
226export const BackendInstanceId = Brand.nominal<BackendInstanceId>();
227
228export const PRIMARY_INSTANCE_ID: BackendInstanceId = BackendInstanceId(
229 PRIMARY_LOCAL_ENVIRONMENT_ID,
230);
231
232export interface BackendStartupFailure {
233 readonly reason: string;
234 /** The server refused to start under this machine's managed policy. */
235 readonly managedPolicy: boolean;
236 /** Tail of the last run's stdout and stderr. */
237 readonly output: string;
238}
239
240export interface DesktopBackendInstance {
241 readonly id: BackendInstanceId;
242 readonly label: Effect.Effect<string>;
243 readonly start: Effect.Effect<void>;
244 /** Timeout controls the warning; stop still waits for process teardown and exit. */
245 readonly stop: (options?: { readonly timeout?: Duration.Duration }) => Effect.Effect<void>;
246 readonly currentConfig: Effect.Effect<Option.Option<DesktopBackendStartConfig>>;
247 readonly snapshot: Effect.Effect<DesktopBackendSnapshot>;
248 readonly waitForReady: (timeout: Duration.Duration) => Effect.Effect<boolean>;
249}
250
251export interface BackendInstanceSpec {
252 readonly id: BackendInstanceId;
253 readonly label: Effect.Effect<string>;
254 readonly configResolve: Effect.Effect<DesktopBackendStartConfig, PlatformError.PlatformError>;
255 /** Receives the ready run's URL and pid; reading current config could race a subsequent restart. */
256 readonly onReady?: (httpBaseUrl: URL, pid: number) => Effect.Effect<void>;
257 readonly onShutdown?: () => Effect.Effect<void>;
258 readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect<boolean>;
259 readonly onStateDirOwned?: () => Effect.Effect<void>;
260 /** Fires on each repeated crash or readiness timeout until the backend becomes ready. */
261 readonly onStartupFailure?: (failure: BackendStartupFailure) => Effect.Effect<void>;
262}
263
264interface ActiveBackendRun {
265 readonly id: number;
266 readonly scope: Scope.Closeable;
267 readonly fiber: Option.Option<Fiber.Fiber<void, never>>;
268 readonly pid: Option.Option<number>;
269 readonly exitObserved: boolean;
270 readonly stopRequested: boolean;
271}
272
273interface BackendManagerState {
274 readonly desiredRunning: boolean;
275 readonly ready: boolean;
276 readonly config: Option.Option<DesktopBackendStartConfig>;
277 readonly active: Option.Option<ActiveBackendRun>;
278 readonly restartAttempt: number;
279 readonly preflightFailureAttempt: number;
280 readonly restartFiber: Option.Option<Fiber.Fiber<void, never>>;
281 readonly nextRunId: number;
282}
283
284const initialState: BackendManagerState = {
285 desiredRunning: false,
286 ready: false,
287 config: Option.none(),
288 active: Option.none(),
289 restartAttempt: 0,
290 preflightFailureAttempt: 0,
291 restartFiber: Option.none(),
292 nextRunId: 1,
293};
294
295const activePid = (active: Option.Option<ActiveBackendRun>): Option.Option<number> =>
296 Option.flatMap(active, (run) => run.pid);
297
298const withActiveRun =
299 (runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) =>
300 (state: BackendManagerState): BackendManagerState => ({
301 ...state,
302 active: Option.map(state.active, (run) => (run.id === runId ? f(run) : run)),
303 });
304
305const calculateRestartDelay = (attempt: number): Duration.Duration =>
306 Duration.min(Duration.times(INITIAL_RESTART_DELAY, 2 ** attempt), MAX_RESTART_DELAY);
307
308const closeRun = (
309 run: ActiveBackendRun,
310 parentScope: Scope.Scope,
311 options?: { readonly timeout?: Duration.Duration },
312) => {
313 const waitForFiber = Option.match(run.fiber, {
314 onNone: () => Effect.void,
315 onSome: (fiber) => Fiber.await(fiber).pipe(Effect.asVoid),
316 });
317 const close = Scope.close(run.scope, Exit.void).pipe(
318 Effect.withSpan("desktop.backendInstance.closeScope"),
319 Effect.andThen(waitForFiber.pipe(Effect.withSpan("desktop.backendInstance.awaitRun"))),
320 );
321 const timeout = options?.timeout;
322
323 if (!timeout) {
324 return close;
325 }
326
327 return Effect.forkIn(close, parentScope, { startImmediately: true }).pipe(
328 Effect.flatMap((closeFiber) =>
329 Fiber.await(closeFiber).pipe(
330 Effect.timeoutOption(timeout),
331 Effect.tap((completed) =>
332 Option.isNone(completed)
333 ? logBackendProcessWarning("backend shutdown exceeded timeout; waiting for exit", {
334 timeoutMs: Duration.toMillis(timeout),
335 })
336 : Effect.void,
337 ),
338 // Parent-scope interruption still finishes release; do not cancel sibling stops.
339 Effect.andThen(Fiber.await(closeFiber)),
340 Effect.asVoid,
341 ),
342 ),
343 );
344};
345
346const waitForHttpReady = (
347 options: BackendProcessContext & { readonly timeout: Duration.Duration },
348): Effect.Effect<void, BackendReadinessTimeoutError, HttpClient.HttpClient> => {
349 const readinessUrl = new URL(BACKEND_READINESS_PATH, options.httpBaseUrl);
350 return waitForHttpReadyShared({
351 baseUrl: options.httpBaseUrl.href,
352 path: BACKEND_READINESS_PATH,
353 timeoutMs: Duration.toMillis(options.timeout),
354 intervalMs: Duration.toMillis(DEFAULT_BACKEND_READINESS_INTERVAL),
355 probeTimeoutMs: Duration.toMillis(DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT),
356 makeError: ({ cause }) =>
357 new BackendReadinessTimeoutError({
358 executablePath: options.executablePath,
359 entryPath: options.entryPath,
360 cwd: options.cwd,
361 httpBaseUrl: options.httpBaseUrl,
362 readinessUrl,
363 timeoutMs: Duration.toMillis(options.timeout),
364 cause,
365 }),
366 });
367};
368
369function drainBackendOutput(
370 context: BackendProcessContext & { readonly pid: number },
371 streamName: BackendProcessOutputStream,
372 stream: Stream.Stream<Uint8Array, PlatformError.PlatformError>,
373 onOutput: (
374 streamName: BackendProcessOutputStream,
375 chunk: Uint8Array,
376 ) => Effect.Effect<void, Error>,
377 onOutputFailure: (error: BackendProcessOutputError) => Effect.Effect<void>,
378): Effect.Effect<void> {
379 return stream.pipe(
380 Stream.mapError(
381 (cause) =>
382 new BackendProcessOutputReadError({
383 ...context,
384 streamName,
385 cause,
386 }),
387 ),
388 Stream.runForEach((chunk) =>
389 onOutput(streamName, chunk).pipe(
390 Effect.mapError(
391 (cause) =>
392 new BackendProcessOutputHandlingError({
393 ...context,
394 streamName,
395 chunkByteLength: chunk.byteLength,
396 cause,
397 }),
398 ),
399 Effect.catchTags({ BackendProcessOutputHandlingError: onOutputFailure }),
400 ),
401 ),
402 Effect.catchTags({
403 BackendProcessOutputReadError: onOutputFailure,
404 }),
405 );
406}
407
408const encodeBootstrapJson = Schema.encodeEffect(Schema.fromJsonString(DesktopBackendBootstrap));
409const decodeDesktopTelemetryControlLine = Schema.decodeUnknownEffect(
410 Schema.fromJsonString(DesktopTelemetryControlMessage),
411);
412
413export const runBackendProcess = Effect.fn("runBackendProcess")(function* (
414 options: RunBackendProcessOptions,
415): Effect.fn.Return<BackendProcessExit, BackendProcessError, BackendProcessRunRequirements> {
416 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
417 const bootstrapJson = yield* encodeBootstrapJson(options.bootstrap).pipe(
418 Effect.mapError(
419 (cause) =>
420 new BackendProcessBootstrapEncodeError({
421 executablePath: options.executablePath,
422 entryPath: options.entryPath,
423 cwd: options.cwd,
424 httpBaseUrl: options.httpBaseUrl,
425 cause,
426 }),
427 ),
428 );
429 const onOutput = options.onOutput ?? (() => Effect.void);
430 const bootstrapStream = Stream.encodeText(Stream.make(`${bootstrapJson}\n`));
431 const additionalFds: Record<`fd${number}`, ChildProcess.AdditionalFdConfig> = {};
432 if (options.bootstrapDelivery === "fd3") {
433 additionalFds.fd3 = {
434 type: "input",
435 stream: bootstrapStream,
436 };
437 if (options.bootstrap.desktopTelemetryFd !== undefined) {
438 additionalFds[`fd${options.bootstrap.desktopTelemetryFd}`] = {
439 type: "input",
440 stream: options.desktopTelemetryStream,
441 };
442 }
443 if (options.bootstrap.desktopTelemetryControlFd !== undefined) {
444 additionalFds[`fd${options.bootstrap.desktopTelemetryControlFd}`] = {
445 type: "output",
446 };
447 }
448 if (
449 options.bootstrap.desktopBrowserFd !== undefined &&
450 options.desktopBrowserStream !== undefined
451 ) {
452 additionalFds[`fd${options.bootstrap.desktopBrowserFd}`] = {
453 type: "input",
454 stream: options.desktopBrowserStream,
455 };
456 }
457 if (options.bootstrap.desktopBrowserControlFd !== undefined) {
458 additionalFds[`fd${options.bootstrap.desktopBrowserControlFd}`] = { type: "output" };
459 }
460 }
461 const command = ChildProcess.make(options.executablePath, options.args, {
462 cwd: options.cwd,
463 env: options.env,
464 extendEnv: options.extendEnv,
465 stdin: options.bootstrapDelivery === "stdin" ? bootstrapStream : "ignore",
466 stdout: options.captureOutput ? "pipe" : "inherit",
467 stderr: options.captureOutput ? "pipe" : "inherit",
468 killSignal: "SIGTERM",
469 forceKillAfter: DEFAULT_BACKEND_TERMINATE_GRACE,
470 // wsl.exe drops extra file descriptors; deliver its bootstrap via stdin.
471 ...(options.bootstrapDelivery === "fd3" ? { additionalFds } : {}),
472 });
473
474 const handle = yield* spawner.spawn(command).pipe(
475 Effect.mapError(
476 (cause) =>
477 new BackendProcessSpawnError({
478 executablePath: options.executablePath,
479 entryPath: options.entryPath,
480 cwd: options.cwd,
481 httpBaseUrl: options.httpBaseUrl,
482 cause,
483 }),
484 ),
485 );
486 const outputFibers: Array<Fiber.Fiber<void, never>> = [];
487
488 yield* options.onStarted?.(handle.pid) ?? Effect.void;
489 if (
490 options.bootstrap.desktopTelemetryControlFd !== undefined &&
491 options.onDesktopTelemetryControl !== undefined
492 ) {
493 const controlFd = options.bootstrap.desktopTelemetryControlFd;
494 const handleControl = options.onDesktopTelemetryControl;
495 yield* handle.getOutputFd(controlFd).pipe(
496 Stream.decodeText(),
497 Stream.splitLines,
498 Stream.filter((line) => line.trim().length > 0),
499 Stream.runForEach((line) =>
500 decodeDesktopTelemetryControlLine(line).pipe(
501 Effect.flatMap(handleControl),
502 Effect.catchCause((cause) =>
503 logBackendProcessWarning("ignored invalid desktop telemetry control message", {
504 fd: controlFd,
505 cause: Cause.pretty(cause),
506 }),
507 ),
508 ),
509 ),
510 Effect.catchCause((cause) =>
511 logBackendProcessWarning("desktop telemetry control stream stopped", {
512 fd: controlFd,
513 cause: Cause.pretty(cause),
514 }),
515 ),
516 Effect.ensuring(
517 handleControl({
518 version: 1,
519 type: "setDiagnosticsDemand",
520 enabled: false,
521 }),
522 ),
523 Effect.forkScoped,
524 );
525 }
526 if (
527 options.bootstrap.desktopBrowserControlFd !== undefined &&
528 options.onDesktopBrowserCommand !== undefined
529 ) {
530 const browserFd = options.bootstrap.desktopBrowserControlFd;
531 const handleCommand = options.onDesktopBrowserCommand;
532 yield* handle.getOutputFd(browserFd).pipe(
533 Stream.decodeText(),
534 Stream.splitLines,
535 Stream.filter((line) => line.length > 0),
536 Stream.runForEach(handleCommand),
537 Effect.catchCause((cause) =>
538 logBackendProcessWarning("desktop browser command stream stopped", {
539 fd: browserFd,
540 cause: Cause.pretty(cause),
541 }),
542 ),
543 Effect.forkScoped,
544 );
545 }
546 if (options.captureOutput) {
547 const outputContext = {
548 executablePath: options.executablePath,
549 entryPath: options.entryPath,
550 cwd: options.cwd,
551 httpBaseUrl: options.httpBaseUrl,
552 pid: Number(handle.pid),
553 };
554 const onOutputFailure = options.onOutputFailure ?? (() => Effect.void);
555 outputFibers.push(
556 yield* drainBackendOutput(
557 outputContext,
558 "stdout",
559 handle.stdout,
560 onOutput,
561 onOutputFailure,
562 ).pipe(Effect.forkScoped),
563 yield* drainBackendOutput(
564 outputContext,
565 "stderr",
566 handle.stderr,
567 onOutput,
568 onOutputFailure,
569 ).pipe(Effect.forkScoped),
570 );
571 }
572 // Cold WSL boots across /mnt/c can exceed one probe budget; retry while the backend is alive.
573 const probeReadiness = Effect.fn("desktop.backendProcess.probeReadiness")(() =>
574 waitForHttpReady({
575 executablePath: options.executablePath,
576 entryPath: options.entryPath,
577 cwd: options.cwd,
578 httpBaseUrl: options.httpBaseUrl,
579 timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT,
580 }).pipe(
581 Effect.flatMap(() => options.onReady?.(handle.pid) ?? Effect.void),
582 Effect.as(true),
583 Effect.catchTags({
584 BackendReadinessTimeoutError: (error) =>
585 (options.onReadinessFailure?.(error) ?? Effect.void).pipe(Effect.as(false)),
586 }),
587 ),
588 );
589
590 yield* probeReadiness().pipe(Effect.repeat({ while: (ready) => !ready }), Effect.forkScoped);
591
592 const exit = yield* handle.exitCode.pipe(
593 Effect.withSpan("desktop.backendProcess.awaitExit"),
594 Effect.map((exitCode) => ({
595 code: Option.some(exitCode),
596 reason: `code=${exitCode}`,
597 })),
598 Effect.catch((cause) => {
599 // Effect's Node spawner exposes signal exits only through this cause message.
600 const signal =
601 cause.cause instanceof Error
602 ? /^Process interrupted due to receipt of signal: '(SIG[A-Z0-9]+)'$/.exec(
603 cause.cause.message,
604 )?.[1]
605 : undefined;
606 if (signal !== undefined) {
607 return Effect.succeed({ code: Option.none(), reason: `killed by ${signal}` });
608 }
609 return Effect.fail(
610 new BackendProcessExitStatusError({
611 executablePath: options.executablePath,
612 entryPath: options.entryPath,
613 cwd: options.cwd,
614 httpBaseUrl: options.httpBaseUrl,
615 pid: Number(handle.pid),
616 cause,
617 }),
618 );
619 }),
620 Effect.exit,
621 );
622 yield* options.onExitObserved?.() ?? Effect.void;
623 yield* Effect.forEach(outputFibers, Fiber.await, {
624 concurrency: "unbounded",
625 discard: true,
626 }).pipe(
627 Effect.timeout(options.outputDrainTimeout ?? DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT),
628 Effect.ignore,
629 Effect.withSpan("desktop.backendProcess.drainOutput"),
630 );
631 if (Exit.isFailure(exit)) {
632 return yield* Effect.failCause(exit.cause);
633 }
634 return exit.value;
635});
636
637export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
638 spec: BackendInstanceSpec,
639): Effect.fn.Return<
640 DesktopBackendInstance,
641 never,
642 | FileSystem.FileSystem
643 | ChildProcessSpawner.ChildProcessSpawner
644 | HttpClient.HttpClient
645 | DesktopObservability.DesktopBackendOutputLogFactory
646 | DesktopTelemetryPublisher.DesktopTelemetryPublisher
647 | DesktopBrowserHost.DesktopBrowserHost
648 | DesktopWslEnvironment.DesktopWslEnvironment
649 | Scope.Scope
650> {
651 const parentScope = yield* Scope.Scope;
652 const fileSystem = yield* FileSystem.FileSystem;
653 const backendOutputLogFactory = yield* DesktopObservability.DesktopBackendOutputLogFactory;
654 const backendOutputLog = yield* backendOutputLogFactory.forInstance(spec.id);
655 const desktopTelemetryPublisher = yield* DesktopTelemetryPublisher.DesktopTelemetryPublisher;
656 const desktopBrowserHost = yield* DesktopBrowserHost.DesktopBrowserHost;
657 const wslEnvironment = yield* DesktopWslEnvironment.DesktopWslEnvironment;
658 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
659 const httpClient = yield* HttpClient.HttpClient;
660 const state = yield* SubscriptionRef.make(initialState);
661 const mutex = yield* Semaphore.make(1);
662 const outputTail = yield* Ref.make("");
663 // One streaming decoder per stream keeps UTF-8 characters split across chunks intact.
664 const makeOutputDecoders = () => ({ stdout: new TextDecoder(), stderr: new TextDecoder() });
665 let outputDecoders = makeOutputDecoders();
666 const reportStartupFailure = (reason: string, managedPolicy = false) =>
667 Ref.get(outputTail).pipe(
668 Effect.flatMap(
669 (output) => spec.onStartupFailure?.({ reason, managedPolicy, output }) ?? Effect.void,
670 ),
671 );
672
673 const { logWarning: logInstanceWarning, logError: logInstanceError } =
674 DesktopObservability.makeComponentLogger(`desktop-backend-instance:${spec.id}`);
675
676 const updateActiveRun = (runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) =>
677 SubscriptionRef.update(state, withActiveRun(runId, f));
678
679 const snapshot = SubscriptionRef.get(state).pipe(
680 Effect.map((current): DesktopBackendSnapshot => ({
681 desiredRunning: current.desiredRunning,
682 ready: current.ready,
683 activePid: activePid(current.active),
684 restartAttempt: current.restartAttempt,
685 restartScheduled: Option.isSome(current.restartFiber),
686 })),
687 );
688 const currentConfig = SubscriptionRef.get(state).pipe(Effect.map((current) => current.config));
689
690 const cancelRestart = Effect.gen(function* () {
691 const restartFiber = yield* SubscriptionRef.modify(state, (current) => [
692 current.restartFiber,
693 {
694 ...current,
695 restartFiber: Option.none(),
696 },
697 ]);
698
699 yield* Option.match(restartFiber, {
700 onNone: () => Effect.void,
701 onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid),
702 });
703 });
704
705 const start: Effect.Effect<void> = Effect.suspend(() =>
706 mutex.withPermits(1)(
707 Effect.gen(function* () {
708 const current = yield* SubscriptionRef.get(state);
709 if (Option.isSome(current.active)) {
710 if (!current.desiredRunning) {
711 yield* SubscriptionRef.update(state, (latest) => ({
712 ...latest,
713 desiredRunning: true,
714 }));
715 }
716 return;
717 }
718
719 if (current.ready) {
720 yield* spec.onShutdown?.() ?? Effect.void;
721 yield* SubscriptionRef.update(state, (latest) =>
722 latest.ready ? { ...latest, ready: false } : latest,
723 );
724 }
725 const config = yield* spec.configResolve.pipe(
726 Effect.tapError((error) =>
727 logInstanceError("failed to generate desktop backend configuration", {
728 cause: error.message,
729 }),
730 ),
731 Effect.option,
732 );
733 if (Option.isNone(config)) {
734 if (current.desiredRunning) {
735 yield* scheduleRestart("failed to generate desktop backend configuration");
736 }
737 return;
738 }
739 const entryExists = yield* fileSystem
740 .exists(config.value.entryPath)
741 .pipe(Effect.orElseSucceed(() => false));
742
743 const resetFatalPreflightCounter =
744 !current.desiredRunning && current.preflightFailureAttempt > 0;
745 yield* cancelRestart;
746 yield* SubscriptionRef.update(state, (latest) => ({
747 ...latest,
748 desiredRunning: true,
749 ready: false,
750 config: Option.some(config.value),
751 preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt,
752 }));
753
754 const preflightFailure = config.value.preflightFailure;
755 if (Option.isSome(preflightFailure)) {
756 const { reason, fatal, retryLimit } = preflightFailure.value;
757 if (!fatal && retryLimit === undefined) {
758 yield* SubscriptionRef.update(state, (latest) =>
759 latest.preflightFailureAttempt === 0
760 ? latest
761 : { ...latest, preflightFailureAttempt: 0 },
762 );
763 yield* scheduleRestart(reason);
764 return;
765 }
766 const attemptLimit = retryLimit ?? MAX_PREFLIGHT_FAILURE_ATTEMPTS;
767 const attempt = yield* SubscriptionRef.modify(state, (latest) => {
768 const next = latest.preflightFailureAttempt + 1;
769 return [next, { ...latest, preflightFailureAttempt: next }] as const;
770 });
771 if (attempt > attemptLimit) {
772 yield* logInstanceError("backend preflight still failing after fallback; stopping", {
773 reason,
774 attempt,
775 });
776 yield* SubscriptionRef.update(state, (latest) => ({
777 ...latest,
778 desiredRunning: false,
779 ready: false,
780 }));
781 return;
782 }
783 if (attempt === attemptLimit) {
784 yield* logInstanceError(
785 "backend preflight failed repeatedly; surfacing and falling back",
786 { reason, attempt },
787 );
788 const shouldRestart = yield* (
789 spec.onPreflightFailed?.(preflightFailure.value) ?? Effect.succeed(false)
790 );
791 if (shouldRestart) {
792 yield* scheduleRestart(reason);
793 } else {
794 yield* SubscriptionRef.update(state, (latest) => ({
795 ...latest,
796 desiredRunning: false,
797 ready: false,
798 }));
799 }
800 return;
801 }
802 yield* scheduleRestart(reason);
803 return;
804 }
805 yield* SubscriptionRef.update(state, (latest) =>
806 latest.preflightFailureAttempt === 0 ? latest : { ...latest, preflightFailureAttempt: 0 },
807 );
808
809 if (!entryExists) {
810 yield* scheduleRestart(`missing server entry at ${config.value.entryPath}`);
811 return;
812 }
813
814 const runScope = yield* Scope.make("sequential");
815 const runId = yield* SubscriptionRef.modify(state, (latest) => [
816 latest.nextRunId,
817 {
818 ...latest,
819 active: Option.some({
820 id: latest.nextRunId,
821 scope: runScope,
822 fiber: Option.none(),
823 pid: Option.none(),
824 exitObserved: false,
825 stopRequested: false,
826 } satisfies ActiveBackendRun),
827 nextRunId: latest.nextRunId + 1,
828 },
829 ]);
830
831 const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* (
832 reason: string,
833 exitCode?: number,
834 ) {
835 const stateDirOwned = exitCode === SERVER_EXIT_CODE_STATE_DIR_OWNED;
836 yield* mutex.withPermits(1)(
837 Effect.gen(function* () {
838 const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } =
839 yield* SubscriptionRef.modify(
840 state,
841 (
842 latest,
843 ): readonly [
844 {
845 readonly isCurrentRun: boolean;
846 readonly nextState: BackendManagerState;
847 readonly pid: Option.Option<number>;
848 readonly exitObserved: boolean;
849 readonly stopRequested: boolean;
850 readonly wasReady: boolean;
851 },
852 BackendManagerState,
853 ] => {
854 const currentRun = Option.getOrUndefined(latest.active);
855 if (currentRun?.id !== runId) {
856 return [
857 {
858 isCurrentRun: false,
859 nextState: latest,
860 pid: Option.none<number>(),
861 exitObserved: false,
862 stopRequested: false,
863 wasReady: false,
864 },
865 latest,
866 ] as const;
867 }
868
869 const next = {
870 ...latest,
871 active: Option.none<ActiveBackendRun>(),
872 ready: false,
873 };
874 return [
875 {
876 isCurrentRun: true,
877 nextState: next,
878 pid: currentRun.pid,
879 exitObserved: currentRun.exitObserved,
880 stopRequested: currentRun.stopRequested,
881 wasReady: latest.ready,
882 },
883 next,
884 ] as const;
885 },
886 );
887
888 if (isCurrentRun) {
889 yield* desktopTelemetryPublisher.removeControlSource(spec.id);
890 if (Option.isSome(pid)) {
891 if (exitObserved && !stopRequested) {
892 yield* backendOutputLog.persistFailure({
893 details: `pid=${pid.value} ${reason}`,
894 });
895 } else {
896 yield* backendOutputLog.discardSession;
897 }
898 }
899 if (wasReady) {
900 yield* spec.onShutdown?.() ?? Effect.void;
901 }
902 }
903
904 if (isCurrentRun && nextState.desiredRunning) {
905 if (stateDirOwned && !stopRequested) {
906 yield* SubscriptionRef.update(state, (latest) => ({
907 ...latest,
908 desiredRunning: false,
909 }));
910 yield* (spec.onStateDirOwned?.() ?? Effect.void).pipe(Effect.ignore);
911 return;
912 }
913 yield* scheduleRestart(reason, exitCode);
914 }
915 }),
916 );
917 });
918
919 const program = runBackendProcess({
920 ...config.value,
921 desktopTelemetryStream: desktopTelemetryPublisher.encoded,
922 desktopBrowserStream: desktopBrowserHost.events,
923 onDesktopBrowserCommand: desktopBrowserHost.handleCommandLine,
924 onDesktopTelemetryControl: (message) =>
925 desktopTelemetryPublisher.handleControlForSource(spec.id, message),
926 onStarted: Effect.fn("desktop.backendInstance.onStarted")(function* (pid) {
927 yield* updateActiveRun(runId, (run) => ({
928 ...run,
929 pid: Option.some(pid),
930 }));
931 yield* Ref.set(outputTail, "");
932 outputDecoders = makeOutputDecoders();
933 yield* backendOutputLog.beginSession({
934 details: `pid=${pid} port=${config.value.bootstrap.port} cwd=${config.value.cwd}`,
935 });
936 }),
937 onExitObserved: () =>
938 updateActiveRun(runId, (run) => ({
939 ...run,
940 exitObserved: true,
941 })),
942 onReady: Effect.fn("desktop.backendInstance.onReady")(function* (pid) {
943 const isCurrentRun = yield* SubscriptionRef.modify(state, (latest) => {
944 const activeRun = Option.getOrUndefined(latest.active);
945 if (activeRun?.id !== runId) {
946 return [false, latest] as const;
947 }
948
949 return [
950 true,
951 {
952 ...latest,
953 restartAttempt: 0,
954 ready: true,
955 },
956 ] as const;
957 });
958 if (!isCurrentRun) {
959 return;
960 }
961
962 yield* spec.onReady?.(config.value.httpBaseUrl, pid) ?? Effect.void;
963 if (
964 config.value.runningDistro !== undefined &&
965 config.value.wslRuntimeId !== undefined
966 ) {
967 yield* wslEnvironment.pruneRuntimes(
968 config.value.runningDistro,
969 config.value.wslRuntimeId,
970 );
971 }
972 }),
973 onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")(
974 function* (error) {
975 if (isRunningDatabaseMigrations(yield* Ref.get(outputTail))) {
976 yield* logInstanceWarning("backend is still migrating its database; waiting", {
977 error: error.message,
978 });
979 return;
980 }
981 yield* logInstanceWarning("backend readiness check failed during bootstrap", {
982 error: error.message,
983 });
984 yield* backendOutputLog.persistFailureSnapshot({
985 details: error.message,
986 });
987 yield* reportStartupFailure("The server started but did not become ready in time.");
988 },
989 ),
990 onOutput: (streamName, chunk) =>
991 Ref.update(outputTail, (tail) =>
992 // A cut through a surrogate pair would make encodeURIComponent throw when rendering.
993 (tail + outputDecoders[streamName].decode(chunk, { stream: true }))
994 .slice(-STARTUP_FAILURE_OUTPUT_CHARS)
995 .toWellFormed(),
996 ).pipe(Effect.andThen(backendOutputLog.writeOutputChunk(streamName, chunk))),
997 }).pipe(
998 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
999 Effect.provideService(HttpClient.HttpClient, httpClient),
1000 Scope.provide(runScope),
1001 Effect.matchEffect({
1002 onFailure: (error) => finalizeRun(error.message),
1003 onSuccess: (exit) => finalizeRun(exit.reason, Option.getOrUndefined(exit.code)),
1004 }),
1005 Effect.ensuring(Scope.close(runScope, Exit.void).pipe(Effect.ignore)),
1006 );
1007
1008 const fiber = yield* Effect.forkIn(program, parentScope);
1009 yield* updateActiveRun(runId, (run) => ({
1010 ...run,
1011 fiber: Option.some(fiber),
1012 }));
1013 }),
1014 ),
1015 ).pipe(Effect.withSpan("desktop.backendInstance.start", { attributes: { id: spec.id } }));
1016
1017 const scheduleRestart = Effect.fn("desktop.backendInstance.scheduleRestart")(function* (
1018 reason: string,
1019 exitCode?: number,
1020 ) {
1021 const scheduled = yield* SubscriptionRef.modify(state, (latest) => {
1022 if (!latest.desiredRunning || Option.isSome(latest.restartFiber)) {
1023 return [Option.none<{ delay: Duration.Duration; attempt: number }>(), latest] as const;
1024 }
1025
1026 const attempt = latest.restartAttempt + 1;
1027 return [
1028 Option.some({ delay: calculateRestartDelay(latest.restartAttempt), attempt }),
1029 { ...latest, restartAttempt: attempt },
1030 ] as const;
1031 });
1032
1033 yield* Option.match(scheduled, {
1034 onNone: () => Effect.void,
1035 onSome: Effect.fn("desktop.backendInstance.scheduleRestartFiber")(function* ({
1036 delay,
1037 attempt,
1038 }) {
1039 yield* logInstanceError("backend exited unexpectedly; restart scheduled", {
1040 reason,
1041 delayMs: Duration.toMillis(delay),
1042 });
1043 // Retrying cannot fix a policy refusal, so say so at once; retries still pick up a fix.
1044 const managedPolicy = exitCode === SERVER_EXIT_CODE_MANAGED_POLICY;
1045 if (managedPolicy || attempt >= STARTUP_FAILURE_RESTART_ATTEMPTS) {
1046 yield* reportStartupFailure(`The server could not start (${reason}).`, managedPolicy);
1047 }
1048 const restartFiber = yield* Effect.forkIn(
1049 Effect.sleep(delay).pipe(
1050 Effect.andThen(
1051 SubscriptionRef.modify(state, (latest) => {
1052 const shouldRestart = latest.desiredRunning;
1053 return [
1054 shouldRestart,
1055 {
1056 ...latest,
1057 restartFiber: Option.none(),
1058 },
1059 ] as const;
1060 }),
1061 ),
1062 Effect.flatMap((shouldRestart) => (shouldRestart ? start : Effect.void)),
1063 Effect.catchCause((cause) =>
1064 logInstanceError("desktop backend restart fiber failed", {
1065 cause: Cause.pretty(cause),
1066 }),
1067 ),
1068 ),
1069 parentScope,
1070 );
1071 yield* SubscriptionRef.update(state, (latest) =>
1072 Option.isNone(latest.restartFiber)
1073 ? {
1074 ...latest,
1075 restartFiber: Option.some(restartFiber),
1076 }
1077 : latest,
1078 );
1079 }),
1080 });
1081 });
1082
1083 const stop = Effect.fn("desktop.backendInstance.stop")(function* (options?: {
1084 readonly timeout?: Duration.Duration;
1085 }) {
1086 const { active, restartFiber, notifyShutdown } = yield* mutex.withPermits(1)(
1087 Effect.gen(function* () {
1088 const result = yield* SubscriptionRef.modify(state, (latest) => {
1089 const active = Option.map(latest.active, (run) =>
1090 run.exitObserved ? run : { ...run, stopRequested: true },
1091 );
1092 return [
1093 {
1094 active,
1095 restartFiber: latest.restartFiber,
1096 notifyShutdown: latest.ready,
1097 },
1098 {
1099 ...latest,
1100 desiredRunning: false,
1101 ready: false,
1102 active,
1103 restartFiber: Option.none<Fiber.Fiber<void, never>>(),
1104 },
1105 ] as const;
1106 });
1107 return result;
1108 }),
1109 );
1110
1111 if (notifyShutdown) {
1112 yield* (spec.onShutdown?.() ?? Effect.void).pipe(Effect.ignore);
1113 }
1114 yield* Option.match(restartFiber, {
1115 onNone: () => Effect.void,
1116 onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid),
1117 });
1118 yield* Option.match(active, {
1119 onNone: () => Effect.void,
1120 // The enclosing operation owns the span for this inline handler.
1121 onSome: Effect.fnUntraced(function* (run) {
1122 yield* closeRun(run, parentScope, options);
1123 const cleanup = yield* mutex.withPermits(1)(
1124 SubscriptionRef.modify(
1125 state,
1126 (
1127 latest,
1128 ): readonly [
1129 {
1130 readonly needsCleanup: boolean;
1131 readonly shouldStart: boolean;
1132 },
1133 BackendManagerState,
1134 ] => {
1135 const current = Option.getOrUndefined(latest.active);
1136 if (current?.id !== run.id) {
1137 return [
1138 {
1139 needsCleanup: false,
1140 shouldStart:
1141 latest.desiredRunning &&
1142 Option.isNone(latest.active) &&
1143 Option.isNone(latest.restartFiber),
1144 },
1145 latest,
1146 ];
1147 }
1148 return [
1149 {
1150 needsCleanup: true,
1151 shouldStart: latest.desiredRunning,
1152 },
1153 {
1154 ...latest,
1155 active: Option.none<ActiveBackendRun>(),
1156 },
1157 ];
1158 },
1159 ),
1160 );
1161 if (cleanup.needsCleanup) {
1162 yield* desktopTelemetryPublisher.removeControlSource(spec.id);
1163 yield* backendOutputLog.discardSession;
1164 }
1165 if (cleanup.shouldStart) {
1166 yield* start;
1167 }
1168 }),
1169 });
1170 });
1171
1172 const waitForReady = Effect.fn("DesktopBackendManager.waitForReady")(
1173 (timeout: Duration.Duration) =>
1174 SubscriptionRef.changes(state).pipe(
1175 Stream.filter((current) => current.ready || !current.desiredRunning),
1176 Stream.map((current) => current.ready),
1177 Stream.runHead,
1178 Effect.timeoutOption(timeout),
1179 Effect.map(
1180 (result) => Option.isSome(result) && Option.getOrElse(result.value, () => false),
1181 ),
1182 ),
1183 );
1184
1185 yield* Effect.addFinalizer(() => stop());
1186
1187 return {
1188 id: spec.id,
1189 label: spec.label,
1190 start,
1191 stop,
1192 currentConfig,
1193 snapshot,
1194 waitForReady,
1195 } satisfies DesktopBackendInstance;
1196});