Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

agentScope.test.ts

417 lines16.5 KB

1// @effect-diagnostics nodeBuiltinImport:off -- fake systemd binaries live in a temp dir.
2import * as NodeFS from "node:fs";
3import * as NodeOS from "node:os";
4import * as NodePath from "node:path";
5​
6import * as NodeServices from "@effect/platform-node/NodeServices";
7import { assert, describe, expect, it } from "@effect/vitest";
8import { AgentScope } from "@t3tools/shared/AgentScope";
9import * as HostProcess from "@t3tools/shared/HostProcess";
10import * as Effect from "effect/Effect";
11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";
13import * as Logger from "effect/Logger";
14import { ChildProcess, ChildProcessSpawner } from "effect/process";
15​
16import * as ServerConfig from "../config.ts";
17import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts";
18import * as ProcessRunner from "../processRunner.ts";
19import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";
20​
21const running: ScopeState = {
22 loadState: "loaded",
23 activeState: "active",
24 result: "success",
25 oomKills: 0,
26 populated: true,
27};
28​
29describe("classifyScope", () => {
30 it("reports an OOM kill only once systemd records it", () => {
31 expect(classifyScope({ ...running, activeState: "failed", result: "oom-kill" })).toBe(
32 "oom-killed",
33 );
34 expect(classifyScope({ ...running, oomKills: 1 })).toBe("stopping");
35 expect(classifyScope({ ...running, populated: false })).toBe("stopping");
36 expect(classifyScope({ ...running, activeState: "deactivating" })).toBe("stopping");
37 });
38});
39​
40interface FakeUnit {
41 activeState: string;
42 result: string;
43 controlGroup?: string;
45​
46const withFakeSystemd = <A, E>(
47 body: (input: {
48 readonly scope: Effect.Success<typeof make>;
49 readonly units: Map<string, FakeUnit>;
50 readonly calls: ReadonlyArray<ReadonlyArray<string>>;
51 readonly stateDir: string;
52 readonly cgroupRoot: string;
53 }) => Effect.Effect<A, E>,
54 // A user manager older than systemd 253 rejects OOMPolicy on scopes.
55 {
56 oomPolicy = true,
57 stopFails = false,
58 }: { readonly oomPolicy?: boolean; readonly stopFails?: boolean } = {},
59) =>
60 Effect.gen(function* () {
61 const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
62 for (const name of ["systemd-run", "systemctl", "agent"]) {
63 NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
64 }
65 const cgroupRoot = NodePath.join(bin, "cgroup");
66 const units = new Map<string, FakeUnit>();
67 const calls: Array<ReadonlyArray<string>> = [];
68 const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({
69 stdout,
70 stderr: "",
71 code: ChildProcessSpawner.ExitCode(code),
72 timedOut: false,
73 stdoutTruncated: false,
74 stderrTruncated: false,
75 stdoutInvalidUtf8: false,
76 stderrInvalidUtf8: false,
77 });
78 const runner = ProcessRunner.ProcessRunner.of({
79 run: ({ args }) =>
80 Effect.sync(() => {
81 calls.push(args);
82 if (!oomPolicy && args.some((arg) => arg.startsWith("--property=OOMPolicy="))) {
83 return output("", 1);
84 }
85 const [, verb, unit = ""] = args;
86 if (stopFails && verb === "stop") {
87 return {
88 ...output(""),
89 stderr: `Failed to stop ${args.at(-1)}: Unit not loaded.`,
90 code: ChildProcessSpawner.ExitCode(5),
91 };
92 }
93 if (verb === "reset-failed") units.delete(unit);
94 if (verb !== "show") return output("");
95 const state = units.get(unit);
96 return output(
97 state === undefined
98 ? "LoadState=not-found\nActiveState=inactive\nResult=success\n"
99 : `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n` +
100 `ControlGroup=${state.controlGroup ?? ""}\n`,
101 );
102 }),
103 });
104 const fileSystem = yield* FileSystem.FileSystem;
105 const scope = yield* make.pipe(
106 Effect.provideService(ProcessRunner.ProcessRunner, runner),
107 Effect.provideService(HostProcess.Platform, "linux"),
108 Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
109 Effect.provideService(FileSystem.FileSystem, {
110 ...fileSystem,
111 readFileString: (path, encoding) =>
112 fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
113 writeFileString: (path, data, options) =>
114 fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
115 }),
116 );
117 const { stateDir } = yield* ServerConfig.ServerConfig;
118 return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe(
119 Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
120 );
121 }).pipe(
122 Effect.scoped,
123 Effect.provide(
124 ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-scope-" }).pipe(
125 Layer.provideMerge(NodeServices.layer),
126 ),
127 ),
128 );
129​
130const unitOf = (launch: Pick<ChildProcess.StandardCommand, "args">) =>
131 launch.args
132 .map(String)
133 .find((arg) => arg.startsWith("--unit="))
134 ?.slice("--unit=".length) ?? "";
135​
136const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
137 launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
138​
139/** An active scope as systemd and the kernel report it, with the agent itself gone. */
140const exitedAgent = (
141 units: Map<string, FakeUnit>,
142 cgroupRoot: string,
143 unit: string,
144 { populated, oomKills }: { readonly populated: boolean; readonly oomKills: number | undefined },
145) => {
146 const controlGroup = `/${unit}`;
147 NodeFS.mkdirSync(NodePath.join(cgroupRoot, controlGroup), { recursive: true });
148 NodeFS.writeFileSync(
149 NodePath.join(cgroupRoot, controlGroup, "cgroup.events"),
150 `populated ${populated ? 1 : 0}\nfrozen 0\n`,
151 );
152 // An undefined count leaves memory.events unreadable.
153 if (oomKills !== undefined) {
154 NodeFS.writeFileSync(
155 NodePath.join(cgroupRoot, controlGroup, "memory.events"),
156 `low 0\nhigh 0\nmax 0\noom ${oomKills}\noom_kill ${oomKills}\noom_group_kill 0\n`,
157 );
158 }
159 units.set(unit, { activeState: "active", result: "success", controlGroup });
160};
161​
162const stopsOf = (calls: ReadonlyArray<ReadonlyArray<string>>) =>
163 calls.filter(([, verb]) => verb === "stop");
164const launchCodex = (scope: Effect.Success<typeof make>, threadId: string) =>
165 makeCodexAppServerSpawnCommand({ command: "agent", args: ["app-server"], threadId }).pipe(
166 Effect.provideService(AgentScope, scope),
167 Effect.provideService(HostProcess.Platform, "linux"),
168 Effect.map((command) => {
169 assert(ChildProcess.isStandardCommand(command));
170 return command;
171 }),
172 );
173​
174describe("AgentScope service", () => {
175 it.effect("labels a thread from its newest scope until its next session clears it", () =>
176 withFakeSystemd(({ scope, units }) =>
177 Effect.gen(function* () {
178 const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
179 units.set(unitOf(first), { activeState: "failed", result: "oom-kill" });
180 const second = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
181 units.set(unitOf(second), { activeState: "active", result: "success" });
182 expect(yield* scope.oomKilled("a")).toBe(false);
183​
184 units.set(unitOf(second), { activeState: "failed", result: "oom-kill" });
185 expect(yield* scope.oomKilled("a")).toBe(true);
186 units.delete(unitOf(second));
187 expect(yield* scope.oomKilled("a")).toBe(true);
188​
189 yield* scope.clear("a");
190 expect(yield* scope.oomKilled("a")).toBe(false);
191 }),
192 ),
193 );
194​
195 it.effect(
196 "keeps shared Codex scopes running without attributing their OOM to the launch thread",
197 () =>
198 withFakeSystemd(({ scope, units, calls }) =>
199 Effect.gen(function* () {
200 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
201 expect(agent.args).toContain("--property=OOMPolicy=stop");
202 units.set(unitOf(agent), { activeState: "failed", result: "oom-kill" });
203 expect(yield* scope.oomKilled("a")).toBe(true);
204​
205 const codex = yield* launchCodex(scope, "a");
206 expect(codex.args).toContain("--property=OOMPolicy=continue");
207 expect(codex.args).not.toContain("--property=OOMPolicy=stop");
208 expect(codex.args).toContain("--property=TimeoutStopSec=10s");
209 expect(codex.args).toContain("--collect");
210 units.set(unitOf(codex), { activeState: "active", result: "success" });
211 expect(yield* scope.oomKilled("a")).toBe(false);
212 units.set(unitOf(codex), { activeState: "failed", result: "oom-kill" });
213 expect(yield* scope.oomKilled("a")).toBe(false);
214 expect(
215 calls.filter(([, verb, unit]) => verb === "show" && unit === unitOf(codex)),
216 ).toEqual([]);
217 }),
218 ),
219 );
220​
221 it.effect.each([{ oomPolicy: true }, { oomPolicy: false }])(
222 "Force stop kills a thread's own scope but never a shared Codex scope (OOM policy $oomPolicy)",
223 ({ oomPolicy }) =>
224 withFakeSystemd(
225 ({ scope, units, calls, cgroupRoot }) =>
226 Effect.gen(function* () {
227 const agent = yield* scope.wrap({
228 command: "agent",
229 args: [],
230 name: "t",
231 threadId: "a",
232 });
233 exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
234 const codex = yield* launchCodex(scope, "b");
235 exitedAgent(units, cgroupRoot, unitOf(codex), { populated: true, oomKills: 0 });
236​
237 expect(yield* scope.kill("a")).toBe(true);
238 expect(
239 NodeFS.readFileSync(NodePath.join(cgroupRoot, unitOf(agent), "cgroup.kill"), "utf8"),
240 ).toBe("1");
241 expect(yield* scope.kill("b")).toBe(false);
242 expect(NodeFS.existsSync(NodePath.join(cgroupRoot, unitOf(codex), "cgroup.kill"))).toBe(
243 false,
244 );
245 expect(calls.filter(([, verb]) => verb === "kill")).toEqual([]);
246 }),
247 { oomPolicy },
248 ),
249 );
250​
251 it.effect("keeps a running scope while hundreds of other threads launch", () =>
252 withFakeSystemd(({ scope, units }) =>
253 Effect.gen(function* () {
254 const live = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "live" });
255 units.set(unitOf(live), { activeState: "active", result: "success" });
256 for (let index = 0; index < 600; index++) {
257 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: `other-${index}` });
258 }
259 units.set(unitOf(live), { activeState: "failed", result: "oom-kill" });
260 expect(yield* scope.oomKilled("live")).toBe(true);
261 }),
262 ),
263 );
264​
265 it.effect("delegates every agent scope so rootless containers can manage their cgroups", () =>
266 withFakeSystemd(({ scope }) =>
267 Effect.gen(function* () {
268 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
269 const codex = yield* launchCodex(scope, "a");
270 const terminal = yield* scope.wrap({
271 command: "agent",
272 args: [],
273 name: "terminal",
274 survivesRestart: true,
275 });
276 for (const launch of [agent, codex, terminal]) {
277 expect(launch.args).toContain("--property=Delegate=yes");
278 }
279 }),
280 ),
281 );
282​
283 it.effect("caps agent memory and yields CPU, disk, and tasks to the server", () =>
284 withFakeSystemd(({ scope, calls }) =>
285 Effect.gen(function* () {
286 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
287 const setProperties = calls.filter(([, verb]) => verb === "set-property");
288 expect(setProperties).toEqual([
289 [
290 "--user",
291 "set-property",
292 "--runtime",
293 "app-t3code-agents.slice",
294 expect.stringMatching(/^MemoryHigh=\d+$/),
295 expect.stringMatching(/^MemoryMax=\d+$/),
296 ],
297 [
298 "--user",
299 "set-property",
300 "--runtime",
301 "app-t3code.slice",
302 "CPUWeight=10",
303 "IOWeight=10",
304 "TasksMax=25%",
305 ],
306 ]);
307 }),
308 ),
309 );
310​
311 it.effect("stops agents a killed server left before the first launch", () =>
312 withFakeSystemd(({ scope, units, calls, stateDir }) =>
313 Effect.gen(function* () {
314 units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" });
315 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
316 const stops = calls.filter(([, verb]) => verb === "stop");
317 expect(stops).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]);
318 expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir));
319 expect(agent.args).toContain("--property=TimeoutStopSec=10s");
320 const terminal = yield* scope.wrap({
321 command: "agent",
322 args: [],
323 name: "terminal",
324 survivesRestart: true,
325 });
326 expect(sliceOf(terminal)).toBe("app-t3code-agents.slice");
327 expect(calls.filter(([, verb]) => verb === "stop")).toHaveLength(1);
328 }),
329 ),
330 );
331​
332 it.effect("stops leftover processes only in an active agent scope", () =>
333 withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
334 Effect.gen(function* () {
335 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
336 const unit = unitOf(agent);
337 expect(agent.unit).toBe(unit);
338​
339 yield* scope.stop("session-2.scope");
340 expect(calls.some((args) => args.includes("session-2.scope"))).toBe(false);
341 yield* scope.stop(unit);
342 exitedAgent(units, cgroupRoot, unit, { populated: false, oomKills: 0 });
343 yield* scope.stop(unit);
344 exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: 0 });
345 units.get(unit)!.activeState = "deactivating";
346 yield* scope.stop(unit);
347 expect(stopsOf(calls)).toEqual([]);
348​
349 units.get(unit)!.activeState = "active";
350 yield* scope.stop(unit);
351 expect(stopsOf(calls)).toEqual([["--user", "stop", "--no-block", unit]]);
352 expect(yield* scope.oomKilled("a")).toBe(false);
353 }),
354 ),
355 );
356​
357 it.effect.each([true, false])(
358 "cleans up an OOM exit without losing its diagnosis (OOMPolicy=%s)",
359 (oomPolicy) =>
360 withFakeSystemd(
361 ({ scope, units, calls, cgroupRoot }) =>
362 Effect.gen(function* () {
363 const agent = yield* scope.wrap({
364 command: "agent",
365 args: [],
366 name: "t",
367 threadId: "a",
368 });
369 const unit = unitOf(agent);
370 exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: undefined });
371 yield* scope.stop(unit);
372 expect(stopsOf(calls)).toEqual(
373 oomPolicy ? [] : [["--user", "stop", "--no-block", unit]],
374 );
375​
376 // The kernel records the kill before systemd records the OOM result.
377 exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: 1 });
378 yield* scope.stop(unit);
379 expect(stopsOf(calls)).toHaveLength(oomPolicy ? 0 : 2);
380​
381 if (oomPolicy) {
382 units.set(unit, { activeState: "failed", result: "oom-kill" });
383 expect(yield* scope.oomKilled("a")).toBe(true);
384 units.delete(unit);
385 expect(yield* scope.oomKilled("a")).toBe(true);
386 }
387 }),
388 { oomPolicy },
389 ),
390 );
391​
392 it.effect("reports failed daemon cleanup without failing session teardown", () => {
393 const warnings: Array<unknown> = [];
394 const logger = Logger.make(({ message }) => {
395 warnings.push(message);
396 });
397 return withFakeSystemd(
398 ({ scope, units, cgroupRoot }) =>
399 Effect.gen(function* () {
400 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
401 exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
402 yield* scope.stop(unitOf(agent));
403 expect(warnings.flat()).toContain("Could not stop an exited agent's scope");
404 }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))),
405 { stopFails: true },
406 );
407 });
408​
409 it.effect("leaves a quiet slice alone", () =>
410 withFakeSystemd(({ scope, calls }) =>
411 Effect.gen(function* () {
412 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
413 expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
414 }),
415 ),
416 );
417});