Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

agentScope.ts

430 lines16.6 KB

1// @effect-diagnostics nodeBuiltinImport:off -- unit names and RAM size come from the host.
2import * as NodeCrypto from "node:crypto";
3import * as NodeOS from "node:os";
4​
5import {
6 AgentScope,
7 agentScopeCommand,
8 type AgentScopeShape,
9 type AgentSystemdScope,
10} from "@t3tools/shared/AgentScope";
11import * as HostProcess from "@t3tools/shared/HostProcess";
12import { resolveCommandPath } from "@t3tools/shared/shell";
13import * as Effect from "effect/Effect";
14import * as FileSystem from "effect/FileSystem";
15import * as Layer from "effect/Layer";
16import * as Path from "effect/Path";
17​
18import * as ServerConfig from "../config.ts";
19import * as ProcessRunner from "../processRunner.ts";
20​
21const AGENT_SLICE = "app-t3code-agents.slice";
22// Owns every agent slice and competes with the server's unit or scope under app.slice.
23const AGENTS_PARENT_SLICE = "app-t3code.slice";
24// Low weights yield CPU and disk to the server under contention, where the user manager
25// has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents
26// below logind's 33% per-user task limit, leaving forks for the server.
27const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"];
28​
29export function ownedAgentSlice(stateDir: string): string {
30 const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
31 return `app-t3code-agents-${hash}.slice`;
33​
34const GiB = 1024 ** 3;
35​
36/**
37 * Agents together leave about 6 GB free before they are throttled and 4 GB
38 * before the kernel kills one, so the server, sshd, and tailscaled keep
39 * working. Small machines still give agents at least half of RAM before
40 * throttling and three quarters before a kill.
41 */
42function agentSliceMemoryLimits(totalBytes: number) {
43 return {
44 high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)),
45 max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)),
46 };
48​
49export interface ScopeState {
50 readonly loadState: string;
51 readonly activeState: string;
52 readonly result: string;
53 readonly oomKills: number | undefined;
54 readonly populated: boolean | undefined;
56​
57/** Pipes can close before systemd records oom-kill; stopping, empty or OOM-counted scopes remain pending. */
58export function classifyScope(state: ScopeState): "oom-killed" | "gone" | "running" | "stopping" {
59 if (state.result === "oom-kill") return "oom-killed";
60 if (
61 state.loadState !== "loaded" ||
62 state.activeState === "inactive" ||
63 state.activeState === "failed"
64 ) {
65 return "gone";
66 }
67 if (
68 state.activeState === "deactivating" ||
69 (state.oomKills ?? 0) > 0 ||
70 state.populated === false
71 ) {
72 return "stopping";
73 }
74 return "running";
76​
77function parseKeyValues(text: string): Map<string, string> {
78 const values = new Map<string, string>();
79 for (const line of text.split("\n")) {
80 const index = line.indexOf("=");
81 if (index > 0) values.set(line.slice(0, index), line.slice(index + 1).trim());
82 }
83 return values;
85​
86function parseCgroupCounter(text: string, key: string): number | undefined {
87 const match = new RegExp(`^${key} (\\d+)$`, "m").exec(text);
88 return match === null ? undefined : Number(match[1]);
90​
91// systemd's default 90 s stop timeout would outlast our leftover-agent wait.
92// Delegation lets rootless containers (Podman, kind) manage their own cgroups.
93const SCOPE_OPTIONS = ["--property=TimeoutStopSec=10s", "--property=Delegate=yes"];
94const STOPPING_POLL_INTERVAL = "100 millis";
95const STOPPING_POLL_ATTEMPTS = 20;
96const CLEAR_FAILED_INTERVAL = "1 second";
97const CLEAR_FAILED_ATTEMPTS = 120;
98const MAX_TRACKED_THREADS = 500;
99​
100export const make = Effect.gen(function* () {
101 const platform = yield* HostProcess.Platform;
102 if (platform !== "linux") return AgentScope.defaultValue();
103​
104 const runner = yield* ProcessRunner.ProcessRunner;
105 const fileSystem = yield* FileSystem.FileSystem;
106 const path = yield* Path.Path;
107 const hostEnvironment = yield* HostProcess.Environment;
108 const { stateDir } = yield* ServerConfig.ServerConfig;
109 const ownedSlice = ownedAgentSlice(stateDir);
110 const layerScope = yield* Effect.scope;
111​
112 const resolve = (command: string, env: NodeJS.ProcessEnv) =>
113 resolveCommandPath(command, { env }).pipe(
114 Effect.provideService(FileSystem.FileSystem, fileSystem),
115 Effect.provideService(Path.Path, path),
116 Effect.provideService(HostProcess.Platform, platform),
117 );
118​
119 const run = (command: string, args: ReadonlyArray<string>) =>
120 runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option);
121​
122 const readCgroupFile = (controlGroup: string, file: string) =>
123 fileSystem
124 .readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
125 .pipe(Effect.orElseSucceed(() => ""));
126​
127 // Old agents keep their scopes when the server is killed, and the restarted
128 // server resumes their sessions. Stop them first, so two copies of one
129 // session never work in the same worktree.
130 const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* (
131 systemctl: string,
132 ) {
133 const shown = yield* run(systemctl, [
134 "--user",
135 "show",
136 ownedSlice,
137 "--property=ActiveState,ControlGroup",
138 ]);
139 if (shown._tag === "None") return;
140 const values = parseKeyValues(shown.value.stdout);
141 if (values.get("ActiveState") !== "active") return;
142 const controlGroup = values.get("ControlGroup") ?? "";
143 const events = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
144 if (parseCgroupCounter(events, "populated") === 0) return;
145 yield* Effect.logWarning("Stopping agents left by a previous server", { slice: ownedSlice });
146 const stopped = yield* runner
147 .run({ command: systemctl, args: ["--user", "stop", ownedSlice], timeout: "30 seconds" })
148 .pipe(Effect.option);
149 if (stopped._tag === "None" || stopped.value.code !== 0) {
150 yield* Effect.logWarning("Could not stop agents left by a previous server", {
151 slice: ownedSlice,
152 stderr: stopped._tag === "Some" ? stopped.value.stderr.trim() : undefined,
153 });
154 }
155 });
156​
157 const systemd = yield* Effect.cached(
158 Effect.gen(function* () {
159 const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR;
160 if (runtimeDir === undefined || runtimeDir.length === 0) return undefined;
161 const systemdRun = yield* resolve("systemd-run", hostEnvironment);
162 const systemctl = yield* resolve("systemctl", hostEnvironment);
163 if (systemdRun === null || systemctl === null) return undefined;
164 yield* stopLeftoverAgents(systemctl);
165 const probe = (options: ReadonlyArray<string>) =>
166 run(systemdRun, [
167 "--user",
168 "--scope",
169 "--quiet",
170 "--collect",
171 `--slice=${AGENT_SLICE}`,
172 ...options,
173 "--",
174 "true",
175 ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0));
176 // Scopes take OOMPolicy from systemd 253; older managers still get scopes.
177 const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]);
178 if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) {
179 yield* Effect.logInfo("Agent scopes are off: no systemd user manager");
180 return undefined;
181 }
182 const setProperties = Effect.fn("agentScope.setProperties")(function* (
183 slice: string,
184 properties: ReadonlyArray<string>,
185 ) {
186 const result = yield* run(systemctl, [
187 "--user",
188 "set-property",
189 "--runtime",
190 slice,
191 ...properties,
192 ]);
193 if (result._tag === "None" || result.value.code !== 0) {
194 yield* Effect.logWarning("Could not set resource limits on an agents slice", {
195 slice,
196 properties,
197 stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined,
198 });
199 }
200 });
201 const limits = agentSliceMemoryLimits(NodeOS.totalmem());
202 yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]);
203 yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES);
204 yield* Effect.logInfo("Agent scopes are on", {
205 slice: AGENT_SLICE,
206 ...limits,
207 oomPolicy: supportsOomPolicy,
208 });
209 return {
210 systemdRun,
211 systemctl,
212 runtimeDir,
213 supportsOomPolicy,
214 };
215 }),
216 );
217​
218 // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd.
219 const scopeByThread = new Map<string, { unit: string; oomKilled: boolean; shared: boolean }>();
220​
221 const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) {
222 const shown = yield* run(systemctl, [
223 "--user",
224 "show",
225 unit,
226 "--property=LoadState,ActiveState,Result,ControlGroup",
227 ]);
228 if (shown._tag === "None" || shown.value.code !== 0) return undefined;
229 const values = parseKeyValues(shown.value.stdout);
230 const controlGroup = values.get("ControlGroup") ?? "";
231 const memoryEvents = controlGroup ? yield* readCgroupFile(controlGroup, "memory.events") : "";
232 const cgroupEvents = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
233 const populated = parseCgroupCounter(cgroupEvents, "populated");
234 return {
235 loadState: values.get("LoadState") ?? "",
236 activeState: values.get("ActiveState") ?? "",
237 result: values.get("Result") ?? "",
238 oomKills: parseCgroupCounter(memoryEvents, "oom_kill"),
239 populated: populated === undefined ? undefined : populated === 1,
240 } satisfies ScopeState;
241 });
242​
243 const clearFailed = Effect.fn("agentScope.clearFailed")(function* (
244 systemctl: string,
245 unit: string,
246 ) {
247 for (let attempt = 0; attempt < CLEAR_FAILED_ATTEMPTS; attempt++) {
248 const state = yield* readScope(systemctl, unit);
249 if (state === undefined) return;
250 if (state.activeState === "failed") {
251 yield* run(systemctl, ["--user", "reset-failed", unit]);
252 return;
253 }
254 if (state.activeState !== "deactivating") return;
255 yield* Effect.sleep(CLEAR_FAILED_INTERVAL);
256 }
257 });
258​
259 const forget = (systemctl: string, unit: string) =>
260 clearFailed(systemctl, unit).pipe(Effect.forkIn(layerScope), Effect.asVoid);
261​
262 const evictOldest = Effect.fn("agentScope.evictOldest")(function* (systemctl: string) {
263 const oldest = scopeByThread.entries().next();
264 if (oldest.done === true) return;
265 const [threadId, tracked] = oldest.value;
266 const state = yield* readScope(systemctl, tracked.unit);
267 if (scopeByThread.get(threadId) !== tracked) return;
268 scopeByThread.delete(threadId);
269 const status = state === undefined ? "running" : classifyScope(state);
270 if (status === "running" || status === "stopping") {
271 scopeByThread.set(threadId, tracked);
272 return;
273 }
274 yield* forget(systemctl, tracked.unit);
275 });
276​
277 const track = Effect.fn("agentScope.track")(function* (
278 systemctl: string,
279 threadId: string,
280 unit: string | undefined,
281 shared = false,
282 ) {
283 const previous = scopeByThread.get(threadId);
284 scopeByThread.delete(threadId);
285 if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared });
286 if (previous !== undefined) yield* forget(systemctl, previous.unit);
287 if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl);
288 });
289​
290 const oomKilled: AgentScopeShape["oomKilled"] = Effect.fn("agentScope.oomKilled")(
291 function* (threadId) {
292 const tracked = scopeByThread.get(threadId);
293 if (tracked === undefined) return false;
294 if (tracked.oomKilled) return true;
295 const scope = yield* systemd;
296 if (scope === undefined) return false;
297 for (let attempt = 0; attempt < STOPPING_POLL_ATTEMPTS; attempt++) {
298 const state = yield* readScope(scope.systemctl, tracked.unit);
299 if (state === undefined) return false;
300 const status = classifyScope(state);
301 if (status === "running") return false;
302 if (status === "stopping") {
303 yield* Effect.sleep(STOPPING_POLL_INTERVAL);
304 continue;
305 }
306 yield* forget(scope.systemctl, tracked.unit);
307 if (status === "gone") return false;
308 yield* Effect.logWarning("Agent scope was killed: out of memory", {
309 threadId,
310 unit: tracked.unit,
311 });
312 tracked.oomKilled = true;
313 return true;
314 }
315 return false;
316 },
317 );
318​
319 const stop = Effect.fn("AgentScope.stop")(function* (unit: string) {
320 if (!unit.startsWith("t3code-") || !unit.endsWith(".scope")) return;
321 const scope = yield* systemd;
322 if (scope === undefined) return;
323 const state = yield* readScope(scope.systemctl, unit);
324 if (
325 state === undefined ||
326 state.loadState !== "loaded" ||
327 state.activeState !== "active" ||
328 state.result === "oom-kill" ||
329 state.populated === false
330 )
331 return;
332 // The kernel counts OOM kills before signaling the victim. Let OOMPolicy
333 // record the result before a manual stop can overwrite it; an unreadable
334 // counter cannot rule out that race. Older managers need us to stop it.
335 if (scope.supportsOomPolicy && (state.oomKills === undefined || state.oomKills > 0)) return;
336 const stopped = yield* run(scope.systemctl, ["--user", "stop", "--no-block", unit]);
337 if (stopped._tag === "None" || stopped.value.code !== 0) {
338 yield* Effect.logWarning("Could not stop an exited agent's scope", {
339 unit,
340 exitCode: stopped._tag === "Some" ? stopped.value.code : undefined,
341 });
342 }
343 });
344​
345 const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) {
346 const tracked = scopeByThread.get(threadId);
347 if (tracked === undefined || tracked.shared) return false;
348 const scope = yield* systemd;
349 if (scope === undefined) return false;
350 const shown = yield* run(scope.systemctl, [
351 "--user",
352 "show",
353 tracked.unit,
354 "--property=ActiveState,ControlGroup",
355 ]);
356 if (shown._tag === "None" || shown.value.code !== 0) return false;
357 const values = parseKeyValues(shown.value.stdout);
358 if (values.get("ActiveState") !== "active") return false;
359 const controlGroup = values.get("ControlGroup") ?? "";
360 // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback.
361 const killedByCgroup =
362 controlGroup.length > 0 &&
363 (yield* fileSystem
364 .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1")
365 .pipe(
366 Effect.as(true),
367 Effect.orElseSucceed(() => false),
368 ));
369 if (killedByCgroup) return true;
370 const killed = yield* run(scope.systemctl, [
371 "--user",
372 "kill",
373 "--signal=SIGKILL",
374 tracked.unit,
375 ]);
376 return killed._tag === "Some" && killed.value.code === 0;
377 });
378​
379 const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) {
380 if (!scopeByThread.has(threadId)) return;
381 const scope = yield* systemd;
382 if (scope !== undefined) yield* track(scope.systemctl, threadId, undefined);
383 });
384​
385 return {
386 wrap: Effect.fn("AgentScope.wrap")(function* ({
387 command,
388 args,
389 name,
390 threadId,
391 env,
392 survivesRestart,
393 shared,
394 }) {
395 const resolved = yield* resolve(command, env ?? hostEnvironment);
396 if (resolved === null) return { command, args };
397 const scope = yield* systemd;
398 if (scope === undefined) return agentScopeCommand({ command: resolved, args });
399 const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`;
400 // A continuing shared scope cannot attribute a tool's OOM to one thread.
401 // Clear the launch thread's previous scope instead.
402 const continuing = shared === true && scope.supportsOomPolicy;
403 const owned = threadId !== undefined && !continuing;
404 if (threadId !== undefined)
405 yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true);
406 const systemdScope: AgentSystemdScope = {
407 systemdRun: scope.systemdRun,
408 runtimeDir: scope.runtimeDir,
409 slice: survivesRestart === true ? AGENT_SLICE : ownedSlice,
410 unit,
411 // Without an owning thread nothing reads the result, so let systemd
412 // unload the unit even when it fails.
413 options: [
414 ...SCOPE_OPTIONS,
415 ...(scope.supportsOomPolicy
416 ? [`--property=OOMPolicy=${continuing ? "continue" : "stop"}`]
417 : []),
418 ...(owned ? [] : ["--collect"]),
419 ],
420 };
421 return { ...agentScopeCommand({ command: resolved, args, scope: systemdScope }), unit };
422 }),
423 oomKilled,
424 stop,
425 kill,
426 clear,
427 } satisfies AgentScopeShape;
428});
429​
430export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer));