Open History to see the last commit
1import * as Effect from "effect/Effect";
2import * as FileSystem from "effect/FileSystem";
3import * as Option from "effect/Option";
4import * as Path from "effect/Path";
5import * as Schema from "effect/Schema";
6import * as ChildProcess from "effect/process/ChildProcess";
7import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
8
9import * as ProcessRunner from "../processRunner.ts";
10import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts";
11
12const PROTECT_DROP_IN = "90-t3code-protect.conf";
13const SYSTEM_UNIT_DIR = "/etc/systemd/system";
14export const REMOTE_ACCESS_UNITS = ["tailscaled.service", "ssh.service", "sshd.service"];
15
16// Covers app.slice (the server's 1536M) and init.scope (the user manager's 256M).
17const USER_CHAIN_LOW = "MemoryLow=2G";
18// Near the floor, but unlike -1000 the kernel can still kill it as a last resort.
19const PROTECTED_OOM_SCORE = "OOMScoreAdjust=-900";
20
21export interface DropIn {
22 readonly unit: string;
23 readonly path: string;
24 readonly contents: string;
25 /** Host-wide, so undo keeps it while another user's protection remains. */
26 readonly shared?: boolean;
27}
28
29export interface ProtectionPlan {
30 readonly systemUnitDir: string;
31 /** Written through sudo. */
32 readonly system: ReadonlyArray<DropIn>;
33 /**
34 * Written before the manager's own score drops. Unset, a user manager hands
35 * its services its own score plus 100: -800 once it runs at -900.
36 */
37 readonly managerDefaults: DropIn;
38 readonly user: ReadonlyArray<DropIn>;
39 /** System units whose running main process takes its configured OOM score at once. */
40 readonly oomUnits: ReadonlyArray<string>;
41}
42
43const dropIn = (
44 directory: string,
45 unit: string,
46 section: string,
47 properties: ReadonlyArray<string>,
48 shared = false,
49): DropIn => ({
50 unit,
51 path: `${directory}/${unit}.d/${PROTECT_DROP_IN}`,
52 contents: [`[${section}]`, ...properties, ""].join("\n"),
53 ...(shared ? { shared } : {}),
54});
55
56/**
57 * Memory protection only reaches the server when every ancestor cgroup grants
58 * it, and only root can grant it on user.slice and the user manager.
59 */
60export function protectionPlan(input: {
61 readonly uid: number;
62 readonly userUnitDir: string;
63 readonly remoteAccessUnits: ReadonlyArray<string>;
64 readonly systemUnitDir?: string;
65}): ProtectionPlan {
66 const manager = `user@${input.uid}.service`;
67 const remote = input.remoteAccessUnits;
68 const systemUnitDir = input.systemUnitDir ?? SYSTEM_UNIT_DIR;
69 const system = (unit: string, section: string, properties: ReadonlyArray<string>) =>
70 dropIn(systemUnitDir, unit, section, properties);
71 const shared = (unit: string, section: string, properties: ReadonlyArray<string>) =>
72 dropIn(systemUnitDir, unit, section, properties, true);
73 const user = (unit: string, section: string, properties: ReadonlyArray<string>) =>
74 dropIn(input.userUnitDir, unit, section, properties);
75 return {
76 systemUnitDir,
77 system: [
78 shared("user.slice", "Slice", [USER_CHAIN_LOW]),
79 system(`user-${input.uid}.slice`, "Slice", [USER_CHAIN_LOW]),
80 system(manager, "Service", [USER_CHAIN_LOW, PROTECTED_OOM_SCORE]),
81 ...(remote.length > 0 ? [shared("system.slice", "Slice", ["MemoryLow=512M"])] : []),
82 ...remote.map((unit) => shared(unit, "Service", ["MemoryLow=256M", PROTECTED_OOM_SCORE])),
83 ],
84 managerDefaults: {
85 unit: "user.conf",
86 path: `${input.userUnitDir.replace(/\/user$/, "")}/user.conf.d/${PROTECT_DROP_IN}`,
87 contents: "[Manager]\nDefaultOOMScoreAdjust=200\n",
88 },
89 user: [
90 user("app.slice", "Slice", ["MemoryLow=1536M"]),
91 user("init.scope", "Scope", ["MemoryLow=256M"]),
92 user(BOOT_SERVICE_UNIT_FILE, "Service", [PROTECTED_OOM_SCORE]),
93 ],
94 oomUnits: [manager, ...remote],
95 };
96}
97
98const quoteShell = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`;
99const directoryOf = (file: string) => file.slice(0, file.lastIndexOf("/"));
100
101export function renderPrivilegedScript(plan: ProtectionPlan, mode: "apply" | "undo"): string {
102 const lines = ["set -e"];
103 const remove = (dropIns: ReadonlyArray<DropIn>, indent = "") => {
104 for (const { path } of dropIns) {
105 lines.push(
106 `${indent}rm -f ${quoteShell(path)}`,
107 `${indent}rmdir ${quoteShell(directoryOf(path))} 2>/dev/null || true`,
108 );
109 }
110 };
111 if (mode === "apply") {
112 for (const { path, contents } of plan.system) {
113 lines.push(
114 `mkdir -p ${quoteShell(directoryOf(path))}`,
115 `cat >${quoteShell(path)} <<'T3CODE_PROTECT'`,
116 contents.trimEnd(),
117 "T3CODE_PROTECT",
118 );
119 }
120 } else {
121 remove(plan.system.filter((dropIn) => dropIn.shared !== true));
122 const otherUsers = `${quoteShell(plan.systemUnitDir)}/user@*.service.d/${PROTECT_DROP_IN}`;
123 lines.push(`set -- ${otherUsers}`, 'if [ ! -e "$1" ]; then');
124 remove(
125 plan.system.filter((dropIn) => dropIn.shared === true),
126 " ",
127 );
128 lines.push("fi");
129 }
130 lines.push(
131 "systemctl daemon-reload",
132 // OOMScoreAdjust applies at exec; this gives running main processes the configured score.
133 `for unit in ${plan.oomUnits.map(quoteShell).join(" ")}; do`,
134 ' pid=$(systemctl show --property=MainPID --value "$unit")',
135 ' if [ "${pid:-0}" -gt 0 ]; then',
136 ' systemctl show --property=OOMScoreAdjust --value "$unit" >"/proc/$pid/oom_score_adj" || true',
137 " fi",
138 "done",
139 "",
140 );
141 return lines.join("\n");
142}
143
144export class ServiceProtectionError extends Schema.TaggedError<ServiceProtectionError>()(
145 "ServiceProtectionError",
146 {
147 step: Schema.String,
148 exitCode: Schema.optional(Schema.Number),
149 cause: Schema.optional(Schema.Defect()),
150 },
151) {
152 override get message(): string {
153 return this.exitCode === undefined
154 ? `Could not ${this.step}.`
155 : `Could not ${this.step} (exit code ${this.exitCode}).`;
156 }
157}
158
159function parseUnits(text: string): Array<Map<string, string>> {
160 return text
161 .split(/\n\s*\n/)
162 .map(
163 (block) =>
164 new Map(
165 block
166 .split("\n")
167 .filter((line) => line.includes("="))
168 .map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]),
169 ),
170 )
171 .filter((values) => values.size > 0);
172}
173
174/** Loaded remote-access units by canonical name, so an alias such as sshd.service counts once. */
175export function loadedRemoteAccessUnits(systemctlShow: string): ReadonlyArray<string> {
176 const units = parseUnits(systemctlShow)
177 .filter((values) => values.get("LoadState") === "loaded")
178 .map((values) => values.get("Id") ?? "")
179 .filter((id) => REMOTE_ACCESS_UNITS.includes(id));
180 return [...new Set(units)];
181}
182
183const run = Effect.fn("ServiceProtection.run")(function* (
184 step: string,
185 command: string,
186 args: ReadonlyArray<string>,
187) {
188 const runner = yield* ProcessRunner.ProcessRunner;
189 const result = yield* runner
190 .run({ command, args, timeout: "30 seconds" })
191 .pipe(Effect.mapError((cause) => new ServiceProtectionError({ step, cause })));
192 if (result.code !== 0) {
193 return yield* new ServiceProtectionError({ step, exitCode: Number(result.code) });
194 }
195 return result.stdout;
196});
197
198const userSettingsError = (cause: unknown) =>
199 new ServiceProtectionError({ step: "change systemd user settings", cause });
200
201/** Undefined when there is no systemd system manager to ask. */
202export const detectRemoteAccessUnits = run("list remote-access services", "systemctl", [
203 "show",
204 "--property=Id,LoadState",
205 ...REMOTE_ACCESS_UNITS,
206]).pipe(
207 Effect.map(loadedRemoteAccessUnits),
208 Effect.option,
209 Effect.map(Option.getOrUndefined),
210 Effect.withSpan("ServiceProtection.detectRemoteAccessUnits"),
211);
212
213const reloadUserManager = run("reload systemd user units", "systemctl", [
214 "--user",
215 "daemon-reload",
216]);
217
218const writeDropIns = Effect.fn("ServiceProtection.writeDropIns")(function* (
219 dropIns: ReadonlyArray<DropIn>,
220) {
221 const fs = yield* FileSystem.FileSystem;
222 for (const { path, contents } of dropIns) {
223 yield* fs.makeDirectory(directoryOf(path), { recursive: true });
224 yield* fs.writeFileString(path, contents);
225 }
226}, Effect.mapError(userSettingsError));
227
228const removeDropIns = Effect.fn("ServiceProtection.removeDropIns")(function* (
229 dropIns: ReadonlyArray<DropIn>,
230) {
231 const fs = yield* FileSystem.FileSystem;
232 for (const { path } of dropIns) {
233 yield* fs.remove(path, { force: true });
234 yield* fs.remove(directoryOf(path)).pipe(Effect.ignore);
235 }
236}, Effect.mapError(userSettingsError));
237
238const runPrivileged = Effect.fn("ServiceProtection.runPrivileged")(function* (script: string) {
239 const step = "change the system's service settings";
240 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
241 const exitCode = yield* spawner
242 .exitCode(
243 // Effect detaches children on POSIX by default; sudo then has no controlling
244 // terminal and cannot ask for a password.
245 ChildProcess.make("sudo", ["/bin/sh", "-c", script], {
246 detached: false,
247 stdin: "inherit",
248 stdout: "inherit",
249 stderr: "inherit",
250 }),
251 )
252 .pipe(Effect.mapError((cause) => new ServiceProtectionError({ step, cause })));
253 if (exitCode !== 0) {
254 return yield* new ServiceProtectionError({ step, exitCode: Number(exitCode) });
255 }
256});
257
258export const applyProtection = Effect.fn("ServiceProtection.apply")(function* (
259 plan: ProtectionPlan,
260) {
261 const fs = yield* FileSystem.FileSystem;
262 const hadManagerDefaults = yield* fs
263 .exists(plan.managerDefaults.path)
264 .pipe(Effect.orElseSucceed(() => false));
265 yield* writeDropIns([plan.managerDefaults]);
266 // A partial run may already have lowered the manager; its default must then stay.
267 const manager = plan.system.find(({ unit }) => unit.startsWith("user@"));
268 const rollBack = Effect.gen(function* () {
269 if (hadManagerDefaults) return;
270 if (manager !== undefined && (yield* fs.exists(manager.path))) return;
271 yield* removeDropIns([plan.managerDefaults]);
272 }).pipe(Effect.ignore);
273 yield* runPrivileged(renderPrivilegedScript(plan, "apply")).pipe(Effect.tapError(() => rollBack));
274 yield* writeDropIns(plan.user);
275 yield* reloadUserManager;
276});
277
278export const removeProtection = Effect.fn("ServiceProtection.remove")(function* (
279 plan: ProtectionPlan,
280) {
281 yield* removeDropIns(plan.user);
282 yield* reloadUserManager;
283 if (plan.system.length > 0) yield* runPrivileged(renderPrivilegedScript(plan, "undo"));
284 yield* removeDropIns([plan.managerDefaults]);
285 yield* reloadUserManager;
286});
287
288/** The plan narrowed to drop-ins that exist, for undo. */
289export const installedProtection = Effect.fn("ServiceProtection.installed")(function* (
290 plan: ProtectionPlan,
291) {
292 const fs = yield* FileSystem.FileSystem;
293 const exists = (dropIn: DropIn) => fs.exists(dropIn.path).pipe(Effect.orElseSucceed(() => false));
294 const system = yield* Effect.filter(plan.system, exists);
295 const user = yield* Effect.filter(plan.user, exists);
296 const managerDefaults = yield* exists(plan.managerDefaults);
297 return system.length + user.length === 0 && !managerDefaults
298 ? undefined
299 : { ...plan, system, user };
300});
301
302export interface ServiceProtectionState {
303 /** The first cgroup from the top that grants the server no memory protection. */
304 readonly unprotected: string | undefined;
305 readonly oomScoreAdj: number | undefined;
306 readonly setByProtect: boolean;
307}
308
309/** Reads what the kernel applies to the running service, whoever configured it. */
310export const readProtection = Effect.fn("ServiceProtection.read")(function* (userUnitDir: string) {
311 const fs = yield* FileSystem.FileSystem;
312 const path = yield* Path.Path;
313 const shown = yield* run("read the service's cgroup", "systemctl", [
314 "--user",
315 "show",
316 BOOT_SERVICE_UNIT_FILE,
317 "--property=ControlGroup,MainPID",
318 ]).pipe(Effect.option);
319 if (Option.isNone(shown)) return undefined;
320 const values = parseUnits(shown.value)[0] ?? new Map<string, string>();
321 const controlGroup = (values.get("ControlGroup") ?? "").split("/").filter(Boolean);
322 if (controlGroup.length === 0) return undefined;
323 const read = (file: string) =>
324 fs.readFileString(file).pipe(
325 Effect.map((text) => text.trim()),
326 Effect.orElseSucceed(() => ""),
327 );
328 let unprotected: string | undefined;
329 const grants = (value: string) => value !== "" && value !== "0";
330 for (let depth = 1; depth <= controlGroup.length && unprotected === undefined; depth++) {
331 const directory = path.join("/sys/fs/cgroup", ...controlGroup.slice(0, depth));
332 const low = yield* read(path.join(directory, "memory.low"));
333 const min = yield* read(path.join(directory, "memory.min"));
334 if (!grants(low) && !grants(min)) unprotected = controlGroup[depth - 1];
335 }
336 const pid = Number(values.get("MainPID") ?? 0);
337 const score = pid > 0 ? yield* read(`/proc/${pid}/oom_score_adj`) : "";
338 return {
339 unprotected,
340 oomScoreAdj: score === "" ? undefined : Number(score),
341 setByProtect: yield* fs
342 .exists(path.join(userUnitDir, `${BOOT_SERVICE_UNIT_FILE}.d`, PROTECT_DROP_IN))
343 .pipe(Effect.orElseSucceed(() => false)),
344 } satisfies ServiceProtectionState;
345});