1// Rebuilds workspace packages whose build output is missing or stale.
2//
3// Some packages are consumed from their build output even in the
4// workspace: cloudflare-runtime's source imports its bundled workers from
5// `dist/`, and frontend-frameworks only exports `dist/`. Commands that need
6// them (tests, type checking, local deploys) run `ensure` first, which is a
7// no-op unless a package or one of its workspace dependencies changed since
8// its last build.
9//
10// Every successful build of these packages records itself in
11// `<outDir>/.build-stamp` (see package-build.ts). A package is stale when
12// that stamp is missing or older than any tracked or untracked-but-not-ignored
13// file of the package or its workspace dependencies, test files excepted.
14//
15// Distilled SDKs are consumed the same way by bundlers: their `default`
16// export condition points at `lib/`, which distilled's `scripts/bundle-lib.ts`
17// turns from `tsc -b` output into the chunked layout npm ships, with the
18// `distilled-lazy` stubs in `lib/services.lazy/` (only bundle-lib writes
19// those). After the builds above, every distilled package that has a `lib/`
20// but no bundle stamp newer than its source and its `lib/` is bundled. Stamps live in `node_modules/.cache/distilled-bundle/<package>`
21// (bundle-lib prunes unknown files from `lib/`).
22//
23// Usage: pnpm -w ensure:built [package...]
24import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
25import * as NodeServices from "@effect/platform-node/NodeServices";
26import { Argument, Command } from "effect/cli";
27import * as Effect from "effect/Effect";
28import * as FileSystem from "effect/FileSystem";
29import * as Option from "effect/Option";
30import * as Path from "effect/Path";
31import { ChildProcess } from "effect/process";
32import { ChildProcessSpawner } from "effect/process/ChildProcessSpawner";
33import { command as platformCommand, stampFile } from "./package-build.ts";
34
35/** Packages consumed from their build output, by directory name under packages/. */
36const packages = {
37 "cloudflare-runtime": { outDir: "dist" },
38 "frontend-frameworks": { outDir: "dist" },
39} as const;
40
41type PackageName = keyof typeof packages;
42const packageNames = Object.keys(packages) as Array<PackageName>;
43
44/** Changes to tests never affect build output. */
45const isTestFile = (file: string) =>
46 /(^|\/)(test|tests|__tests__)\//.test(file) || /\.test\.[cm]?[jt]sx?$/.test(file);
47
48const workspaceRoot = Effect.gen(function* () {
49 const path = yield* Path.Path;
50 return path.resolve(import.meta.dirname, "..");
51});
52
53/** Directories of a package and all of its workspace dependencies. */
54const dependencyDirectories = Effect.fn(function* (root: string, name: PackageName) {
55 const spawner = yield* ChildProcessSpawner;
56 const output = yield* spawner.string(
57 yield* platformCommand(
58 "pnpm",
59 [
60 "ls",
61 "-r",
62 "--filter",
63 `{./packages/${name}}...`,
64 "--depth",
65 "-1",
66 "--json",
67 "--loglevel",
68 "error",
69 ],
70 { cwd: root },
71 ),
72 );
73 return (JSON.parse(output) as Array<{ path: string }>).map((project) => project.path);
74});
75
76/** Newest modification time across a directory's source files. */
77const newestSourceTime = Effect.fn(function* (directory: string) {
78 const fs = yield* FileSystem.FileSystem;
79 const path = yield* Path.Path;
80 const spawner = yield* ChildProcessSpawner;
81
82 // `git -C` also works inside submodules, which the parent repo can't list.
83 const files = yield* spawner.lines(
84 yield* platformCommand(
85 "git",
86 ["-C", directory, "ls-files", "--cached", "--others", "--exclude-standard", "."],
87 { cwd: directory },
88 ),
89 );
90 const times = yield* Effect.forEach(
91 files.filter((file) => file !== "" && !isTestFile(file)),
92 (file) =>
93 fs.stat(path.join(directory, file)).pipe(
94 Effect.map((info) => Option.getOrElse(info.mtime, () => new Date(0)).getTime()),
95 // Deleted but still-tracked files have nothing to compare.
96 Effect.orElseSucceed(() => 0),
97 ),
98 { concurrency: 64 },
99 );
100 return Math.max(0, ...times);
101});
102
103const stampTime = Effect.fn(function* (root: string, name: PackageName) {
104 const fs = yield* FileSystem.FileSystem;
105 const path = yield* Path.Path;
106 return yield* fs.stat(path.join(root, "packages", name, packages[name].outDir, stampFile)).pipe(
107 Effect.map((info) => Option.getOrElse(info.mtime, () => new Date(0)).getTime()),
108 Effect.option,
109 );
110});
111
112const isStale = Effect.fn(function* (root: string, name: PackageName) {
113 const stamp = yield* stampTime(root, name);
114 if (Option.isNone(stamp)) return true;
115 const directories = yield* dependencyDirectories(root, name);
116 const newest = yield* Effect.forEach(directories, newestSourceTime, { concurrency: "unbounded" });
117 return Math.max(...newest) > stamp.value;
118});
119
120/** Where the last successful bundle of a distilled package is recorded. */
121const distilledStamp = (root: string, path: Path.Path, name: string) =>
122 path.join(root, "node_modules", ".cache", "distilled-bundle", name);
123
124/** Distilled packages whose bundled `lib/` is missing or older than their source. */
125const staleDistilled = Effect.fn(function* (root: string) {
126 const fs = yield* FileSystem.FileSystem;
127 const path = yield* Path.Path;
128 const distilled = path.join(root, "submodules", "distilled");
129 if (!(yield* fs.exists(path.join(distilled, "scripts", "bundle-lib.ts")))) return [];
130 const names = yield* fs.readDirectory(path.join(distilled, "packages"));
131 return yield* Effect.filter(
132 names.filter((name) => name !== "core"),
133 Effect.fnUntraced(function* (name) {
134 const directory = path.join(distilled, "packages", name);
135 if (!(yield* fs.exists(path.join(directory, "lib")))) return false;
136 if (!(yield* fs.exists(path.join(directory, "src", "services")))) return false;
137 const mtime = (file: string) =>
138 fs.stat(file).pipe(
139 Effect.map((info) => Option.getOrElse(info.mtime, () => new Date(0)).getTime()),
140 Effect.option,
141 );
142 const stamp = yield* mtime(distilledStamp(root, path, name));
143 if (Option.isNone(stamp)) return true;
144 // A wiped and re-emitted `lib/` (fresh tsc output) is newer than the stamp.
145 const lib = yield* mtime(path.join(directory, "lib"));
146 if (Option.isSome(lib) && lib.value > stamp.value) return true;
147 return (yield* newestSourceTime(path.join(directory, "src"))) > stamp.value;
148 }),
149 { concurrency: 16 },
150 );
151});
152
153const bundleDistilled = Effect.fn(function* (root: string) {
154 const fs = yield* FileSystem.FileSystem;
155 const path = yield* Path.Path;
156 const spawner = yield* ChildProcessSpawner;
157 const stale = yield* staleDistilled(root);
158 if (stale.length === 0) return;
159 yield* Effect.logInfo(`Bundling stale distilled packages: ${stale.join(", ")}`);
160 const pathEnv = yield* Effect.sync(() => process.env.PATH ?? "");
161 const exitCode = yield* spawner.exitCode(
162 ChildProcess.make(
163 "node",
164 ["scripts/bundle-lib.ts", ...stale.map((name) => `packages/${name}`)],
165 {
166 cwd: path.join(root, "submodules", "distilled"),
167 // bundle-lib resolves rolldown and the formatters from the workspace.
168 env: {
169 PATH: [path.join(root, "node_modules", ".bin"), pathEnv].join(
170 process.platform === "win32" ? ";" : ":",
171 ),
172 },
173 extendEnv: true,
174 stdin: "inherit",
175 stdout: "inherit",
176 stderr: "inherit",
177 },
178 ),
179 );
180 if (exitCode !== 0) {
181 return yield* Effect.fail(new Error(`Distilled bundle failed with exit code ${exitCode}`));
182 }
183 yield* fs.makeDirectory(path.dirname(distilledStamp(root, path, stale[0]!)), {
184 recursive: true,
185 });
186 yield* Effect.forEach(stale, (name) => fs.writeFileString(distilledStamp(root, path, name), ""));
187});
188
189const command = Command.make(
190 "ensure-built",
191 {
192 packages: Argument.Literals("package", packageNames).pipe(
193 Argument.withDescription("Packages to check (defaults to all output-consumed packages)"),
194 Argument.atLeast(0),
195 ),
196 },
197 Effect.fn(function* ({ packages: selected }) {
198 const root = yield* workspaceRoot;
199 const spawner = yield* ChildProcessSpawner;
200
201 const candidates = selected.length > 0 ? selected : packageNames;
202 const stale = yield* Effect.filter(candidates, (name) => isStale(root, name), {
203 concurrency: "unbounded",
204 });
205 if (stale.length === 0) return yield* bundleDistilled(root);
206
207 yield* Effect.logInfo(`Rebuilding stale packages: ${stale.join(", ")}`);
208 const exitCode = yield* spawner.exitCode(
209 yield* platformCommand(
210 "pnpm",
211 [
212 "-r",
213 ...stale.flatMap((name) => ["--filter", `{./packages/${name}}...`]),
214 "run",
215 "build:package",
216 ],
217 { cwd: root, stdin: "inherit", stdout: "inherit", stderr: "inherit" },
218 ),
219 );
220 if (exitCode !== 0) {
221 return yield* Effect.fail(new Error(`Build failed with exit code ${exitCode}`));
222 }
223 yield* bundleDistilled(root);
224 }),
225).pipe(Command.withDescription("Rebuild packages whose build output is missing or stale"));
226
227Command.run(command, { version: "0.0.0" }).pipe(
228 Effect.provide(NodeServices.layer),
229 NodeRuntime.runMain,
230);