Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

PreviewBrowser.ts

447 lines17.0 KB

1// @effect-diagnostics nodeBuiltinImport:off - Effect has no incremental digest.
2import * as EffectNodeStream from "@effect/platform-node/NodeStream";
3import * as HostProcess from "@t3tools/shared/HostProcess";
4import * as Cause from "effect/Cause";
5import * as Clock from "effect/Clock";
6import * as Context from "effect/Context";
7import * as Deferred from "effect/Deferred";
8import type * as Duration from "effect/Duration";
9import * as Effect from "effect/Effect";
10import * as Exit from "effect/Exit";
11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";
13import * as Option from "effect/Option";
14import * as Path from "effect/Path";
15import * as Schema from "effect/Schema";
16import * as Semaphore from "effect/Semaphore";
17import * as Stream from "effect/Stream";
18import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
19import * as NodeCrypto from "node:crypto";
20​
21import * as ServerConfig from "../config.ts";
22import { openZipArchive } from "../zipArchive.ts";
23​
24// Update the version, sizes and hashes from each chrome-headless-shell zip listed at
25// https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json
26// Never pin below 156.0.8078.70: the shell has no CRLSet and older SPKI blocklists trust
27// compromised eMudhra intermediates (crrev.com/c/8627687).
28const VERSION = "157.0.8103.38";
29const ARCHIVES = {
30 linux64: {
31 bytes: 128_168_542,
32 sha256: "deafffa74660b34853a4119c361fd427ac8c9fcb62db51ddb8a797cd83a4ba2d",
33 },
34 "linux-arm64": {
35 bytes: 128_171_139,
36 sha256: "b9b910a57b2a15eb3e55000a2339f93a973934a413ebfc6e585a50ec6c42fd6c",
37 },
38 "mac-arm64": {
39 bytes: 104_773_711,
40 sha256: "e6a6b13551aa12f951d0891d79282606b13742837c8fa0505e1272bae116ce82",
41 },
42 "mac-x64": {
43 bytes: 110_442_262,
44 sha256: "79821745d0a72d2ba752d981284ed0a9e6fa320dad133596941a17dba6186de3",
45 },
46 win32: {
47 bytes: 120_390_416,
48 sha256: "35e0d45c0136448f7b587f6fb7195a0686127a4dfdc23ef3b0e04b5e9ffe16cd",
49 },
50 win64: {
51 bytes: 126_513_060,
52 sha256: "1ff4540855478ba0848e03f1ef8d455f77d058b38285df91bc34273788a73446",
53 },
54} as const;
55​
56const chromePlatform = (platform: NodeJS.Platform, arch: NodeJS.Architecture) => {
57 switch (platform) {
58 case "linux":
59 return arch === "x64" ? "linux64" : arch === "arm64" ? "linux-arm64" : null;
60 case "darwin":
61 return arch === "arm64" ? "mac-arm64" : arch === "x64" ? "mac-x64" : null;
62 case "win32":
63 // There is no Windows arm64 build; Windows on Arm runs the x64 one under emulation.
64 return arch === "ia32" ? "win32" : arch === "x64" || arch === "arm64" ? "win64" : null;
65 default:
66 return null;
67 }
68};
69​
70export interface PreviewBrowserRelease {
71 readonly version: string;
72 readonly platform: string;
73 readonly url: string;
74 readonly bytes: number;
75 readonly sha256: string;
77​
78const previewBrowserRelease = (
79 platform: NodeJS.Platform,
80 arch: NodeJS.Architecture,
81): PreviewBrowserRelease | null => {
82 const chrome = chromePlatform(platform, arch);
83 return chrome === null
84 ? null
85 : {
86 version: VERSION,
87 platform: chrome,
88 url: `https://storage.googleapis.com/chrome-for-testing-public/${VERSION}/${chrome}/chrome-headless-shell-${chrome}.zip`,
89 ...ARCHIVES[chrome],
90 };
91};
92​
93const megabytes = (bytes: number) => Math.round(bytes / 1_000_000);
94​
95export class PreviewBrowserInstallError extends Schema.TaggedError<PreviewBrowserInstallError>()(
96 "PreviewBrowserInstallError",
97 { detail: Schema.String, cause: Schema.optional(Schema.Defect()) },
98) {
99 override get message(): string {
100 return `T3 could not install its headless browser: ${this.detail} Try again.`;
101 }
103const isInstallError = Schema.is(PreviewBrowserInstallError);
104​
105export class PreviewBrowserInstallingError extends Schema.TaggedError<PreviewBrowserInstallingError>()(
106 "PreviewBrowserInstallingError",
107 { downloadedBytes: Schema.Number, totalBytes: Schema.Number, unpacking: Schema.Boolean },
108) {
109 override get message(): string {
110 const progress = this.unpacking
111 ? "unpacking"
112 : `${megabytes(this.downloadedBytes)} of ${megabytes(this.totalBytes)} MB downloaded`;
113 return `T3 is installing its headless browser (${progress}). Try again in a minute.`;
114 }
116​
117export class PreviewBrowserUnsupportedError extends Schema.TaggedError<PreviewBrowserUnsupportedError>()(
118 "PreviewBrowserUnsupportedError",
119 { platform: Schema.String, arch: Schema.String },
120) {
121 override get message(): string {
122 return `T3's headless browser is not available on ${this.platform}-${this.arch}: Chrome for Testing has no headless shell for it.`;
123 }
125​
126export class PreviewBrowser extends Context.Service<
127 PreviewBrowser,
128 {
129 /** Starts installation on first call; returns progress after a bounded wait while installation continues. */
130 readonly executable: Effect.Effect<
131 string,
132 PreviewBrowserInstallError | PreviewBrowserInstallingError | PreviewBrowserUnsupportedError
133 >;
134 readonly installed: Effect.Effect<Option.Option<string>>;
135 }
136>()("t3/preview/PreviewBrowser") {}
137​
138export interface PreviewBrowserOptions {
139 readonly baseDir: string;
140 readonly release?: PreviewBrowserRelease | null;
141 /** How long a caller waits on an install in progress. Provider tool calls time out near 60s. */
142 readonly wait?: Duration.Input;
144​
145const DOWNLOAD_TIMEOUT = "15 minutes";
146const STAGING_STALE_AFTER_MS = 60 * 60_000;
147​
148const wrapFailure = (detail: string) => (cause: unknown) =>
149 isInstallError(cause) ? cause : new PreviewBrowserInstallError({ detail, cause });
150​
151type InstallState =
152 | { readonly _tag: "idle" }
153 | {
154 readonly _tag: "installing";
155 readonly done: Deferred.Deferred<string, PreviewBrowserInstallError>;
156 }
157 | { readonly _tag: "installed"; readonly executable: string }
158 | { readonly _tag: "failed"; readonly error: PreviewBrowserInstallError };
159​
160export const makePreviewBrowser = Effect.fn("PreviewBrowser.make")(function* (
161 options: PreviewBrowserOptions,
162) {
163 const fs = yield* FileSystem.FileSystem;
164 const path = yield* Path.Path;
165 const http = yield* HttpClient.HttpClient;
166 // Installs belong to the service, so they finish even when no caller is still waiting.
167 const serviceScope = yield* Effect.scope;
168 const platform = yield* HostProcess.Platform;
169 const arch = yield* HostProcess.Architecture;
170 const release =
171 options.release === undefined ? previewBrowserRelease(platform, arch) : options.release;
172 const wait = options.wait ?? "45 seconds";
173 const installRoot = path.join(
174 options.baseDir,
175 "tools",
176 "chrome-headless-shell",
177 release?.platform ?? `${platform}-${arch}`,
178 );
179 const executableName =
180 platform === "win32" ? "chrome-headless-shell.exe" : "chrome-headless-shell";
181 const gate = yield* Semaphore.make(1);
182 let state: InstallState = { _tag: "idle" };
183 const progress = { downloadedBytes: 0, unpacking: false };
184​
185 const installedExecutable = (release: PreviewBrowserRelease) => {
186 const executable = path.join(installRoot, release.version, executableName);
187 return fs.stat(executable).pipe(
188 Effect.map((info) =>
189 info.type === "File" && (platform === "win32" || (info.mode & 0o111) !== 0)
190 ? Option.some(executable)
191 : Option.none<string>(),
192 ),
193 Effect.orElseSucceed(() => Option.none<string>()),
194 );
195 };
196​
197 const download = Effect.fn("PreviewBrowser.download")(
198 function* (release: PreviewBrowserRelease, archivePath: string) {
199 const response = yield* http
200 .execute(HttpClientRequest.get(release.url))
201 .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk));
202 const contentLength = response.headers["content-length"];
203 const contentEncoding = response.headers["content-encoding"]?.trim().toLowerCase();
204 if (
205 (contentEncoding === undefined || contentEncoding === "identity") &&
206 contentLength !== undefined &&
207 Number(contentLength) !== release.bytes
208 ) {
209 return yield* new PreviewBrowserInstallError({
210 detail: "The download size did not match the pinned release.",
211 });
212 }
213 const hash = NodeCrypto.createHash("sha256");
214 yield* response.stream.pipe(
215 Stream.tap((chunk) =>
216 Effect.suspend(() => {
217 progress.downloadedBytes += chunk.byteLength;
218 if (progress.downloadedBytes > release.bytes) {
219 return Effect.fail(
220 new PreviewBrowserInstallError({
221 detail: "The download was larger than the pinned release.",
222 }),
223 );
224 }
225 hash.update(chunk);
226 return Effect.void;
227 }),
228 ),
229 Stream.run(fs.sink(archivePath, { flag: "wx", mode: 0o600 })),
230 );
231 if (progress.downloadedBytes !== release.bytes || hash.digest("hex") !== release.sha256) {
232 return yield* new PreviewBrowserInstallError({
233 detail: "The download failed its size or SHA-256 check. Nothing was installed.",
234 });
235 }
236 },
237 Effect.timeout(DOWNLOAD_TIMEOUT),
238 Effect.mapError(
239 wrapFailure(
240 "The download from storage.googleapis.com failed. Check this machine's network access.",
241 ),
242 ),
243 );
244​
245 const extract = Effect.fn("PreviewBrowser.extract")(
246 function* (release: PreviewBrowserRelease, archivePath: string, destination: string) {
247 const archive = yield* openZipArchive(
248 archivePath,
249 (detail, cause) => new PreviewBrowserInstallError({ detail, cause }),
250 );
251 const unsafe = new PreviewBrowserInstallError({
252 detail: "The archive contains an unexpected or unsafe entry.",
253 });
254 const root = `chrome-headless-shell-${release.platform}/`;
255 yield* fs.makeDirectory(destination);
256 let foundExecutable = false;
257 for (;;) {
258 const entry = yield* archive.next;
259 if (!entry) break;
260 const unixMode = entry.externalFileAttributes >>> 16;
261 const unixType = unixMode & 0o170000;
262 const isDirectory = entry.fileName.endsWith("/");
263 const relative = entry.fileName.startsWith(root) ? entry.fileName.slice(root.length) : null;
264 const target = relative === null ? null : path.resolve(destination, relative);
265 if (
266 relative === null ||
267 target === null ||
268 (relative !== "" && !target.startsWith(`${destination}${path.sep}`)) ||
269 relative.split("/").includes("..") ||
270 (unixType !== 0 && unixType !== (isDirectory ? 0o040000 : 0o100000)) ||
271 (entry.generalPurposeBitFlag & 1) !== 0 ||
272 ![0, 8].includes(entry.compressionMethod)
273 ) {
274 return yield* unsafe;
275 }
276 if (isDirectory) {
277 yield* fs.makeDirectory(target, { recursive: true });
278 continue;
279 }
280 yield* fs.makeDirectory(path.dirname(target), { recursive: true });
281 yield* Effect.gen(function* () {
282 const readable = yield* archive.streamEntry(entry);
283 yield* EffectNodeStream.fromReadable<Uint8Array, PreviewBrowserInstallError>({
284 evaluate: () => readable,
285 onError: wrapFailure("Could not unpack the browser."),
286 }).pipe(Stream.run(fs.sink(target, { flag: "wx" })));
287 }).pipe(Effect.scoped);
288 // The binary and its shared libraries carry exec bits in the archive.
289 if (platform !== "win32" && (unixMode & 0o777) !== 0) {
290 yield* fs.chmod(target, unixMode & 0o777);
291 }
292 if (relative === executableName) foundExecutable = true;
293 }
294 if (!foundExecutable) {
295 return yield* new PreviewBrowserInstallError({
296 detail: "The archive does not contain chrome-headless-shell.",
297 });
298 }
299 },
300 Effect.scoped,
301 Effect.mapError(
302 wrapFailure("Could not unpack the browser. Check free disk space in T3's home directory."),
303 ),
304 );
305​
306 const install = Effect.fn("PreviewBrowser.install")(
307 function* (release: PreviewBrowserRelease) {
308 yield* Effect.gen(function* () {
309 yield* fs.makeDirectory(installRoot, { recursive: true });
310 // Staging shares the install root's filesystem so publishing is one rename.
311 const staging = yield* fs.makeTempDirectoryScoped({
312 directory: installRoot,
313 prefix: ".install-",
314 });
315 const archivePath = path.join(staging, "download.zip");
316 const unpacked = path.join(staging, "browser");
317 yield* download(release, archivePath);
318 progress.unpacking = true;
319 yield* extract(release, archivePath, unpacked);
320 const destination = path.join(installRoot, release.version);
321 yield* fs.remove(destination, { recursive: true, force: true });
322 yield* fs.rename(unpacked, destination);
323 }).pipe(Effect.scoped);
324 // Shared homes may still use the newest other version or staging directories touched within the last hour.
325 const entries = yield* fs.readDirectory(installRoot);
326 const previous = entries
327 .filter((name) => name !== release.version && !name.startsWith("."))
328 .toSorted((left, right) => right.localeCompare(left, "en", { numeric: true }))[0];
329 const now = yield* Clock.currentTimeMillis;
330 yield* Effect.forEach(
331 entries.filter((name) => name !== release.version && name !== previous),
332 (name) =>
333 Effect.gen(function* () {
334 const target = path.join(installRoot, name);
335 if (name.startsWith(".install-")) {
336 const info = yield* fs.stat(target);
337 const modifiedAt = Option.match(info.mtime, {
338 onNone: () => now,
339 onSome: (date) => date.getTime(),
340 });
341 if (now - modifiedAt < STAGING_STALE_AFTER_MS) return;
342 }
343 yield* fs.remove(target, { recursive: true, force: true });
344 }).pipe(
345 Effect.catch((cause) =>
346 Effect.logWarning("Could not remove an old preview browser.", { name, cause }),
347 ),
348 ),
349 { discard: true },
350 );
351 return path.join(installRoot, release.version, executableName);
352 },
353 Effect.mapError(wrapFailure("Could not save the browser in T3's home directory.")),
354 );
355​
356 const join = (release: PreviewBrowserRelease) =>
357 gate.withPermit(
358 Effect.gen(function* () {
359 if (state._tag === "installing") return state;
360 if (state._tag === "failed") {
361 const failed = state;
362 state = { _tag: "idle" };
363 return failed;
364 }
365 const installed = yield* installedExecutable(release);
366 if (Option.isSome(installed)) {
367 state = { _tag: "installed", executable: installed.value };
368 return state;
369 }
370 const done = yield* Deferred.make<string, PreviewBrowserInstallError>();
371 progress.downloadedBytes = 0;
372 progress.unpacking = false;
373 const installing: InstallState = { _tag: "installing", done };
374 state = installing;
375 yield* install(release).pipe(
376 Effect.onExit((exit) =>
377 gate
378 .withPermit(
379 Effect.sync(() => {
380 state = Exit.isSuccess(exit)
381 ? { _tag: "installed", executable: exit.value }
382 : {
383 _tag: "failed",
384 error: Option.getOrElse(
385 Cause.findErrorOption(exit.cause),
386 () => new PreviewBrowserInstallError({ detail: "The install stopped." }),
387 ),
388 };
389 }),
390 )
391 .pipe(Effect.andThen(Deferred.done(done, exit))),
392 ),
393 Effect.ignoreCause,
394 Effect.interruptible,
395 Effect.forkIn(serviceScope),
396 );
397 return installing;
398 }),
399 );
400​
401 const clearFailure = (error: PreviewBrowserInstallError) =>
402 gate.withPermit(
403 Effect.sync(() => {
404 if (state._tag === "failed" && state.error === error) state = { _tag: "idle" };
405 }),
406 );
407​
408 const executable = Effect.gen(function* () {
409 if (release === null) {
410 return yield* new PreviewBrowserUnsupportedError({ platform, arch });
411 }
412 const current = yield* join(release);
413 switch (current._tag) {
414 case "installed":
415 return current.executable;
416 case "failed":
417 return yield* current.error;
418 case "installing":
419 return yield* Deferred.await(current.done).pipe(
420 Effect.tapError(clearFailure),
421 Effect.timeoutOrElse({
422 duration: wait,
423 orElse: () =>
424 Effect.fail(
425 new PreviewBrowserInstallingError({
426 downloadedBytes: progress.downloadedBytes,
427 totalBytes: release.bytes,
428 unpacking: progress.unpacking,
429 }),
430 ),
431 }),
432 );
433 }
434 }).pipe(Effect.withSpan("PreviewBrowser.executable"));
435​
436 const installed = release === null ? Effect.succeedNone : installedExecutable(release);
437​
438 return PreviewBrowser.of({ executable, installed });
439});
440​
441export const layer = Layer.effect(
442 PreviewBrowser,
443 Effect.gen(function* () {
444 const config = yield* ServerConfig.ServerConfig;
445 return yield* makePreviewBrowser({ baseDir: config.baseDir });
446 }),
447);