Open History to see the last commit
1#!/usr/bin/env node
2// @effect-diagnostics nodeBuiltinImport:off globalDate:off - Host-side device automation uses Node subprocess and filesystem APIs directly.
3
4import * as NodeChildProcess from "node:child_process";
5import * as NodeFS from "node:fs";
6import * as NodeFSP from "node:fs/promises";
7import * as NodeOS from "node:os";
8import * as NodePath from "node:path";
9import * as NodeProcess from "node:process";
10import * as NodeURL from "node:url";
11
12import { Device, type Pool, poolError, run } from "./mobile-e2e/device.ts";
13import { runJourney, type StepResult } from "./mobile-e2e/journey.ts";
14import { POSES, type Pose } from "./mobile-e2e/poses.ts";
15import {
16 failedChecks,
17 poolBlocked,
18 poseFailed,
19 type PoseResult,
20 renderReport,
21} from "./mobile-e2e/report.ts";
22import { reviewPose } from "./mobile-e2e/review.ts";
23import {
24 reserveAvailablePort,
25 startShowcaseEnvironments,
26 stopProcess,
27 waitForPort,
28} from "./mobile-showcase.ts";
29import { SHOWCASE_ENVIRONMENTS } from "./mobile-showcase-environment.ts";
30
31const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "..");
32
33const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
34
35/** This machine's Tailscale address (100.64.0.0/10), which pool devices reach it on. */
36function tailnetAddress(): string {
37 for (const addresses of Object.values(NodeOS.networkInterfaces())) {
38 for (const address of addresses ?? []) {
39 const [first, second] = address.address.split(".").map(Number);
40 if (address.family === "IPv4" && first === 100 && second! >= 64 && second! < 128)
41 return address.address;
42 }
43 }
44 throw new Error("No Tailscale IPv4 address: pool devices reach Metro and the fixture over it.");
45}
46
47async function mapConcurrent<T, R>(
48 items: ReadonlyArray<T>,
49 limit: number,
50 task: (item: T) => Promise<R>,
51): Promise<R[]> {
52 const results: R[] = Array.from({ length: items.length });
53 let next = 0;
54 await Promise.all(
55 Array.from({ length: Math.min(limit, items.length) }, async () => {
56 while (next < items.length) {
57 const index = next++;
58 results[index] = await task(items[index]!);
59 }
60 }),
61 );
62 return results;
63}
64
65async function issuePairUrl(
66 environment: { readonly baseDir: string; readonly port: number },
67 host: string,
68) {
69 const { stdout } = await run(
70 NodeProcess.execPath,
71 [
72 "apps/server/src/bin.ts",
73 "auth",
74 "pairing",
75 "create",
76 "--base-dir",
77 environment.baseDir,
78 "--base-url",
79 `http://${host}:${environment.port}`,
80 "--ttl",
81 "15m",
82 "--json",
83 ],
84 { cwd: REPO_ROOT },
85 );
86 const { pairUrl } = JSON.parse(
87 stdout.slice(stdout.indexOf("{"), stdout.lastIndexOf("}") + 1),
88 ) as {
89 pairUrl?: string;
90 };
91 if (!pairUrl) throw new Error("The pairing command returned no pairUrl.");
92 return pairUrl;
93}
94
95async function runPose(
96 pose: Pose,
97 pool: Pool,
98 runId: string,
99 outputDirectory: string,
100 metroUrl: string,
101 pair: () => Promise<string>,
102): Promise<PoseResult> {
103 const directory = NodePath.join(outputDirectory, pose.id);
104 await NodeFSP.mkdir(directory, { recursive: true });
105 const steps: StepResult[] = [];
106 let device: Device | null = null;
107 try {
108 log(pose.id, `leasing ${pose.device}`);
109 device = await Device.lease(pose, pool, runId);
110 log(pose.id, "installing the dev client");
111 await device.install();
112 await device.prepare();
113 await device.launch(metroUrl);
114 log(pose.id, "pairing and walking the app");
115 await runJourney(device, directory, pair, steps);
116 return { pose, steps };
117 } catch (error) {
118 const message = error instanceof Error ? error.message : String(error);
119 if (!device) return { pose, steps, error: message };
120 // What the device showed when the journey stopped is usually the whole diagnosis.
121 const stopped = NodePath.join(directory, "stopped.png");
122 const shot = await device.screenshot(stopped).then(
123 () => stopped,
124 () => undefined,
125 );
126 const labels = await device.snapshot().then(
127 (snapshot) =>
128 snapshot.nodes
129 .map((node) => node.label)
130 .filter(Boolean)
131 .slice(0, 12)
132 .join(" | "),
133 () => "",
134 );
135 return {
136 pose,
137 steps,
138 error: labels ? `${message} (screen: ${labels})` : message,
139 ...(shot ? { stopped: shot } : {}),
140 };
141 } finally {
142 await device?.release();
143 }
144}
145
146async function main() {
147 const args = NodeProcess.argv.slice(2);
148 if (args.includes("--list")) {
149 for (const pose of POSES)
150 NodeProcess.stdout.write(
151 `${pose.id}\t${pose.device}${pose.pending ? `\t(pending: ${pose.pending})` : ""}\n`,
152 );
153 return;
154 }
155 const values = (flag: string) =>
156 args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : []));
157 const selected = values("--pose");
158 const unknown = selected.filter((id) => !POSES.some((pose) => pose.id === id));
159 if (unknown.length) throw new Error(`Unknown pose ${unknown.join(", ")}. Run with --list.`);
160 const platforms = values("--platform");
161 const poses = POSES.filter(
162 (pose) =>
163 (selected.length === 0 ? !pose.pending : selected.includes(pose.id)) &&
164 (platforms.length === 0 || platforms.includes(pose.platform)),
165 );
166 const concurrencyFlag = values("--concurrency")[0];
167 const concurrency = concurrencyFlag ? Number(concurrencyFlag) : undefined;
168 const review = !args.includes("--no-review");
169
170 const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL;
171 const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN;
172 if (!agentDeviceUrl || !token)
173 throw new Error(
174 "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.",
175 );
176 const host = tailnetAddress();
177
178 const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
179 const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-e2e", runId);
180 await NodeFSP.mkdir(outputDirectory, { recursive: true });
181 const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-e2e-"));
182 const workspace = await NodeFSP.realpath(outputDirectory);
183 const config = NodePath.join(rootDir, "agent-device.json");
184 await NodeFSP.writeFile(
185 config,
186 JSON.stringify({
187 daemonBaseUrl: agentDeviceUrl,
188 daemonAuthToken: token,
189 stateDir: NodePath.join(rootDir, "agent-device"),
190 }),
191 { mode: 0o600 },
192 );
193 const pool: Pool = {
194 repoRoot: REPO_ROOT,
195 agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
196 agentDeviceUrl,
197 workspace,
198 env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace },
199 };
200
201 const servers: NodeChildProcess.ChildProcess[] = [];
202 let metro: NodeChildProcess.ChildProcess | null = null;
203 try {
204 log("e2e", "starting the fixture server");
205 const [environment] = await startShowcaseEnvironments(
206 rootDir,
207 (server) => servers.push(server),
208 SHOWCASE_ENVIRONMENTS.slice(0, 1),
209 );
210 if (!environment) throw new Error("The fixture server did not start.");
211 const metroPort = await reserveAvailablePort();
212 metro = NodeChildProcess.spawn(
213 "vp",
214 [
215 "exec",
216 "expo",
217 "start",
218 "--dev-client",
219 "--scheme",
220 "t3code-dev",
221 "--lan",
222 "--port",
223 String(metroPort),
224 ],
225 {
226 cwd: NodePath.join(REPO_ROOT, "apps/mobile"),
227 // Lazily fetched chunks (dynamic import()) sometimes arrive broken on the pool's Android
228 // emulators and throw SyntaxErrors; release builds have no lazy chunks either.
229 env: { ...NodeProcess.env, APP_VARIANT: "development", CI: "1", EXPO_NO_METRO_LAZY: "1" },
230 // The app's console output arrives through Metro; keep it with the run.
231 stdio: [
232 "ignore",
233 NodeFS.openSync(NodePath.join(outputDirectory, "metro.log"), "w"),
234 "inherit",
235 ],
236 // Its own group, so stopping it also stops the expo child that owns the port.
237 detached: true,
238 },
239 );
240 await waitForPort(metroPort, "Metro", 120_000);
241 // Warm each platform's bundle once, instead of on every device at once, and have its dev
242 // client cached before any lease: a cache miss builds, which can outlast a lease.
243 for (const platform of new Set(poses.map((pose) => pose.platform))) {
244 log("e2e", `preparing the ${platform} dev client`);
245 await run(
246 NodeProcess.execPath,
247 ["scripts/mobile-native-client.ts", "ensure", platform, "--prepare"],
248 { cwd: REPO_ROOT, timeoutMs: 1_800_000 },
249 );
250 log("e2e", `bundling for ${platform}`);
251 await run("curl", [
252 "-sf",
253 "-o",
254 "/dev/null",
255 `http://127.0.0.1:${metroPort}/apps/mobile/index.bundle?platform=${platform}&dev=true&minify=false`,
256 ]);
257 }
258
259 const attempt = (pose: Pose, session: string) =>
260 runPose(pose, pool, session, outputDirectory, `http://${host}:${metroPort}`, () =>
261 issuePairUrl(environment, host),
262 );
263 const runOne = async (pose: Pose) => {
264 let result = await attempt(pose, runId);
265 // A pose that failed before its first screen usually met the pool, not the app.
266 for (
267 let retry = 1;
268 retry <= 2 && result.error && (result.steps.length === 0 || poolError(result.error));
269 retry++
270 ) {
271 log(pose.id, `retrying after: ${result.error.split("\n")[0]}`);
272 result = await attempt(pose, `${runId}-retry${retry}`);
273 }
274 return result;
275 };
276 // Each platform has its own workers, so each gets its own limit; one Mac runs every
277 // iOS simulator, and recording several at once on a shared pool times it out.
278 const byPlatform = await Promise.all(
279 (["ios", "android"] as const).map((platform) =>
280 mapConcurrent(
281 poses.filter((pose) => pose.platform === platform),
282 concurrency ?? (platform === "ios" ? 2 : 3),
283 runOne,
284 ),
285 ),
286 );
287 const results = poses.map((pose) => byPlatform.flat().find((result) => result.pose === pose)!);
288
289 const reviewed = review
290 ? await mapConcurrent(results, 4, async (result) => {
291 log(result.pose.id, "agent review");
292 const findings = await reviewPose(result, outputDirectory).catch((error) => {
293 log(result.pose.id, `review failed: ${error instanceof Error ? error.message : error}`);
294 return [];
295 });
296 return { ...result, findings };
297 })
298 : results;
299
300 const report = NodePath.join(outputDirectory, "index.html");
301 await NodeFSP.writeFile(report, renderReport(reviewed, outputDirectory));
302 await NodeFSP.writeFile(
303 NodePath.join(outputDirectory, "report.json"),
304 JSON.stringify(reviewed, null, 2),
305 );
306 for (const result of reviewed) {
307 const failures = failedChecks(result).map(
308 (item) => `${item.step}: ${item.name} (${item.detail})`,
309 );
310 log(
311 result.pose.id,
312 [poolBlocked(result) ? `blocked by the pool: ${result.error}` : result.error, ...failures]
313 .filter(Boolean)
314 .join("; ") ||
315 `passed${result.findings?.length ? `, ${result.findings.length} review findings` : ""}`,
316 );
317 }
318 const failed = reviewed.filter((result) => poseFailed(result) && !poolBlocked(result)).length;
319 const blocked = reviewed.filter(poolBlocked).length;
320 NodeProcess.stdout.write(
321 `${JSON.stringify({
322 report,
323 poses: reviewed.length,
324 failed,
325 blocked,
326 findings: reviewed.reduce((count, result) => count + (result.findings?.length ?? 0), 0),
327 })}\n`,
328 );
329 // A pose the pool blocked was not tested, so it fails the run too.
330 process.exitCode = failed || blocked ? 1 : 0;
331 } finally {
332 if (metro?.pid) {
333 try {
334 NodeProcess.kill(-metro.pid, "SIGTERM");
335 } catch {}
336 await stopProcess(metro);
337 }
338 await Promise.all(servers.map((server) => stopProcess(server)));
339 await NodeFSP.rm(rootDir, { recursive: true, force: true });
340 }
341}
342
343if (import.meta.main) {
344 await main();
345}