Open History to see the last commit
1#!/usr/bin/env node
2// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off globalFetch:off - Host-side device automation uses Node subprocess and filesystem APIs directly.
3
4import * as NodeChildProcess from "node:child_process";
5import * as NodeFS from "node:fs";
6import * as NodeFSP from "node:fs/promises";
7import * as NodeOS from "node:os";
8import * as NodePath from "node:path";
9import * as NodeProcess from "node:process";
10import * as NodeReadline from "node:readline";
11import * as NodeURL from "node:url";
12
13import { CASES } from "./mobile-behavior/cases.ts";
14import { blockingResults, resultsFor, summarize, type SampleRow } from "./mobile-behavior/grade.ts";
15import { Device, type Pool, poolError, run } from "./mobile-e2e/device.ts";
16import type { Pose } from "./mobile-e2e/poses.ts";
17
18const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "..");
19
20/** The devices the behavior cases run on; each runs in parallel with its own server and Metro. */
21const TARGETS = [
22 { id: "iphone-18-pro", device: "iPhone 18 Pro", form: "phone", behaviorPose: "phone" },
23 { id: "ipad-pro-13", device: "iPad Pro 13-inch (M5)", form: "tablet", behaviorPose: "ipad" },
24] as const;
25type Target = (typeof TARGETS)[number];
26
27const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
28
29interface TargetRun {
30 readonly target: Target;
31 readonly directory: string;
32 readonly error?: string;
33}
34
35/** Keeps a lease alive while the cases run: they go minutes without an agent-device call. */
36function renewWhileRunning(pool: Pool, stateDir: string, session: string, token: string) {
37 const gateway = pool.agentDeviceUrl.replace(/\/v1\/agent-device\/?$/, "");
38 const timer = setInterval(() => {
39 const record = NodePath.join(stateDir, "remote-connections", `${session}.json`);
40 void NodeFSP.readFile(record, "utf8")
41 .then((text) => (JSON.parse(text) as { leaseId: string }).leaseId)
42 .then((leaseId) =>
43 fetch(`${gateway}/v1/leases/${encodeURIComponent(leaseId)}/renew`, {
44 method: "POST",
45 headers: { authorization: `Bearer ${token}` },
46 }),
47 )
48 .catch(() => undefined);
49 }, 90_000);
50 return () => clearInterval(timer);
51}
52
53async function runTarget(
54 target: Target,
55 pool: Pool,
56 runId: string,
57 outputDirectory: string,
58 config: string,
59 stateDir: string,
60 token: string,
61 passThrough: ReadonlyArray<string>,
62): Promise<TargetRun> {
63 const directory = NodePath.join(outputDirectory, target.id);
64 await NodeFSP.mkdir(directory, { recursive: true });
65 const pose: Pose = {
66 id: target.id,
67 platform: "ios",
68 device: target.device,
69 form: target.form,
70 appearance: "light",
71 };
72 let device: Device | null = null;
73 for (let attempt = 1; !device; attempt++) {
74 try {
75 log(target.id, `leasing ${target.device}`);
76 device = await Device.lease(pose, pool, attempt === 1 ? runId : `${runId}-retry`);
77 log(target.id, "installing the dev client");
78 await device.install();
79 } catch (error) {
80 const message = error instanceof Error ? error.message : String(error);
81 await device?.release();
82 device = null;
83 if (attempt >= 2 || !poolError(message)) return { target, directory, error: message };
84 log(target.id, `retrying after: ${message.split("\n")[0]}`);
85 }
86 }
87 const stopRenewing = renewWhileRunning(pool, stateDir, device.session, token);
88 try {
89 log(target.id, "running the behavior cases");
90 const child = NodeChildProcess.spawn(
91 NodeProcess.execPath,
92 [
93 "scripts/mobile-behavior/run.ts",
94 "--out",
95 directory,
96 "--label",
97 target.id,
98 "--pose",
99 target.behaviorPose,
100 "--device-config",
101 config,
102 "--device-session",
103 device.session,
104 "--agent-device",
105 pool.agentDevice,
106 ...passThrough,
107 ],
108 { cwd: REPO_ROOT, env: NodeProcess.env, stdio: ["ignore", "pipe", "pipe"] },
109 );
110 const transcript = NodeFS.createWriteStream(NodePath.join(directory, "run.log"));
111 // The scoreboard and any fatal error go to stdout, progress to stderr; the transcript keeps both.
112 NodeReadline.createInterface({ input: child.stdout }).on("line", (line) =>
113 transcript.write(`${line}\n`),
114 );
115 NodeReadline.createInterface({ input: child.stderr }).on("line", (line) => {
116 transcript.write(`${line}\n`);
117 if (/#\d+: /.test(line)) log(target.id, line);
118 });
119 const code = await new Promise<number>((resolve) =>
120 child.on("close", (exit) => resolve(exit ?? 1)),
121 );
122 transcript.end();
123 const hasResults = NodeFS.existsSync(NodePath.join(directory, "results.jsonl"));
124 return hasResults || code === 0
125 ? { target, directory }
126 : { target, directory, error: `the cases stopped (exit ${code}); see ${directory}/run.log` };
127 } finally {
128 stopRenewing();
129 await device.release();
130 }
131}
132
133async function main() {
134 const args = NodeProcess.argv.slice(2);
135 const values = (flag: string) =>
136 args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : []));
137 if (args.includes("--list")) {
138 for (const target of TARGETS) NodeProcess.stdout.write(`${target.id}\t${target.device}\n`);
139 for (const testCase of CASES)
140 NodeProcess.stdout.write(`${testCase.id}\t${testCase.kind}\t${testCase.source}\n`);
141 return;
142 }
143 const selected = values("--device");
144 const unknown = selected.filter((id) => !TARGETS.some((target) => target.id === id));
145 if (unknown.length) throw new Error(`Unknown device ${unknown.join(", ")}. Run with --list.`);
146 const targets = TARGETS.filter((target) => selected.length === 0 || selected.includes(target.id));
147 const passThrough = [
148 ...values("--samples").flatMap((value) => ["--samples", value]),
149 ...values("--case").flatMap((value) => ["--case", value]),
150 ];
151
152 const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL;
153 const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN;
154 if (!agentDeviceUrl || !token)
155 throw new Error(
156 "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.",
157 );
158
159 const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
160 const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-behavior", runId);
161 await NodeFSP.mkdir(outputDirectory, { recursive: true });
162 const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-behavior-"));
163 const workspace = await NodeFSP.realpath(outputDirectory);
164 const config = NodePath.join(rootDir, "agent-device.json");
165 const stateDir = NodePath.join(rootDir, "agent-device");
166 await NodeFSP.writeFile(
167 config,
168 JSON.stringify({ daemonBaseUrl: agentDeviceUrl, daemonAuthToken: token, stateDir }),
169 { mode: 0o600 },
170 );
171 const pool: Pool = {
172 repoRoot: REPO_ROOT,
173 agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
174 agentDeviceUrl,
175 workspace,
176 env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace },
177 };
178 try {
179 // A cache miss builds the dev client, which can outlast a lease, so it is ready first.
180 log("behavior", "preparing the ios dev client");
181 await run(
182 NodeProcess.execPath,
183 ["scripts/mobile-native-client.ts", "ensure", "ios", "--prepare"],
184 { cwd: REPO_ROOT, timeoutMs: 1_800_000 },
185 );
186 const runs = await Promise.all(
187 targets.map((target) =>
188 runTarget(target, pool, runId, outputDirectory, config, stateDir, token, passThrough),
189 ),
190 );
191
192 const rows = runs.flatMap((entry) => {
193 const file = NodePath.join(entry.directory, "results.jsonl");
194 if (!NodeFS.existsSync(file)) return [];
195 return NodeFS.readFileSync(file, "utf8")
196 .split("\n")
197 .filter(Boolean)
198 .map((line) => JSON.parse(line) as SampleRow);
199 });
200 const results = resultsFor(rows, CASES);
201 const stopped = runs.filter((entry) => entry.error);
202 const summary = summarize({
203 cases: CASES,
204 results,
205 header: [
206 `# Mobile behavior · ${runId}`,
207 "",
208 "Cells show pass or FAIL, passed/samples, and the key metric's median. `bad` cases reproduce known bugs and are expected to fail until they are fixed.",
209 ...stopped.map((entry) => `\n**${entry.target.id} did not finish:** ${entry.error}`),
210 ].join("\n"),
211 });
212 const summaryFile = NodePath.join(outputDirectory, "summary.md");
213 await NodeFSP.writeFile(summaryFile, summary);
214 const blocking = blockingResults(results, CASES);
215 for (const result of blocking)
216 log(result.device, `${result.id}: ${result.pass === null ? "incomplete" : "FAIL"}`);
217 for (const entry of stopped) log(entry.target.id, entry.error!);
218 NodeProcess.stdout.write(
219 `${JSON.stringify({ summary: summaryFile, cases: results.length, blocking: blocking.length, stopped: stopped.length })}\n`,
220 );
221 process.exitCode = blocking.length || stopped.length ? 1 : 0;
222 } finally {
223 await NodeFSP.rm(rootDir, { recursive: true, force: true });
224 }
225}
226
227if (import.meta.main) {
228 await main();
229}