Open History to see the last commit
1#!/usr/bin/env node
2// @effect-diagnostics nodeBuiltinImport:off globalJsonInEffect:off - owns Metro and writes result files.
3/**
4 * Drives the T3 Code iOS dev client on one simulator against an isolated server with scripted
5 * replies, and grades app behavior from the dev-only probe and server state. See README.md.
6 */
7import * as NodeChildProcess from "node:child_process";
8import * as NodeFS from "node:fs";
9import * as NodeOS from "node:os";
10import * as NodePath from "node:path";
11import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
12import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
13import * as NodeServices from "@effect/platform-node/NodeServices";
14import * as Clock from "effect/Clock";
15import * as Console from "effect/Console";
16import * as Effect from "effect/Effect";
17import * as Option from "effect/Option";
18import { Command, Flag } from "effect/cli";
19import { FetchHttpClient, HttpClient } from "effect/http";
20
21import { CASES, type MobileBehaviorCase } from "./cases.ts";
22import { APP_ID, DeviceError, makeDevice, SCHEME } from "./device.ts";
23import { freePort, startEnvironment, uuid } from "./environment.ts";
24import { blockingResults, checkFails, resultsFor, summarize, type SampleRow } from "./grade.ts";
25import { SCENARIOS, type Pose, type ScenarioContext } from "./scenarios.ts";
26
27const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../..");
28const POSES = ["phone", "duo-closed", "duo-open", "ipad"] as const satisfies ReadonlyArray<Pose>;
29
30/** This machine's address as the simulator sees it: the tailnet address, else the first LAN IPv4. */
31function reachableHost(): string {
32 const tailnet = NodeChildProcess.spawnSync("tailscale", ["ip", "-4"], { encoding: "utf8" });
33 const ip = tailnet.status === 0 ? tailnet.stdout.trim().split("\n")[0] : undefined;
34 if (ip) return ip;
35 const lan = Object.values(NodeOS.networkInterfaces())
36 .flatMap((entries) => entries ?? [])
37 .find((entry) => entry.family === "IPv4" && !entry.internal);
38 return lan?.address ?? "127.0.0.1";
39}
40
41/** Metro with the behavior probe compiled in, stopped with its process group when the scope ends. */
42const startMetro = Effect.fn("run.startMetro")(function* (host: string, out: string) {
43 const port = yield* freePort();
44 const log = NodeFS.openSync(NodePath.join(out, "metro.log"), "w");
45 yield* Effect.acquireRelease(
46 Effect.sync(() =>
47 NodeChildProcess.spawn(
48 "npx",
49 ["expo", "start", "--dev-client", "--scheme", SCHEME, "--lan", "--port", String(port)],
50 {
51 cwd: NodePath.join(REPO_ROOT, "apps/mobile"),
52 detached: true,
53 stdio: ["ignore", log, log],
54 env: {
55 ...process.env,
56 APP_VARIANT: "development",
57 CI: "1",
58 // Lazily fetched chunks sometimes arrive broken on pool emulators and throw SyntaxErrors.
59 EXPO_NO_METRO_LAZY: "1",
60 EXPO_PUBLIC_T3_BEHAVIOR_PROBE: "1",
61 REACT_NATIVE_PACKAGER_HOSTNAME: host,
62 },
63 },
64 ),
65 ),
66 (child) => Effect.sync(() => child.pid !== undefined && process.kill(-child.pid, "SIGTERM")),
67 );
68 const metro = `http://${host}:${port}`;
69 for (let attempt = 0; attempt < 120; attempt++) {
70 const ready = yield* HttpClient.get(`${metro}/status`).pipe(
71 Effect.flatMap((response) => response.text),
72 Effect.provide(FetchHttpClient.layer),
73 Effect.orElseSucceed(() => ""),
74 );
75 if (ready.includes("running")) return metro;
76 yield* Effect.sleep("1 second");
77 }
78 return yield* new DeviceError({ message: `Metro never started; see ${out}/metro.log.` });
79});
80
81const decodeRows = (file: string): ReadonlyArray<SampleRow> =>
82 NodeFS.readFileSync(file, "utf8")
83 .split("\n")
84 .filter(Boolean)
85 .map((line) => JSON.parse(line) as SampleRow);
86
87const command = Command.make(
88 "mobile-behavior",
89 {
90 out: Flag.String("out").pipe(Flag.withDescription("Directory for results and run state.")),
91 device: Flag.String("device").pipe(
92 Flag.withDescription(
93 "Simulator UDID; leave out for a Simlock lease, whose gateway picks it.",
94 ),
95 Flag.withDefault(""),
96 ),
97 deviceConfig: Flag.String("device-config").pipe(
98 Flag.withDescription("The --config path device_open returned for this device."),
99 ),
100 deviceSession: Flag.String("device-session").pipe(
101 Flag.withDescription("The --session name device_open returned for this device."),
102 ),
103 deviceName: Flag.String("device-name").pipe(
104 Flag.withDescription(
105 'Simulator name as Metro lists it, e.g. "iPhone 17 Pro". Leave out when this run\'s Metro serves one device.',
106 ),
107 Flag.withDefault(""),
108 ),
109 pose: Flag.Literals("pose", POSES).pipe(Flag.withDefault<Pose>("phone")),
110 label: Flag.String("label").pipe(
111 Flag.withDescription("Column name in the scoreboard; defaults to the pose."),
112 Flag.optional,
113 ),
114 cases: Flag.String("case").pipe(
115 Flag.withDescription("Comma-separated case ids."),
116 Flag.optional,
117 ),
118 samples: Flag.Int("samples").pipe(Flag.withDefault(3)),
119 baseline: Flag.String("baseline").pipe(
120 Flag.withDescription("results.jsonl from an earlier run to compare against."),
121 Flag.optional,
122 ),
123 agentDevice: Flag.String("agent-device").pipe(
124 Flag.withDefault(
125 NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
126 ),
127 ),
128 forgetOtherEnvironments: Flag.Boolean("forget-other-environments").pipe(
129 Flag.withDescription(
130 "Remove the app's other saved environments first. Only for a simulator kept for this eval.",
131 ),
132 Flag.withDefault(false),
133 ),
134 host: Flag.String("host").pipe(
135 Flag.withDescription(
136 "Address the simulator uses to reach this machine; defaults to the tailnet IP.",
137 ),
138 Flag.optional,
139 ),
140 },
141 Effect.fn("mobileBehavior.main")(function* (flags) {
142 const out = NodePath.resolve(flags.out);
143 // Read first: the baseline may be this --out's own results.jsonl, which the run rewrites.
144 const baseline = Option.map(flags.baseline, (file) => resultsFor(decodeRows(file), CASES));
145 NodeFS.mkdirSync(out, { recursive: true });
146 const label = Option.getOrElse(flags.label, () => flags.pose);
147 const wanted = Option.map(flags.cases, (ids) => new Set(ids.split(",").map((id) => id.trim())));
148 const selected = CASES.filter(
149 (testCase) =>
150 Option.match(wanted, { onNone: () => true, onSome: (ids) => ids.has(testCase.id) }) &&
151 (testCase.poses?.includes(flags.pose) ?? true),
152 );
153 const unknown = Option.match(wanted, {
154 onNone: () => [],
155 onSome: (ids) => [...ids].filter((id) => !CASES.some((testCase) => testCase.id === id)),
156 });
157 if (unknown.length > 0)
158 return yield* new DeviceError({ message: `Unknown case id(s): ${unknown.join(", ")}.` });
159 if (selected.length === 0) return yield* new DeviceError({ message: "No cases match." });
160 const host = Option.getOrElse(flags.host, reachableHost);
161 const startedAt = yield* Clock.currentTimeMillis;
162
163 const env = yield* startEnvironment({ baseDir: NodePath.join(out, "env"), deviceHost: host });
164 const tag = (yield* uuid).slice(0, 4);
165 yield* Console.error(`Seeding a 60-turn thread on ${env.origin}...`);
166 const longThread = yield* env.longThread(`Long settled thread ${tag}`, 60);
167
168 const metro = yield* startMetro(host, out);
169 const device = makeDevice({
170 agentDevice: flags.agentDevice,
171 targetArgs: [
172 "--platform",
173 "ios",
174 ...(flags.device ? ["--udid", flags.device] : []),
175 "--config",
176 flags.deviceConfig,
177 "--session",
178 flags.deviceSession,
179 ],
180 metro,
181 deviceName: flags.deviceName,
182 });
183 yield* Console.error(`Opening the dev client against ${metro}...`);
184 // agent-device answers iOS's "Open in" prompt; the flags keep dev-menu chrome off the app.
185 const metroUrl = new URL(metro);
186 const launchUrl = `${SCHEME}://expo-development-client/?url=${encodeURIComponent(metro)}&disableOnboarding=1&__expo_disable_fab=1&__expo_disable_auto_launch=1`;
187 yield* device
188 .run(
189 [
190 "open",
191 APP_ID,
192 "--relaunch",
193 "--metro-host",
194 metroUrl.hostname,
195 "--metro-port",
196 metroUrl.port,
197 "--launch-url",
198 launchUrl,
199 ],
200 240_000,
201 )
202 .pipe(Effect.retry({ times: 2 }));
203 for (let attempt = 0; ; attempt++) {
204 const loaded = yield* device
205 .evaluate("typeof __t3Behavior")
206 .pipe(Effect.orElseSucceed(() => ""));
207 if (loaded === "object") break;
208 // iOS asks before a URL opens another app; busy hosts can miss the first accept.
209 if (attempt % 5 === 0) yield* device.acceptOpenPrompt.pipe(Effect.ignore);
210 if (attempt > 120)
211 return yield* new DeviceError({
212 message:
213 "The probe never loaded. Is this a dev client built from this checkout's native fingerprint?",
214 });
215 yield* Effect.sleep("2 seconds");
216 }
217 if (flags.pose === "duo-closed" || flags.pose === "duo-open")
218 yield* device.fold(flags.pose === "duo-closed" ? "closed" : "open");
219 const forget = (ids: ReadonlyArray<string>) =>
220 device
221 .evaluate(`__t3Behavior.forgetEnvironments(${JSON.stringify(ids)})`)
222 .pipe(Effect.andThen(Effect.sleep("3 seconds")));
223 const pairing = yield* env.pairingUrl();
224 yield* device.openUrl(
225 `${SCHEME}://connections/new?pairingUrl=${encodeURIComponent(pairing)}&autoConnect=1`,
226 );
227 // Registered once pairing starts, so a stopped run never leaves its environment in the app.
228 yield* Effect.addFinalizer(() => forget([env.environmentId]).pipe(Effect.ignore));
229 const seeded = (state: { homeList: unknown; threads: Record<string, string> }) =>
230 state.homeList !== null && longThread in state.threads;
231 yield* device.waitFor("the paired environment's threads on Home", seeded, 60_000);
232 // Other saved environments keep cached threads (some stuck working) that skew status cases.
233 const others = (yield* device.state).environments.filter((id) => id !== env.environmentId);
234 if (others.length > 0 && !flags.forgetOtherEnvironments)
235 return yield* new DeviceError({
236 message: `The app has ${others.length} other saved environment(s). Use a simulator with no other connections, or pass --forget-other-environments to remove them.`,
237 });
238 if (others.length > 0) yield* forget(others);
239 const ctx: ScenarioContext = {
240 env,
241 device,
242 pose: flags.pose,
243 fixtures: { longThread },
244 };
245 const scenarios = [...new Set(selected.map((testCase) => testCase.scenario))];
246 const rows: SampleRow[] = [];
247 const resultsFile = NodeFS.openSync(NodePath.join(out, "results.jsonl"), "w");
248 for (const scenario of scenarios) {
249 const cases = selected.filter((testCase) => testCase.scenario === scenario);
250 for (let sample = 1; sample <= flags.samples; sample++) {
251 const outcome = yield* env.interruptAll().pipe(
252 Effect.andThen(env.archiveAllExcept(longThread)),
253 Effect.andThen(device.reload(seeded)),
254 Effect.andThen(SCENARIOS[scenario](ctx)),
255 Effect.map((metrics) => ({ metrics, error: undefined })),
256 Effect.catch((error) => Effect.succeed({ metrics: {}, error: error.message })),
257 // A bad probe reply or snapshot must cost one sample, not the run.
258 Effect.catchDefect((defect) => Effect.succeed({ metrics: {}, error: String(defect) })),
259 );
260 for (const testCase of cases) {
261 const failures = outcome.error
262 ? []
263 : testCase.checks.flatMap((check) => checkFails(check, outcome.metrics) ?? []);
264 const row: SampleRow = {
265 id: testCase.id,
266 device: label,
267 sample,
268 pass: outcome.error ? null : failures.length === 0,
269 failures,
270 metrics: outcome.metrics,
271 ...(outcome.error ? { error: outcome.error } : {}),
272 };
273 rows.push(row);
274 NodeFS.writeSync(resultsFile, `${JSON.stringify(row)}\n`);
275 yield* Console.error(
276 `${testCase.id} #${sample}: ${row.pass === null ? row.error : row.pass ? "pass" : `FAIL ${failures.join("; ")}`}`,
277 );
278 }
279 }
280 }
281 NodeFS.closeSync(resultsFile);
282 const minutes = Math.round(((yield* Clock.currentTimeMillis) - startedAt) / 60_000);
283 const results = resultsFor(rows, CASES);
284 const summary = summarize({
285 cases: selected as ReadonlyArray<MobileBehaviorCase>,
286 results,
287 ...Option.match(baseline, { onNone: () => ({}), onSome: (rows) => ({ baseline: rows }) }),
288 header: `# Mobile behavior: ${label}\n\n${flags.deviceName || label}, ${flags.samples} samples per case, ${minutes} min. Cells show pass or FAIL and the key metric's median.`,
289 });
290 NodeFS.writeFileSync(NodePath.join(out, "summary.md"), summary);
291 yield* Console.log(summary);
292 if (blockingResults(results, CASES).length > 0) process.exitCode = 1;
293 }, Effect.scoped),
294);
295
296Command.run(command, { version: "0.0.0" }).pipe(
297 Effect.provide([NodeServices.layer, NodeCrypto.layer]),
298 NodeRuntime.runMain,
299);