1import * as Schema from "effect/Schema";
2
3export const DeviceExpect = Schema.Struct({
4 briefs: Schema.optional(Schema.Number),
5 recording: Schema.optional(Schema.String),
6 artifact: Schema.optional(Schema.String),
7 noRawUi: Schema.optional(Schema.Boolean),
8 capture: Schema.optional(
9 Schema.Struct({
10 hostId: Schema.String,
11 deviceId: Schema.String,
12 app: Schema.optional(Schema.String),
13 }),
14 ),
15 noShutdown: Schema.optional(
16 Schema.Array(Schema.Struct({ hostId: Schema.String, deviceId: Schema.String })),
17 ),
18 report: Schema.optional(Schema.Array(Schema.String)),
19 reportOrCapture: Schema.optional(Schema.Boolean),
20 maxFailedAttemptsPerTurn: Schema.optional(Schema.Number),
21 maxCaptureAttempts: Schema.optional(Schema.Number),
22});
23
24type DeviceRun = {
25 readonly launchers?: ReadonlyArray<string>;
26 readonly text?: string;
27 readonly finalText?: string;
28 readonly stopReason?: string;
29 readonly transcript?: ReadonlyArray<unknown>;
30 readonly calls: ReadonlyArray<{
31 readonly step: number;
32 readonly tool: string;
33 readonly input: unknown;
34 readonly turn?: number;
35 readonly failed?: boolean;
36 readonly output?: string;
37 }>;
38};
39const record = (value: unknown): value is Readonly<Record<string, unknown>> =>
40 typeof value === "object" && value !== null && !Array.isArray(value);
41const inputString = (input: unknown, key: string) =>
42 record(input) && typeof input[key] === "string" ? input[key] : "";
43const resultTarget = (call: DeviceRun["calls"][number]) => {
44 try {
45 const device: unknown = JSON.parse(call.output ?? "").device;
46 if (record(device) && typeof device.hostId === "string" && typeof device.id === "string")
47 return { hostId: device.hostId, deviceId: device.id };
48 } catch {}
49};
50const inputTarget = (call: DeviceRun["calls"][number]) => ({
51 hostId: inputString(call.input, "hostId") || "local",
52 deviceId: inputString(call.input, "deviceId"),
53});
54const openTargets = (previous: DeviceRun["calls"]) => {
55 const sessions = new Map<string, ReturnType<typeof inputTarget>>();
56 for (const prior of previous) {
57 if (prior.failed !== false) continue;
58 if (prior.tool === "device_open") {
59 const target = resultTarget(prior) ?? inputTarget(prior);
60 const key = `${target.hostId}/${target.deviceId}`;
61 sessions.delete(key);
62 sessions.set(key, target);
63 } else if (prior.tool === "device_close") {
64 const host = inputString(prior.input, "hostId"),
65 id = inputString(prior.input, "deviceId");
66 for (const [key, target] of sessions)
67 if ((!host || target.hostId === host) && (!id || target.deviceId === id))
68 sessions.delete(key);
69 }
70 }
71 return [...sessions.values()];
72};
73const flagValue = (words: ReadonlyArray<string>, flag: string) => {
74 for (const [i, word] of words.entries()) {
75 if (i === 0) continue;
76 if (word === flag) return words[i + 1];
77 if (word.startsWith(`${flag}=`)) return word.slice(flag.length + 1);
78 }
79};
80/** Each successful device_open's launcher, with the device and session it targets. */
81const launcherOpens = (previous: DeviceRun["calls"]) =>
82 previous.flatMap((call) => {
83 if (call.tool !== "device_open" || call.failed !== false) return [];
84 try {
85 const { device, agentDevice }: Record<string, unknown> = JSON.parse(call.output ?? "");
86 if (
87 !record(device) ||
88 !record(agentDevice) ||
89 typeof device.hostId !== "string" ||
90 typeof device.id !== "string" ||
91 typeof agentDevice.command !== "string"
92 )
93 return [];
94 const targetArgs = Array.isArray(agentDevice.targetArgs)
95 ? agentDevice.targetArgs.filter((arg): arg is string => typeof arg === "string")
96 : [];
97 const session =
98 typeof agentDevice.session === "string"
99 ? agentDevice.session
100 : flagValue(["", ...targetArgs], "--session");
101 return [
102 { command: agentDevice.command, hostId: device.hostId, deviceId: device.id, session },
103 ];
104 } catch {
105 return [];
106 }
107 });
108/**
109 * The device a launcher command drives: the one its --udid, --serial or --session names among
110 * the devices its launcher was returned for, else its launcher's only device. Without launcher
111 * results, only the sole open device.
112 */
113const launcherTarget = (words: ReadonlyArray<string>, previous: DeviceRun["calls"]) => {
114 const id = flagValue(words, "--udid") ?? flagValue(words, "--serial");
115 const session = flagValue(words, "--session");
116 const launched = launcherOpens(previous).filter((open) => open.command === words[0]);
117 const candidates = (
118 launched.length > 0
119 ? launched
120 : openTargets(previous).map((t) => ({ ...t, session: undefined }))
121 ).filter(
122 (t) =>
123 (id === undefined || t.deviceId === id) &&
124 (session === undefined || t.session === undefined || t.session === session),
125 );
126 return new Set(candidates.map((t) => `${t.hostId}/${t.deviceId}`)).size === 1
127 ? candidates.at(-1)
128 : undefined;
129};
130const screenshotTarget = (call: DeviceRun["calls"][number], previous: DeviceRun["calls"]) => {
131 const result = resultTarget(call);
132 if (result) return result;
133 if (call.failed && /open in this thread/.test(call.output ?? "")) return undefined;
134 const host = inputString(call.input, "hostId"),
135 id = inputString(call.input, "deviceId");
136 if (host && id) return inputTarget(call);
137 const matches = openTargets(previous).filter(
138 (t) => (!host || t.hostId === host) && (!id || t.deviceId === id),
139 );
140 return id ? matches.at(-1) : matches.length === 1 ? matches[0] : undefined;
141};
142const managed =
143 /device_(?:open|screenshot|action)\b|device (?:panel|manager|tools)|returned\s+(?:(?:bound|pinned|agent-device|cli)\s+)*(?:launcher|driver|command)/;
144const ui =
145 /\b(?:ui|screens?|screenshots?|screen captures?|record(?:ing|video)?s?|settings|taps?|typ(?:e|ing)|launch(?:es|ing)?|relaunch(?:es|ing)?|deep links?|swipes?|simulator (?:control|actions)|device actions|(?:drive|control|inspect)\s+(?:the\s+)?(?:simulator|device|app))\b/;
146const rawCommand =
147 /\bsimctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|\bsimctl\s+io\s+(?:[^\s;]+\s+)?(?:screenshot|recordvideo)\b|\bidb\b[^.;]*\b(?:ui|screenshot|launch|video|record)\b|\badb\b[^.;]*\b(?:input|screencap|screenrecord)\b|\blab-(?:shot|ui|launch|capture(?:-set)?)\b|\bmac_agent_device\b/;
148
149export function briefRoute(text: string) {
150 const positive = text
151 .toLowerCase()
152 .replaceAll("’", "'")
153 .replace(/[`*]/g, "")
154 // "SSH for the non-UI operations" restricts SSH rather than routing UI through it.
155 .replace(/\bnon-(?:ui|device|simulator|screen)\b/g, "auxiliary")
156 // A trailing prohibition negates its subject, not an earlier independent predicate.
157 .replace(
158 /(^|[.;\n]\s*)((?:-\s*)?(?:use|capture|take|record|run|call|drive)\b[^.;,\n]*?)\s+and\s+(?=[^.;,\n]*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b)/g,
159 "$1$2. ",
160 )
161 .replace(
162 /\b(?!(?:because|since|as|if|unless|when|so|but)\b)(?:(?!\b(?:because|since|as|if|unless|when|so|but)\b)[^.;,\n])*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b/g,
163 "",
164 )
165 .replace(/\b(?:do not|don't)\s+forget\s+to\b/g, "")
166 .replace(/\b(?:do not|don't)\s+avoid\b/g, "use")
167 .replace(/\b(?:should|must)\s+not\s+be\s+avoided\b/g, "is allowed")
168 .replace(/\b(?:there is )?no need to avoid\b/g, "use")
169 .replace(/\bnot just\b/g, "")
170 .replace(/\bbut (?:also )?for\b/g, "and for")
171 .replace(/\bavoid limiting (ssh|simctl|adb|idb) to [^.;]+;\s*use it\b/g, "use $1")
172 .replace(/(^|[.!;\n]\s*)(for [^.;,\n]+),\s*(?=(?:use|run)\b)/g, "$1$2 ")
173 .split(
174 /[\n.!;]+(?=\s|$)|\b(?:but|however|while)\b|,\s*(?=(?:screenshots?|recordings?)\s+(?:via|through|go))|(?:,\s*|\band\s+|\bthen\s+)(?=(?:use|run|call)\b)/,
175 )
176 .map((clause) =>
177 (
178 clause.split(
179 /\b(?:not|never|no|don't|cannot|can't|avoid|forbidden|prohibited|disallowed|rather than|instead of|without)\b/,
180 )[0] ?? ""
181 )
182 .replace(
183 /\b(?:ui|screen|settings|simulator|device|app)\s+(logs?|builds?|test runners?)\b/g,
184 "$1",
185 )
186 .replace(/\b(?:record|capture|stream)\s+(?:the\s+)?logs\b/g, "logs")
187 .replace(/\b(?:logs? from|reading)\s+screenshot\s+(?:tests?|logs?)\b/g, "logs")
188 .replace(
189 /\b(?:copy(?:ing)?|fetch(?:ing)?|transfer(?:ring)?|download(?:ing)?|sync(?:ing)?)\s+(?:(?:the|already|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\b/g,
190 "artifact transfer",
191 ),
192 );
193 const hasManaged = positive.some((clause) => managed.test(clause));
194 const hasRaw = positive.some((clause) => {
195 if (rawCommand.test(clause)) return true;
196 const routes = clause.matchAll(/\b(?:ssh|simctl|idb|adb|lab scripts?)\b/g);
197 for (const route of routes) {
198 const before = clause.slice(0, route.index);
199 const after = clause.slice(route.index + route[0].length).split(managed)[0] ?? "";
200 // Copying already-saved artifacts is auxiliary; keep later capture verbs visible.
201 const copies =
202 /^\s+(?:to\s+)?(?:copy|fetch|transfer|download|sync)\b/.test(after) ||
203 /\b(?:copy|fetch|transfer|download|sync)\s+(?:(?:the|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\s+(?:over|via|through|using)\s*$/.test(
204 before,
205 );
206 if (copies && !/\b(?:take|capture|inspect|record|drive|control)\b/.test(after)) continue;
207 const purpose =
208 /^\s+(?:(?:is|are|can|may|must|should|be|used|only|allowed|permitted|fine)\s+)*(?:for|to|into)\b/.test(
209 after,
210 );
211 const hostPurpose =
212 /^\s+(?:-\S+\s+\S+\s+)*\S+\s+(?:for|and|then)\b|^\s+(?:handles|and|then)\b/.test(after);
213 if ((purpose || hostPurpose || new RegExp(`^\\s+${ui.source}`).test(after)) && ui.test(after))
214 return true;
215 const via = new RegExp(
216 `${ui.source}(?:\\s+(?:may|can|must|should|be|is|are|taken|captured|saved|recorded|only|directly))*\\s+(?:via|by|over|through|using|with|use|run)\\s+(?:raw\\s+)?$`,
217 );
218 if (
219 via.test(before) ||
220 /\b(?:screenshots?|screen(?: capture)?|recordings?)(?:\s+on\s+\S+)?\s+(?:over|via|using|through|with)\s*$/.test(
221 before,
222 ) ||
223 /\bfor\s+[^.;]*\b(?:screenshots?|recordings?|screen capture)\s*,?\s*$/.test(before)
224 )
225 return true;
226 }
227 return false;
228 });
229 return hasManaged && hasRaw ? "mixed" : hasManaged ? "managed" : hasRaw ? "raw" : "neutral";
230}
231
232const executable = (word = "") => word.split("/").at(-1) ?? "";
233const uiCommand =
234 /^(?:simctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|simctl\s+io\s+\S+\s+(?:screenshot|recordvideo)\b|idb\b.*\b(?:ui|screenshot|launch|video|record)\b|adb\b.*\b(?:shell\s+(?:input|am\s+start|settings)|(?:shell|exec-out)\s+(?:screencap|screenrecord))\b|screencapture\b|lab-(?:shot|ui|launch|capture(?:-set)?)\b)/i;
235const deviceArgs = (words: ReadonlyArray<string>) => {
236 const args: string[] = [];
237 for (let i = 1; i < words.length; i++) {
238 const word = words[i]!;
239 if (word === "--") return [...args, ...words.slice(i + 1)];
240 if (!word.startsWith("-")) args.push(word);
241 else if (
242 /^--(?:platform|udid|serial|device|target|level|config|remote-config|session|state-dir|scope|timeout|timeout-ms|wait|metro-port|out|quality)$/.test(
243 word,
244 )
245 )
246 i++;
247 }
248 return args;
249};
250const auxiliaryAgent =
251 /^(?:(?:devices|apps|doctor|capabilities|install|uninstall|reinstall|install-from-source|logs|help|version)(?: |$)|session list(?: |$)|record contact-sheet(?: |$))/;
252const deviceOperation = (words: ReadonlyArray<string>) => {
253 const args = deviceArgs(words);
254 return !!args[0] && !auxiliaryAgent.test(args.join(" "));
255};
256const interpreter = /^(?:python\d*(?:\.\d+)?|node|ruby|perl|osascript)$/;
257const shells = new Set(["bash", "sh", "zsh"]);
258/** Human and JSON agent-device failure envelopes. */
259const AGENT_DEVICE_ERROR = /\bError \([A-Z_]+\):|"success"\s*:\s*false/;
260
261export type ShellInvocation = {
262 readonly words: ReadonlyArray<string>;
263 readonly remote: boolean;
264 readonly raw: boolean;
265 readonly managed: boolean;
266 /** Raw-route checks inspect possible actions; managed credit requires established execution. */
267 readonly executed: boolean;
268 /** An aggregate success cannot establish success of an action masked by ||. */
269 readonly unmasked: boolean;
270 /** A later `;` or `&` command owns the call's exit status. */
271 readonly sequenced: boolean;
272};
273
274// A deliberately bounded shell grammar, not a shell executor. Keep quotes until expansion so
275// literal command examples, comments and quoted heredocs never become invocations.
276const tokens = (source: string) =>
277 source
278 .replace(/'[^']*'|\\[\s\S]/g, (part) => (part === "\\\n" ? "" : part))
279 .match(
280 /`(?:\\.|[^`])*`|[<>]\((?:[^()]|\([^()]*\))*\)|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:\\.|[^"\\])*"|\([^()]*\))*\)|\d+(?:>>?|<)(?:&\d+)?|(?:"(?:\\.|[^"\\])*"|'[^']*'|\\.|\$\{\w+\}|[^\s;&|(){}<>"'\\])+|&&|\|\||[;\n&|(){}]|\d*(?:>>?|<)(?:&\d+)?/g,
281 ) ?? [];
282const unquote = (word: string, variables: Map<string, string>) =>
283 word.replace(
284 /'([^']*)'|"((?:\\.|[^"\\])*)"|\\(.)|([^'"\\]+)/g,
285 (
286 _part,
287 single: string | undefined,
288 double: string | undefined,
289 escaped: string | undefined,
290 plain: string | undefined,
291 ) => {
292 if (single !== undefined) return single;
293 if (escaped !== undefined) return escaped;
294 return (double ?? plain ?? "").replace(
295 /\\(["\\$])|\$(?:\{(\w+)\}|(\w+))/g,
296 (ref, escaped: string | undefined, braced: string | undefined, bare: string | undefined) =>
297 escaped ?? variables.get(braced ?? bare ?? "") ?? ref,
298 );
299 },
300 );
301
302function unwrap(words: string[]) {
303 while (words.length) {
304 if (/^\w+=/.test(words[0]!)) {
305 words.shift();
306 continue;
307 }
308 const name = executable(words[0]);
309 if (!/^(?:env|sudo|command|nohup|exec|time|caffeinate|timeout|nice)$/.test(name)) break;
310 words.shift();
311 const takesValue: Record<string, RegExp> = {
312 env: /^(?:-u|--unset|-C|--chdir)$/,
313 sudo: /^(?:-[ughpCT]|--user|--group|--host|--prompt)$/,
314 caffeinate: /^-[tw]$/,
315 timeout: /^(?:-[sk]|--signal|--kill-after)$/,
316 nice: /^(?:-n|--adjustment)$/,
317 };
318 while (words[0]?.startsWith("-")) {
319 const option = words.shift()!;
320 if (option === "--") break;
321 if (takesValue[name]?.test(option)) words.shift();
322 }
323 if (name === "timeout") words.shift();
324 }
325 return words;
326}
327
328/** Literal process-launch APIs only. Definitions and strings are not executed programs. */
329function scriptInvocations(
330 script: string,
331 remote: boolean,
332 variables: Map<string, string>,
333 language: string,
334): ShellInvocation[] {
335 const actions: ShellInvocation[] = [];
336 const python = language.startsWith("python");
337 // A function body is source, not an invocation. Calls to computed/script functions are
338 // outside this grammar; do not grant managed credit for them.
339 let definitionIndent: number | undefined;
340 if (python)
341 script = script
342 .split("\n")
343 .map((line) => {
344 const indent = line.match(/^\s*/)?.[0].length ?? 0;
345 if (line.trim() && definitionIndent !== undefined && indent <= definitionIndent)
346 definitionIndent = undefined;
347 if (/^\s*(?:async\s+)?def\s/.test(line)) definitionIndent = indent;
348 return definitionIndent === undefined ? line : "";
349 })
350 .join("\n");
351 const launch =
352 /^(subprocess\.(?:run|Popen|call|check_call|check_output)|os\.(?:system|popen)|(?:child_process\.)?(?:execFileSync|execFile|execSync|exec|spawnSync|spawn))\s*\(/;
353 const literal = "(?:'[^']*'|\"(?:\\\\.|[^\"\\\\])*\")";
354 const quoteArg = (arg: string) => `'${unquote(arg, variables).replaceAll("'", "'\\''")}'`;
355 const array = (source: string) => {
356 const match = new RegExp(`^\\[\\s*(${literal}(?:\\s*,\\s*${literal})*\\s*,?)?\\s*\\]`).exec(
357 source,
358 );
359 return match
360 ? (match[1]?.match(new RegExp(literal, "g")) ?? []).map(quoteArg).join(" ")
361 : undefined;
362 };
363 for (let i = 0; i < script.length; i++) {
364 const char = script[i];
365 if (char === "#" || script.slice(i, i + 2) === "//") {
366 const end = script.indexOf("\n", i);
367 i = end < 0 ? script.length : end;
368 continue;
369 }
370 if (char === "'" || char === '"' || char === "`") {
371 const quote = python && script.slice(i, i + 3) === char.repeat(3) ? char.repeat(3) : char;
372 i += quote.length;
373 while (i < script.length && script.slice(i, i + quote.length) !== quote) {
374 if (script[i] === "\\") i++;
375 i++;
376 }
377 i += quote.length - 1;
378 continue;
379 }
380 const match = launch.exec(script.slice(i));
381 if (!match) continue;
382 const rest = script.slice(i + match[0].length).trimStart();
383 let program: string | undefined;
384 if (rest.startsWith("[")) program = array(rest);
385 else {
386 const arg = rest.match(new RegExp(`^${literal}`))?.[0];
387 const tail = arg === undefined ? "" : rest.slice(arg.length);
388 if (arg && /(?:spawn|execFile)/.test(match[1]!)) {
389 const argv = array(tail.replace(/^\s*,\s*/, ""));
390 if (argv !== undefined) program = `${quoteArg(arg)} ${argv}`;
391 } else if (
392 arg &&
393 (!python || match[1]!.startsWith("os.") || /^\s*,[^)]*\bshell\s*=\s*True\b/.test(tail))
394 ) {
395 program = unquote(arg, variables);
396 }
397 }
398 if (program !== undefined) {
399 const line = script.slice(script.lastIndexOf("\n", i) + 1, i);
400 const boundary = Math.max(
401 script.lastIndexOf(";", i),
402 python ? script.lastIndexOf("\n", i) : -1,
403 );
404 const statement = script.slice(boundary + 1, i);
405 const certain =
406 !/=>|\bfunction\b|\blambda\b/.test(statement) &&
407 (python ? !/^\s/.test(line) : !script.slice(0, i).includes("{"));
408 actions.push(
409 ...shellInvocations(program, new Map(variables), remote).map((action) => ({
410 ...action,
411 executed: action.executed && certain,
412 // A script's exit status needn't reflect its subprocesses (`subprocess.run` doesn't check).
413 sequenced: true,
414 })),
415 );
416 }
417 i += match[0].length - 1;
418 }
419 return actions;
420}
421
422/** Shared by raw-UI, recording and failure grading. Literal aliases may span shell calls. */
423export function shellInvocations(
424 command: string,
425 variables = new Map<string, string>(),
426 remote = false,
427 succeeded = false,
428 /** Shared with function calls: the shell's functions, and whether it exited. */
429 shell: {
430 functions: Map<string, { text: string; certain: boolean }>;
431 exited: boolean;
432 depth: number;
433 } = {
434 functions: new Map(),
435 exited: false,
436 depth: 0,
437 },
438): ShellInvocation[] {
439 if (shell.depth > 8) return [];
440 const bodies = new Map<string, { text: string; expand: boolean }>();
441 command = command.replace(
442 /^([^\n]*?)<<(-?)\s*(['"]?)(\w+)\3([^\n]*)\n([\s\S]*?)(?:\n[ \t]*\4[ \t]*(?=\n|(?![\s\S]))|(?![\s\S]))/gm,
443 (
444 _match,
445 line: string,
446 strip: string,
447 quote: string,
448 _end: string,
449 rest: string,
450 body: string,
451 ) => {
452 const key = `__HEREDOC_${bodies.size}`;
453 bodies.set(key, { text: strip ? body.replace(/^\t+/gm, "") : body, expand: !quote });
454 return `${line} ${key} ${rest}`;
455 },
456 );
457 const actions: ShellInvocation[] = [];
458 let clause: string[] = [];
459 let pipe: string | undefined;
460 let gate = "";
461 let status: boolean | undefined;
462 // `before` says whether an earlier if/elif branch of this statement ran: true, false or unknown.
463 const conditions: Array<{
464 parent: boolean;
465 certain: boolean;
466 take: boolean | undefined;
467 before: boolean | undefined;
468 }> = [];
469 let active = true;
470 let certain = true;
471 let currentCertain = true;
472 let currentUnmasked = true;
473 let currentSequenced = false;
474 // After a certain `exit` or `return`, nothing later in its scope runs.
475 let exited = false;
476 const append = (nested: ShellInvocation[]) =>
477 actions.push(
478 ...nested.map((action) => ({
479 ...action,
480 executed: action.executed && currentCertain,
481 unmasked: action.unmasked && currentUnmasked,
482 sequenced: action.sequenced || currentSequenced,
483 })),
484 );
485 const truth = (words: string[]): boolean | undefined => {
486 if (words[0] === "!") {
487 const value = truth(words.slice(1));
488 return value === undefined ? undefined : !value;
489 }
490 return ["true", ":"].includes(words[0] ?? "") ? true : words[0] === "false" ? false : undefined;
491 };
492 const commandTokens = tokens(command.replace(/(^|\n)[ \t]*#[^\n]*/g, "$1"));
493 let tokenIndex = 0;
494 const functions = shell.functions;
495 const listTokens = () => {
496 let start = tokenIndex - 1,
497 end = tokenIndex;
498 while (start >= 0 && !/^[;\n&]$/.test(commandTokens[start]!)) start--;
499 while (end < commandTokens.length && !/^[;\n&]$/.test(commandTokens[end]!)) end++;
500 return commandTokens.slice(start + 1, end);
501 };
502 const substitutions = (source: string, heredoc = false) => {
503 const expanded = heredoc
504 ? source
505 : source.replace(
506 /"((?:\\.|[^"\\])*)"|'[^']*'|\\./g,
507 (part, double: string | undefined) => double ?? (part.startsWith("'") ? "" : part),
508 );
509 for (const match of expanded.matchAll(/(?<!\\)\$\(((?:[^()]|\([^()]*\))*)\)/g))
510 append(shellInvocations(match[1]!, new Map(variables), remote));
511 for (const match of expanded.matchAll(/(?<!\\)`([^`]*)`|[<>]\(((?:[^()]|\([^()]*\))*)\)/g))
512 append(shellInvocations(match[1] ?? match[2]!, new Map(variables), remote));
513 };
514 const flush = () => {
515 if (!clause.length) return;
516 const rawWords = clause;
517 const redirects = redirectOperands;
518 redirectOperands = [];
519 clause = [];
520 let words = rawWords.map((word) => unquote(word, variables));
521 if (words[0] === "if") {
522 const parent = active;
523 conditions.push({
524 parent,
525 certain,
526 take: undefined,
527 before: false,
528 });
529 words.shift();
530 }
531 if (words[0] === "then") {
532 const condition = conditions.at(-1);
533 // The branch follows the condition list's final status (`if true && false`), not its
534 // first command's.
535 if (condition) condition.take = condition.before === true ? false : status;
536 active =
537 condition?.parent !== false && condition?.take !== false && condition?.before !== true;
538 certain =
539 condition?.certain !== false && condition?.take === true && condition?.before === false;
540 words.shift();
541 }
542 if (words[0] === "else") {
543 const condition = conditions.at(-1);
544 active = !!condition?.parent && condition.take !== true && condition.before !== true;
545 certain = !!condition?.certain && condition.take === false && condition.before === false;
546 words.shift();
547 }
548 if (words[0] === "fi") {
549 const condition = conditions.pop();
550 active = condition?.parent ?? true;
551 certain = condition?.certain ?? true;
552 words.shift();
553 }
554 if (words[0] === "elif") {
555 // An elif's condition runs only when no earlier branch did.
556 const condition = conditions.at(-1);
557 if (condition) {
558 condition.before =
559 condition.before === true || condition.take === true
560 ? true
561 : condition.before === false && condition.take === false
562 ? false
563 : undefined;
564 condition.take = condition.before === true ? false : truth(words.slice(1));
565 active = condition.parent && condition.before !== true;
566 certain = condition.certain && condition.before === false;
567 }
568 words.shift();
569 }
570 const loopList = /^(?:for|select)$/.test(words[0] ?? "");
571 if (/^(?:while|until)$/.test(words[0] ?? "") || loopList) {
572 const value = loopList
573 ? words[2] === "in" && words.length > 3
574 ? true
575 : undefined
576 : truth(words.slice(1));
577 conditions.push({
578 parent: active,
579 certain,
580 take: words[0] === "until" && value !== undefined ? !value : value,
581 before: false,
582 });
583 if (!loopList) words.shift();
584 }
585 if (words[0] === "do") {
586 const condition = conditions.at(-1);
587 active = condition?.parent !== false && condition?.take !== false;
588 certain = condition?.certain !== false && condition?.take === true;
589 words.shift();
590 }
591 if (words[0] === "done") {
592 const condition = conditions.pop();
593 active = condition?.parent ?? true;
594 certain = condition?.certain ?? true;
595 words.shift();
596 }
597 const reachable =
598 gate === "&&" ? status : gate === "||" ? (status === undefined ? undefined : !status) : true;
599 if (!active || exited || !words.length || reachable === false) return;
600 const list = listTokens();
601 currentUnmasked = !list.includes("||") && words[0] !== "!";
602 // The call's exit status is the last command's; an action followed by `;` or `&` doesn't own it.
603 // A command piped into another doesn't own the pipeline's status either.
604 currentSequenced =
605 commandTokens[tokenIndex] === "|" ||
606 commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t));
607 const finalAndSuccess =
608 succeeded &&
609 gate === "&&" &&
610 currentUnmasked &&
611 !commandTokens.some((t) => /^(?:if|for|while|until|then|else|fi|do|done)$/.test(t)) &&
612 !commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t));
613 currentCertain = certain && (reachable === true || finalAndSuccess);
614 // A list's status is known only when this command certainly ran, or when its own result
615 // decides the list either way (`x && false`, `x || true`).
616 const value = truth(words);
617 status =
618 reachable === true
619 ? value
620 : gate === "&&" && value === false
621 ? false
622 : gate === "||" && value === true
623 ? true
624 : undefined;
625 for (const word of [...rawWords, ...redirects]) if (!bodies.has(word)) substitutions(word);
626 if (loopList) return;
627 const body = words.map((word) => bodies.get(word)).find((value) => value !== undefined);
628 if (body?.expand) substitutions(body.text, true);
629 words = words.filter((word) => !bodies.has(word));
630 const literalAssignments = words.every((word) => /^\w+=/.test(word));
631 if (literalAssignments || words[0] === "export") {
632 for (const word of words) {
633 const at = word.indexOf("=");
634 if (at > 0) {
635 if (currentCertain) variables.set(word.slice(0, at), word.slice(at + 1));
636 else variables.delete(word.slice(0, at));
637 }
638 }
639 return;
640 }
641 if (executable(words[0]) === "command" && /^-[vV]$/.test(words[1] ?? "")) return;
642 while (words[0] === "!") words.shift();
643 words = unwrap(words);
644 if (/^(?:exit|return)$/.test(executable(words[0]))) {
645 if (currentCertain) {
646 exited = true;
647 // `exit` in a function ends the shell; `return` ends only the function.
648 if (executable(words[0]) === "exit") shell.exited = true;
649 } else certain = false;
650 return;
651 }
652 const name = executable(words[0]);
653 let input = body?.text ?? pipe;
654 pipe = undefined;
655 if (name === "cat" || name === "tee") {
656 pipe = input;
657 return;
658 }
659 if (name === "printf" || name === "echo") {
660 pipe =
661 name === "echo"
662 ? words.slice(1).join(" ")
663 : words[1]?.includes("%s")
664 ? words.slice(2).join("\n")
665 : words[1]?.replaceAll("\\n", "\n");
666 return;
667 }
668 if (functions.has(name)) {
669 const definition = functions.get(name)!;
670 const call = { functions, exited: false, depth: shell.depth + 1 };
671 append(
672 shellInvocations(definition.text, new Map(variables), remote, succeeded, call).map(
673 (action) => ({ ...action, executed: action.executed && definition.certain }),
674 ),
675 );
676 if (call.exited && definition.certain && currentCertain) exited = shell.exited = true;
677 return;
678 }
679 if (name === "ssh") {
680 let at = 1;
681 while (words[at]?.startsWith("-")) {
682 const option = words[at++]!;
683 if (option === "--") break;
684 if (/^-[bBcDEeFIiJLlmOoPpQRSWw]$/.test(option)) at++;
685 }
686 at++; // destination
687 if (words[at] === "--") at++;
688 const script = words.slice(at).join(" ");
689 // A remote shell/interpreter inherits redirected/piped stdin, never the local aliases.
690 const destination = unwrap(tokens(script).map((word) => unquote(word, new Map())));
691 const reader = executable(destination[0]);
692 // Shells take a program with -c (`bash -se` reads stdin); interpreters with -c or -e.
693 const programFlag = shells.has(reader) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/;
694 const hasProgram = destination.some((word) => programFlag.test(word));
695 if (
696 input !== undefined &&
697 (!reader || ((shells.has(reader) || interpreter.test(reader)) && !hasProgram))
698 ) {
699 if (interpreter.test(reader)) append(scriptInvocations(input, true, new Map(), reader));
700 else append(shellInvocations(input, new Map(), true));
701 }
702 append(shellInvocations(script, new Map(), true));
703 return;
704 }
705 if (shells.has(name) || interpreter.test(name)) {
706 const at = words.findIndex((word) =>
707 (shells.has(name) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/).test(word),
708 );
709 if (at >= 0) input = words[at + 1];
710 if (input !== undefined)
711 append(
712 shells.has(name)
713 ? shellInvocations(input, new Map(variables), remote, succeeded)
714 : scriptInvocations(input, remote, variables, name),
715 );
716 return;
717 }
718 const endOptions = words.indexOf("--");
719 if (
720 words
721 .slice(0, endOptions < 0 ? words.length : endOptions)
722 .some((word) => word === "--help" || word === "-h") ||
723 words[1] === "help"
724 )
725 return;
726 const normalized = name === "xcrun" ? words.slice(1) : [...words];
727 if (name === "xcrun")
728 while (normalized[0]?.startsWith("-")) {
729 const option = normalized.shift();
730 if (/^(?:--sdk|-sdk|--toolchain|-toolchain)$/.test(option ?? "")) normalized.shift();
731 }
732 const cli = executable(normalized[0]);
733 const agent = ["agent-device", "mac_agent_device"].includes(cli) && deviceOperation(normalized);
734 actions.push({
735 words,
736 remote,
737 executed: currentCertain,
738 unmasked: currentUnmasked,
739 sequenced: currentSequenced,
740 raw:
741 uiCommand.test([cli, ...normalized.slice(1)].join(" ")) ||
742 (agent && (remote || cli === "mac_agent_device")),
743 managed: agent && !remote && cli === "agent-device",
744 });
745 };
746 const scopes: Array<{
747 variables: Map<string, string>;
748 active: boolean;
749 certain: boolean;
750 exited: boolean;
751 subshell: boolean;
752 }> = [];
753 let comment = false;
754 let redirect = false;
755 let redirectOperands: string[] = [];
756 for (; tokenIndex < commandTokens.length; tokenIndex++) {
757 const token = commandTokens[tokenIndex]!;
758 // Comments come first, so a commented-out function definition is never registered.
759 if (token === "\n") comment = false;
760 if (comment) continue;
761 if (token.startsWith("#")) {
762 comment = true;
763 continue;
764 }
765 // Shell function definitions don't execute their bodies. Keep literal calls supported.
766 const nameAt = tokenIndex + (token === "function" ? 1 : 0);
767 const bodyAt =
768 commandTokens[nameAt + 1] === "(" && commandTokens[nameAt + 2] === ")"
769 ? nameAt + 3
770 : token === "function"
771 ? nameAt + 1
772 : -1;
773 if (commandTokens[bodyAt] === "{") {
774 flush();
775 let end = bodyAt + 1,
776 depth = 1;
777 for (; end < commandTokens.length && depth; end++) {
778 if (commandTokens[end] === "{") depth++;
779 if (commandTokens[end] === "}") depth--;
780 }
781 if (active)
782 functions.set(commandTokens[nameAt]!, {
783 text: commandTokens.slice(bodyAt + 1, end - 1).join(" "),
784 certain,
785 });
786 tokenIndex = end - 1;
787 continue;
788 }
789 if (/^\d*(?:>>?|<)(?:&\d+)?$/.test(token)) {
790 redirect = !token.includes("&");
791 continue;
792 }
793 if (redirect) {
794 redirect = false;
795 redirectOperands.push(token);
796 continue;
797 }
798 if (/^(?:&&|\|\||[;\n&|(){}])$/.test(token)) {
799 flush();
800 if (token === "(" || token === "{") {
801 scopes.push({ variables, active, certain, exited, subshell: token === "(" });
802 if (token === "(") variables = new Map(variables);
803 const reachable =
804 gate === "&&"
805 ? status
806 : gate === "||"
807 ? status === undefined
808 ? undefined
809 : !status
810 : true;
811 active = active && reachable !== false;
812 certain = certain && reachable === true;
813 }
814 if (token === ")" || token === "}") {
815 const scope = scopes.pop();
816 // A subshell's exit ends only the subshell; a group's ends the shell.
817 if (scope) exited = scope.exited || (!scope.subshell && exited);
818 if (scope) ({ variables, active, certain } = scope);
819 }
820 if (token !== "|") pipe = undefined;
821 gate = token;
822 } else clause.push(token);
823 }
824 flush();
825 return actions;
826}
827
828export function gradeDevice(expect: typeof DeviceExpect.Type | undefined, run: DeviceRun) {
829 const checks: Array<{ check: string; pass: boolean }> = [];
830 if (!expect) return checks;
831 const add = (check: string, pass: boolean) => checks.push({ check, pass });
832 if (expect.briefs !== undefined) {
833 const briefs = run.calls.filter((call) =>
834 ["delegate_task", "t3_thread_launch"].includes(call.tool),
835 );
836 add("device briefs count", briefs.length >= expect.briefs);
837 for (const [i, call] of briefs.entries()) {
838 const route = briefRoute(
839 inputString(call.input, call.tool === "delegate_task" ? "task" : "message"),
840 );
841 add(`device brief ${i + 1} ${call.tool} ${route}`, route === "managed");
842 }
843 }
844 if (expect.artifact !== undefined)
845 add(
846 `device final artifact ${expect.artifact}`,
847 run.stopReason === "end_turn" && (run.finalText ?? "").includes(expect.artifact),
848 );
849 const variables = new Map<string, string>();
850 const shell = run.calls
851 .filter((call) => ["Bash", "Monitor"].includes(call.tool))
852 .map(
853 (call) =>
854 [
855 call,
856 shellInvocations(
857 inputString(call.input, "command"),
858 variables,
859 false,
860 call.failed === false,
861 ).map((action) => ({
862 ...action,
863 args: deviceArgs(action.words),
864 bound: !action.remote && (run.launchers ?? []).includes(action.words[0] ?? ""),
865 })),
866 ] as const,
867 );
868 const allActions = shell.flatMap(([, actions]) => actions);
869 const usableImage = (call: DeviceRun["calls"][number]) => {
870 if (call.failed !== false) return false;
871 try {
872 const image: unknown = JSON.parse(call.output ?? "").screenshot;
873 return (
874 record(image) &&
875 typeof image.width === "number" &&
876 image.width > 0 &&
877 typeof image.height === "number" &&
878 image.height > 0
879 );
880 } catch {
881 return false;
882 }
883 };
884 const successfulScreenshots = ([call, actions]: (typeof shell)[number]) => {
885 const captures = actions.filter((a) => a.managed && a.bound && a.args[0] === "screenshot");
886 if (
887 call.failed !== false ||
888 AGENT_DEVICE_ERROR.test(call.output ?? "") ||
889 captures.length !== 1 ||
890 /"(?:width|height)"\s*:\s*0\b/.test(call.output ?? "")
891 )
892 return [];
893 const paths = (call.output ?? "").split("\n").flatMap((line) => {
894 // The simulated shell words the success line several ways; each must still name the path.
895 const saved = line.match(
896 /^[^\w\n]*(?:Screenshot saved|Saved screenshot|Captured screenshot|Wrote screenshot)(?: to)?:?\s+(.+)$/i,
897 )?.[1];
898 if (saved) return [saved];
899 try {
900 const image: unknown = JSON.parse(line);
901 if (
902 record(image) &&
903 typeof image.path === "string" &&
904 typeof image.width === "number" &&
905 image.width > 0 &&
906 typeof image.height === "number" &&
907 image.height > 0
908 )
909 return [image.path];
910 } catch {}
911 return [];
912 });
913 return captures.filter(
914 (a) =>
915 a.unmasked &&
916 ((paths.length === 0 && a.executed && !a.sequenced) ||
917 paths.some(
918 (path) =>
919 path === a.args[1] ||
920 (a.args[1]?.startsWith("$/") && path.endsWith(a.args[1].slice(1))),
921 )),
922 );
923 };
924 const recovered =
925 expect.reportOrCapture &&
926 (run.calls.some((call) => call.tool === "device_screenshot" && usableImage(call)) ||
927 shell.some((entry) => successfulScreenshots(entry).length > 0));
928 if (expect.noShutdown)
929 add(
930 "callsMustNot shutdown occupied device",
931 !run.calls.some(
932 (call, index) =>
933 call.tool === "device_close" &&
934 record(call.input) &&
935 call.input.shutdown === true &&
936 openTargets(run.calls.slice(0, index)).some(
937 (target) =>
938 expect.noShutdown!.some(
939 (protectedTarget) =>
940 target.hostId === protectedTarget.hostId &&
941 target.deviceId === protectedTarget.deviceId,
942 ) &&
943 (!inputString(call.input, "hostId") ||
944 inputString(call.input, "hostId") === target.hostId) &&
945 (!inputString(call.input, "deviceId") ||
946 inputString(call.input, "deviceId") === target.deviceId),
947 ),
948 ),
949 );
950 for (const pattern of expect.report ?? [])
951 add(
952 `device final report /${pattern}/`,
953 run.stopReason === "end_turn" &&
954 (!!recovered || new RegExp(pattern).test(run.finalText ?? "")),
955 );
956 if (expect.noRawUi) add("shellMustNot device UI", !allActions.some((action) => action.raw));
957 if (expect.maxCaptureAttempts !== undefined)
958 add(
959 `maxCalls device captures <= ${expect.maxCaptureAttempts}`,
960 run.calls.filter((call) => call.tool === "device_screenshot").length +
961 allActions.filter((action) => action.managed && action.args[0] === "screenshot").length <=
962 expect.maxCaptureAttempts,
963 );
964 if (expect.capture) {
965 const target = expect.capture;
966 const matches = (value: ReturnType<typeof inputTarget> | undefined) =>
967 value?.hostId === target.hostId && value?.deviceId === target.deviceId;
968 const appSessions = new Set<string>();
969 let opened = false,
970 captured = false;
971 for (const [index, call] of run.calls.entries()) {
972 if (call.tool === "device_open" && call.failed === false && matches(inputTarget(call)))
973 opened = true;
974 const appReady = !target.app || (call.failed === false && appSessions.size > 0);
975 if (
976 opened &&
977 appReady &&
978 call.tool === "device_screenshot" &&
979 matches(screenshotTarget(call, run.calls.slice(0, index))) &&
980 (!target.app || usableImage(call))
981 )
982 captured = true;
983 const entry = shell.find(([source]) => source === call);
984 const actions = (entry?.[1] ?? []).filter((action) => action.managed && action.bound);
985 const screenshots = entry ? successfulScreenshots(entry) : [];
986 for (const action of actions) {
987 const { args, words } = action;
988 const onTarget = matches(launcherTarget(words, run.calls.slice(0, index)));
989 // Only an open or close that ran and succeeded changes the app session.
990 const ran =
991 call.failed === false &&
992 action.executed &&
993 action.unmasked &&
994 !AGENT_DEVICE_ERROR.test(call.output ?? "");
995 if (target.app && ["open", "close"].includes(args[0] ?? "") && ran) {
996 appSessions.delete(words[0]!);
997 if (
998 args[0] === "open" &&
999 args[1] === target.app &&
1000 onTarget &&
1001 (!action.sequenced ||
1002 (actions.filter((a) => a.args[0] === "open").length === 1 &&
1003 /^Opened:/m.test(call.output ?? "")))
1004 )
1005 appSessions.add(words[0]!);
1006 }
1007 if (
1008 opened &&
1009 onTarget &&
1010 (target.app
1011 ? appSessions.has(words[0]!) && screenshots.includes(action)
1012 : action.executed &&
1013 (["screenshot", "snapshot"].includes(args[0] ?? "") ||
1014 (args[0] === "record" && args[1] === "start")))
1015 )
1016 captured = true;
1017 }
1018 }
1019 add("device successful open and capture attempt", captured);
1020 }
1021 if (expect.recording !== undefined) {
1022 const steps = [...shell].flatMap(([call, actions]) => {
1023 const operations = actions.filter(
1024 ({ args }) => args[0] === "record" && /^(?:start|stop)$/.test(args[1] ?? ""),
1025 );
1026 return operations
1027 .filter(
1028 (action) =>
1029 action.executed &&
1030 action.unmasked &&
1031 action.bound &&
1032 ((!action.sequenced &&
1033 call.failed === false &&
1034 !AGENT_DEVICE_ERROR.test(call.output ?? "")) ||
1035 (operations.filter((o) => o.args[1] === action.args[1]).length === 1 &&
1036 new RegExp(
1037 `(?:^|\\n)[^\\w\\n]*Recording ${action.args[1] === "start" ? "started" : "stopped"}\\b|"kind"\\s*:\\s*"recording${action.args[1] === "start" ? "Started" : "Stopped"}"`,
1038 "i",
1039 ).test(call.output ?? ""))),
1040 )
1041 .map(({ args }) => args[1]!);
1042 });
1043 const start = steps.indexOf("start");
1044 checks.push({
1045 check: `device recording ${expect.recording}`,
1046 pass:
1047 start >= 0 &&
1048 steps.indexOf("stop", start + 1) >= 0 &&
1049 run.stopReason === "end_turn" &&
1050 (run.finalText ?? "").includes(expect.recording),
1051 });
1052 }
1053 if (expect.maxFailedAttemptsPerTurn !== undefined) {
1054 const failedByTurn = new Map<number, number>();
1055 for (const call of run.calls) {
1056 const actions = shell.find(([source]) => source === call)?.[1] ?? [];
1057 // A failed raw attempt, such as agent-device over ssh, is still a failed device attempt.
1058 const device = actions.filter((action) => action.managed || action.raw).length;
1059 // agent-device prints one error per failed command, even when a later command succeeds.
1060 const errors = (call.output ?? "").match(new RegExp(AGENT_DEVICE_ERROR, "g"))?.length ?? 0;
1061 let discoveryFailed = false;
1062 if (call.tool === "device_list") {
1063 try {
1064 const statuses: unknown = JSON.parse(call.output ?? "").hostStatuses;
1065 discoveryFailed =
1066 record(statuses) &&
1067 Object.values(statuses).some((s) => record(s) && s.status === "failed");
1068 } catch {}
1069 }
1070 const attempts =
1071 call.tool === "device_list"
1072 ? Number(!!call.failed || discoveryFailed)
1073 : /^device_(?:open|screenshot|close)$/.test(call.tool)
1074 ? Number(!!call.failed)
1075 : call.failed && device
1076 ? Math.max(1, Math.min(errors, device))
1077 : Math.min(errors, device);
1078 const turn = call.turn ?? 0;
1079 failedByTurn.set(turn, (failedByTurn.get(turn) ?? 0) + attempts);
1080 }
1081 const max = expect.maxFailedAttemptsPerTurn;
1082 add(
1083 `maxCalls device failures per user turn <= ${max}`,
1084 [...failedByTurn.values()].every((count) => count <= max),
1085 );
1086 }
1087 return checks;
1088}