Open History to see the last commit
1export interface Rect {
2 readonly x: number;
3 readonly y: number;
4 readonly width: number;
5 readonly height: number;
6}
7
8/** One node of an agent-device `snapshot --json`, on iOS or Android. */
9export interface SnapshotNode {
10 readonly index: number;
11 readonly parentIndex?: number;
12 /** Valid for commands until the next snapshot. */
13 readonly ref?: string;
14 readonly kind?: string;
15 readonly type?: string;
16 readonly label?: string;
17 readonly identifier?: string;
18 readonly rect?: Rect;
19 readonly hittable?: boolean;
20 readonly editable?: boolean;
21 readonly bundleId?: string;
22 readonly systemBar?: string;
23}
24
25export interface Snapshot {
26 readonly nodes: ReadonlyArray<SnapshotNode>;
27 readonly viewport: { readonly width: number; readonly height: number };
28 readonly appBundleId?: string;
29 readonly keyboard?: { readonly kind?: string; readonly rect?: Rect };
30}
31
32export interface Check {
33 readonly name: string;
34 readonly pass: boolean;
35 readonly detail: string;
36}
37
38export const check = (name: string, pass: boolean, detail: string): Check => ({
39 name,
40 pass,
41 detail,
42});
43
44// Kinds that are never controls a person taps; iOS also marks plain text hittable.
45const NOT_CONTROLS = new Set([
46 "application",
47 "text",
48 "image",
49 "scroll-area",
50 "other",
51 "toolbar",
52 "tab-bar",
53 "activity-indicator",
54 "composeview",
55 "keyboard",
56]);
57
58const area = (rect: Rect) => Math.max(0, rect.width) * Math.max(0, rect.height);
59
60const intersection = (a: Rect, b: Rect) =>
61 Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x)) *
62 Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y));
63
64export const describeRect = (rect: Rect | undefined) =>
65 rect
66 ? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}`
67 : "absent";
68
69const name = (node: SnapshotNode) => node.label || node.identifier || node.type || "unlabelled";
70
71const appOwned = (node: SnapshotNode, snapshot: Snapshot) =>
72 !node.systemBar && (node.bundleId === undefined || node.bundleId === snapshot.appBundleId);
73
74const isControl = (node: SnapshotNode, snapshot: Snapshot) =>
75 node.hittable === true &&
76 node.rect !== undefined &&
77 area(node.rect) > 0 &&
78 !NOT_CONTROLS.has(node.kind ?? "") &&
79 appOwned(node, snapshot);
80
81function ancestors(node: SnapshotNode, snapshot: Snapshot): SnapshotNode[] {
82 const byIndex = new Map(snapshot.nodes.map((candidate) => [candidate.index, candidate]));
83 const chain: SnapshotNode[] = [];
84 for (let parent = node.parentIndex; parent !== undefined;) {
85 const next = byIndex.get(parent);
86 if (!next || chain.includes(next)) break;
87 chain.push(next);
88 parent = next.parentIndex;
89 }
90 return chain;
91}
92
93const viewportRect = (snapshot: Snapshot): Rect => ({ x: 0, y: 0, ...snapshot.viewport });
94
95export function keyboardRect(snapshot: Snapshot): Rect | undefined {
96 if (snapshot.keyboard?.rect) return snapshot.keyboard.rect;
97 return snapshot.nodes.find((node) => node.type === "Keyboard" || node.kind === "keyboard")?.rect;
98}
99
100const TAB_LABELS = new Set(["Chats", "Pulls", "Usage", "Settings"]);
101
102/** The bottom tab bar, as a native tab bar node or the union of its tab buttons. */
103export function tabBarRect(snapshot: Snapshot): Rect | undefined {
104 const native = snapshot.nodes.find((node) => node.kind === "tab-bar" && node.rect)?.rect;
105 if (native) return native;
106 const tabs = snapshot.nodes.filter(
107 (node) =>
108 node.rect &&
109 TAB_LABELS.has(node.label ?? "") &&
110 node.rect.y > snapshot.viewport.height * 0.7 &&
111 appOwned(node, snapshot),
112 );
113 if (tabs.length < 2) return undefined;
114 const left = Math.min(...tabs.map((tab) => tab.rect!.x));
115 const top = Math.min(...tabs.map((tab) => tab.rect!.y));
116 const right = Math.max(...tabs.map((tab) => tab.rect!.x + tab.rect!.width));
117 const bottom = Math.max(...tabs.map((tab) => tab.rect!.y + tab.rect!.height));
118 return { x: left, y: top, width: right - left, height: bottom - top };
119}
120
121const ERROR_TEXT =
122 /\b(Render Error|Uncaught Error|Unhandled JS Exception|Syntax Error|keeps stopping|isn't responding)\b/i;
123
124/**
125 * Checks every screen should pass, whatever it shows: the app is in front without an error
126 * screen, and no control is clipped by the screen edge, stacked on another control, under a
127 * system bar, or an input under the keyboard.
128 */
129export function layoutChecks(snapshot: Snapshot, bundleId: string): Check[] {
130 const viewport = viewportRect(snapshot);
131 const controls = snapshot.nodes.filter((node) => isControl(node, snapshot));
132 const errors = snapshot.nodes.filter((node) => ERROR_TEXT.test(node.label ?? ""));
133
134 const clipped = controls.filter((node) => {
135 const visible = intersection(node.rect!, viewport);
136 if (visible === 0 || visible >= area(node.rect!) * 0.98) return false;
137 // Content scrolled past the edge is expected; fixed chrome cut off by it is not.
138 return !ancestors(node, snapshot).some((parent) => parent.kind === "scroll-area");
139 });
140
141 // Only siblings: a screen covered by another stays in the tree, so cousins overlap
142 // legitimately. Native tab items report overlapping frames.
143 const siblings = controls.filter(
144 (node) => !ancestors(node, snapshot).some((parent) => parent.kind === "tab-bar"),
145 );
146 const stacked: string[] = [];
147 for (const [index, a] of siblings.entries()) {
148 for (const b of siblings.slice(index + 1)) {
149 if (a.parentIndex !== b.parentIndex) continue;
150 const shared = intersection(a.rect!, b.rect!);
151 if (shared < Math.min(area(a.rect!), area(b.rect!)) * 0.25) continue;
152 // iOS exposes some elements twice, at the same frame.
153 const same =
154 describeRect(a.rect) === describeRect(b.rect) &&
155 (!a.label || !b.label || a.label === b.label);
156 if (!same) stacked.push(`"${name(a)}" and "${name(b)}"`);
157 }
158 }
159
160 const bars = snapshot.nodes.filter((node) => node.systemBar && node.rect);
161 const underBars = controls.filter((node) =>
162 bars.some((bar) => intersection(node.rect!, bar.rect!) > area(node.rect!) * 0.3),
163 );
164
165 const keyboard = keyboardRect(snapshot);
166 const coveredInputs = keyboard
167 ? snapshot.nodes.filter(
168 (node) =>
169 (node.kind === "text-field" || node.editable === true) &&
170 node.rect &&
171 appOwned(node, snapshot) &&
172 intersection(node.rect, keyboard) > area(node.rect) * 0.3,
173 )
174 : [];
175
176 const list = (nodes: ReadonlyArray<SnapshotNode>) =>
177 nodes.map((node) => `"${name(node)}" at ${describeRect(node.rect)}`).join(", ");
178 return [
179 check(
180 "app in front",
181 snapshot.appBundleId === undefined || snapshot.appBundleId === bundleId,
182 snapshot.appBundleId ?? "unknown",
183 ),
184 check("no error screen", errors.length === 0, errors.map(name).join(", ") || "none"),
185 check("no control cut off by the screen edge", clipped.length === 0, list(clipped) || "none"),
186 check("no controls stacked on each other", stacked.length === 0, stacked.join("; ") || "none"),
187 check("no control under a system bar", underBars.length === 0, list(underBars) || "none"),
188 check(
189 "no input under the keyboard",
190 coveredInputs.length === 0,
191 keyboard ? list(coveredInputs) || "none" : "no keyboard",
192 ),
193 ];
194}
195
196export const findLabel = (snapshot: Snapshot, pattern: RegExp) =>
197 snapshot.nodes.find(
198 (node) =>
199 node.rect &&
200 pattern.test(node.label ?? "") &&
201 intersection(node.rect, viewportRect(snapshot)) > 0 &&
202 appOwned(node, snapshot),
203 );