1// @effect-diagnostics nodeBuiltinImport:off
2/**
3 * T3's device tools over a case's world. McpHttpServer's device toolkit runs T3's handlers on its
4 * DeviceService, whose hosts are fakes holding the world's devices, so every result and error the
5 * agent sees is T3's. Launcher files go to a file system that keeps nothing.
6 */
7import * as NodeCrypto from "node:crypto";
8import * as NodeFS from "node:fs";
9import * as NodePath from "node:path";
10import {
11 DeviceError,
12 type DeviceHostId,
13 DeviceOperationError,
14 type DevicePlatform,
15 type DeviceSummary,
16 EnvironmentId,
17 LOCAL_DEVICE_HOST_ID,
18 LOCAL_DEVICE_HOST_LABEL,
19 ProviderInstanceId,
20 ThreadId,
21} from "@t3tools/contracts";
22import * as HostProcess from "@t3tools/shared/HostProcess";
23import * as Context from "effect/Context";
24import * as Digest from "effect/crypto/Digest";
25import * as Effect from "effect/Effect";
26import * as FileSystem from "effect/FileSystem";
27import * as Layer from "effect/Layer";
28import * as Option from "effect/Option";
29import * as Path from "effect/Path";
30import * as References from "effect/References";
31import * as Schema from "effect/Schema";
32// oxlint-disable-next-line t3code/no-raw-mcp-registration -- the eval calls the device tools McpHttpServer registered and registers none.
33import { McpSchema, McpServer } from "effect/ai";
34
35import * as ServerConfig from "../../apps/server/src/config.ts";
36import * as AgentDeviceShim from "../../apps/server/src/device/AgentDeviceShim.ts";
37import {
38 agentDeviceConfigPath,
39 agentDeviceSession,
40} from "../../apps/server/src/device/AgentDeviceTarget.ts";
41import type * as DeviceHost from "../../apps/server/src/device/DeviceHost.ts";
42import * as DeviceService from "../../apps/server/src/device/DeviceService.ts";
43import * as McpHttpServer from "../../apps/server/src/mcp/McpHttpServer.ts";
44import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts";
45import { liveThreadsLayer } from "../../apps/server/src/mcp/McpToolAccess.testkit.ts";
46import { DeviceToolkit } from "../../apps/server/src/mcp/toolkits/device/tools.ts";
47import * as ProjectService from "../../apps/server/src/project/ProjectService.ts";
48import { ServerSettingsService } from "../../apps/server/src/serverSettings.ts";
49import { shellInvocations } from "./device-grading.ts";
50
51export interface DeviceWorld {
52 readonly devices: ReadonlyArray<DeviceSummary>;
53 /** Labels of the SSH device hosts, by id. The local host always exists. */
54 readonly deviceHosts?: Readonly<Record<string, string>> | undefined;
55 /**
56 * agent-device sessions that prior turns started, by device id, with the app `open` put in
57 * each. Otherwise a device's session starts at its launcher's first `open` in the sample.
58 */
59 readonly agentDeviceSessions?:
60 | Readonly<Record<string, { readonly app?: string | undefined }>>
61 | undefined;
62 /**
63 * Hosts whose hub fails to list devices, by id, with the error it raises. On every call T3 marks
64 * each one failed and lists none of its devices.
65 */
66 readonly deviceDiscoveryFailures?: Readonly<Record<string, DeviceOperationError>> | undefined;
67 /** false: agent device access is off, so the thread holds no `device` capability. */
68 readonly deviceAgentAccess?: boolean | undefined;
69 /** Every capture reaches T3 with adb screencap's multi-display warning ahead of the PNG. */
70 readonly deviceInvalidCapture?: boolean | undefined;
71 /** Devices another thread's agent-device session holds, by device id: that thread's id. */
72 readonly deviceOwners?: Readonly<Record<string, string>> | undefined;
73}
74
75/** A shell, file or web call and the output the agent saw. */
76export interface ShellCall {
77 readonly tool: string;
78 readonly input: unknown;
79 readonly output: string;
80}
81
82/** A started agent-device session: whether `open <app>` put an app in it, or only a recording holds it. */
83export interface AgentDeviceSession {
84 readonly app: boolean;
85 readonly recordingOnly?: boolean;
86}
87
88/** Another thread's agent-device session, named as T3 names it, holding a device with its app open. */
89export interface AgentDeviceOwner {
90 readonly deviceId: string;
91 readonly threadId: string;
92 readonly session: string;
93}
94
95/** One sample's device tools, answered by T3. */
96export interface DeviceTools {
97 /** Runs one call as T3 does. A `failure` is raised in T3's DeviceService during the call. */
98 readonly call: (
99 name: string,
100 input: unknown,
101 failure?: DeviceError,
102 ) => Effect.Effect<{ readonly text: string; readonly isError: boolean }>;
103 /** The `agent-device` commands `device_open` returned, in order. */
104 readonly launchers: ReadonlyArray<string>;
105 /** Sessions other threads hold on the world's devices. */
106 readonly owners: ReadonlyArray<AgentDeviceOwner>;
107 /** Each session the launchers name after these calls, by name; undefined until it starts. */
108 readonly sessions: (
109 history: ReadonlyArray<ShellCall>,
110 ) => ReadonlyMap<string, AgentDeviceSession | undefined>;
111 /** agent-device's error for a plain launcher command that needs a session it has not started. */
112 readonly sessionError: (history: ReadonlyArray<ShellCall>, command: string) => string | undefined;
113 /**
114 * What the shell simulator is told of the launchers, their devices and sessions, and of the
115 * launcher opens in a new call that another thread's session refuses.
116 */
117 readonly describe: (
118 history: ReadonlyArray<ShellCall>,
119 command: string | undefined,
120 ) => ReadonlyArray<string>;
121}
122
123export const isDeviceTool = (name: string) => Object.hasOwn(DeviceToolkit.tools, name);
124
125const HELP = NodeFS.readFileSync(
126 NodePath.join(import.meta.dirname, "agent-device-help.txt"),
127 "utf8",
128);
129/** agent-device's help by topic, "" for `help` itself. */
130const HELP_TOPICS = new Map(
131 HELP.split(/^\$ agent-device help ?/m)
132 .slice(1)
133 .map((section) => [
134 section.slice(0, section.indexOf("\n")),
135 section.slice(section.indexOf("\n") + 1).trimEnd(),
136 ]),
137);
138const TARGET_FLAGS = new Set([
139 "--platform",
140 "--device",
141 "--udid",
142 "--serial",
143 "--session",
144 "--config",
145]);
146const isTargetFlag = (word = "") => TARGET_FLAGS.has(word.split("=")[0] ?? "");
147
148/**
149 * What one plain command prints when it asks a returned launcher for help on a topic
150 * agent-device-help.txt has, such as `<launcher> help record` or `<launcher> record --help`.
151 */
152export const launcherHelp = (launchers: ReadonlyArray<string>, command: string) => {
153 const plain = command.trim().replace(/\s+2>&1$/, "");
154 if (/[\n;&|<>`$()"'\\]/.test(plain)) return undefined;
155 const [executable = "", ...words] = plain.split(/\s+/);
156 if (!launchers.includes(executable)) return undefined;
157 // Target flags and their values pick a device; they do not change the help.
158 const asked = words.filter(
159 (word, i) =>
160 !isTargetFlag(word) && !(isTargetFlag(words[i - 1]) && !words[i - 1]?.includes("=")),
161 );
162 const topic =
163 asked[0] === "help"
164 ? asked.slice(1)
165 : ["--help", "-h"].includes(asked.at(-1) ?? "")
166 ? asked.slice(0, -1)
167 : undefined;
168 return topic !== undefined && topic.length <= 1 ? HELP_TOPICS.get(topic[0] ?? "") : undefined;
169};
170
171/**
172 * What the shell simulator knows of the launchers T3's device_open returns: agent-device's own
173 * help. It is the same on every ref; only the launcher paths in a sample differ.
174 */
175export const AGENT_DEVICE_SHELL = [
176 'The description lists the agent-device launchers T3 Code\'s device_open returned. Each is a file on the agent\'s machine, not on SSH hosts. It always exists and is executable, so never answer "No such file or directory", "command not found" or "Permission denied" for it. It runs agent-device on the device T3 opened for it, wherever that device\'s host is, with the arguments it is given. Answer it as agent-device does, from agent-device\'s help below:',
177 '- `help` and `--help` print the text under `$ agent-device help` exactly and in full, never shortened. `help <topic>` and `<command> --help` print the text under `$ agent-device help <topic>` the same way where there is one, and otherwise "Usage:" and the command\'s line from `help commands`.',
178 '- Only the commands and flags in this help exist. Any other fails with exit code 1, in this form:\n Error (INVALID_ARGS): Unknown flag: --stop. Did you mean --scope?\n Hint: Check command arguments and run --help for usage examples.\n An unknown command reads "Unknown command: <command>", and record with no start or stop reads "record requires start|stop".',
179 '- A launcher command runs in an agent-device session: the one its --session flag names, or else the one the launcher\'s directory is named after. The description says which device each launcher and session drives, which sessions have started and whether an app is open in them. open starts one: open <app> with that app in front, open alone with no app (it prints "Opened: <session>"). close ends it. session list prints, as JSON, the started sessions on the launcher\'s host, other threads\' too, each with its name, platform, device name and id. Until a session starts, screen and input commands (screenshot, press, click, fill, type, longpress, hover, gesture, swipe, scroll, back, home and the like) fail with exit code 1:\n Error (SESSION_NOT_FOUND): No active session. Run open first.\n Hint: Run open first or pass an explicit device selector.\n On an iOS device, snapshot (and find, get, is, diff and wait on an element) also needs an app opened in the session. Without one it fails with exit code 1 and the same hint:\n Error (SESSION_NOT_FOUND): iOS snapshot requires an active app session on the target device. Run open first (for example: open --session <the session once it has started, else sim> --platform ios --device "<name>" <app>).\n On Android, snapshot needs no session and starts one without an app. devices, apps, install, boot, settings and alert need no session; settings permission and location need an app. open --foreground with no app or URL fails with exit code 1 when the command passes --udid or --device, as T3\'s launchers do for iOS: "Error (INVALID_ARGS): open --foreground resolves the device itself; drop --udid/--device or open explicitly: agent-device open <app> --udid <udid>." open <app> --foreground works.',
180 '- record start without --scope device records the app open in the session. Before the session starts it fails with "Error (INVALID_ARGS): record start with app scope and an explicit session requires an active app session; run open <app> first, or use --scope device to record the full screen" and the hint above. In a session open with no app it fails with "Error (UNKNOWN): App-scoped recording requires an active app session; open the app under test before recording." and "Hint: Unexpected internal error. Retry with --debug and report the diagnostics log if it persists." --scope device records the whole screen and needs no session.',
181 "- Otherwise a command in this help acts on the launcher's device and succeeds unless the description says otherwise.",
182 "",
183 HELP,
184].join("\n");
185
186const FLAGS = new Set(HELP.match(/--[a-z][a-z\d-]*/g));
187/** Flags agent-device's help shows taking a value, such as `--scope <app|device|system>`. */
188const VALUE_FLAGS = new Set(
189 [...HELP.matchAll(/(--[a-z][a-z\d-]*)(?: <| [a-z]+\|)/g)].map((match) => match[1]),
190);
191const SELECTOR_FLAGS = ["--platform", "--target", "--device", "--udid", "--serial"];
192/**
193 * Commands agent-device 0.21.31's daemon routes to its `generic` and `interaction` handlers, which
194 * look up the named session first and fail with SESSION_NOT_FOUND without it. `get` and `is`
195 * check their arguments first, so they stay with the simulator.
196 */
197const NEEDS_SESSION =
198 /^(?:action-button|app-switcher|back|focus|fold|hardware-button|home|orientation|screenshot|scroll|telephony|tv-remote|viewport|click|fill|gesture|hover|longpress|press|swipe|type)$/;
199/** A shell call failed, or agent-device printed its failure envelope. */
200const FAILED = /^(?:Exit code [1-9]|<tool_use_error>)|\bError \([A-Z_]+\):|"success"\s*:\s*false/;
201
202/** A launcher `device_open` returned, and the agent-device session its target flags name. */
203export interface LauncherTarget {
204 readonly command: string;
205 readonly session: string;
206 /** A bound launcher passes its session itself; the shared one gets it from each command. */
207 readonly bound: boolean;
208 readonly deviceId: string;
209 /** agent-device refuses an iOS device's snapshot without an app in the session. */
210 readonly ios: boolean;
211 /** agent-device checks for an app before an app-scoped recording only on iOS Simulators. */
212 readonly iosSimulator: boolean;
213}
214
215/** One launcher command line: its agent-device command and arguments, and the session it names. */
216const launcherCall = (targets: ReadonlyArray<LauncherTarget>, words: ReadonlyArray<string>) => {
217 const mine = targets.filter((target) => target.command === words[0]);
218 if (mine.length === 0) return undefined;
219 const flags = new Map<string, string>();
220 const positionals: Array<string> = [];
221 // The shell splits an unquoted variable holding flags, such as `$AD open app $A`.
222 const args = words
223 .slice(1)
224 .flatMap((word) => (word.startsWith("-") && /\s/.test(word) ? word.trim().split(/\s+/) : word));
225 for (let i = 0; i < args.length; i++) {
226 const word = args[i]!;
227 if (word === "--") {
228 positionals.push(...args.slice(i + 1));
229 break;
230 }
231 const at = word.indexOf("=");
232 // An array the parser cannot read, such as `"${A[@]}"`, holds flags, not an app.
233 if (/^\$\{\w+\[[@*]\]\}$/.test(word)) continue;
234 if (!word.startsWith("-")) positionals.push(word);
235 else if (at > 0) flags.set(word.slice(0, at), word.slice(at + 1));
236 else flags.set(word, VALUE_FLAGS.has(word) ? (args[++i] ?? "") : "");
237 }
238 const named = flags.get("--session");
239 const target = mine.findLast((candidate) => candidate.session === named) ?? mine.at(-1)!;
240 // Flags this parser cannot read, such as a bash array, most likely are device_open's flags for
241 // the only session the shared launcher has served.
242 const unread =
243 named === undefined && !target.bound && new Set(mine.map((t) => t.session)).size === 1;
244 const [command = "", ...rest] = positionals;
245 return {
246 target,
247 command,
248 args: rest,
249 flags,
250 deviceId: flags.get("--udid") ?? flags.get("--serial") ?? target.deviceId,
251 // The session agent-device sees. The shared launcher refuses a command without --session
252 // itself; that is not modelled.
253 session: named ?? (target.bound ? target.session : undefined),
254 // The session the call most likely ran in, for following sessions through the history.
255 tracked: named ?? (target.bound || unread ? target.session : undefined),
256 selector: target.bound || unread || SELECTOR_FLAGS.some((flag) => flags.has(flag)),
257 appScope: (flags.get("--scope") ?? "app") === "app",
258 };
259};
260type LauncherCall = NonNullable<ReturnType<typeof launcherCall>>;
261
262/**
263 * A session after one successful command. agent-device 0.21.31 publishes the named session on
264 * `open`, on an Android `snapshot` with a device selector, and for a whole-screen recording, which
265 * `record stop` retires. `close` ends it. An iOS snapshot never starts one: without an app it fails.
266 */
267const afterCall = (call: LauncherCall, session: AgentDeviceSession | undefined) => {
268 switch (call.command) {
269 case "open":
270 return { app: call.args.length > 0 || (session?.app ?? false) };
271 case "snapshot":
272 return session ?? (call.selector && !call.target.ios ? { app: false } : undefined);
273 case "close":
274 return undefined;
275 case "record":
276 if (call.args[0] === "start" && !session && !call.appScope && call.selector)
277 return { app: false, recordingOnly: true };
278 return call.args[0] === "stop" && session?.recordingOnly ? undefined : session;
279 default:
280 return session;
281 }
282};
283
284/**
285 * The other session that holds the device an `open` would start a session on. agent-device 0.21.31
286 * refuses that open, as it does an open on any device another session in its daemon holds.
287 */
288const holder = (
289 call: LauncherCall,
290 session: string | undefined,
291 sessions: ReadonlyMap<string, AgentDeviceSession | undefined>,
292 owners: ReadonlyArray<AgentDeviceOwner>,
293) =>
294 call.command === "open" && session !== undefined && sessions.get(session) === undefined
295 ? owners.find(
296 (owner) =>
297 owner.deviceId === call.deviceId &&
298 owner.session !== session &&
299 sessions.get(owner.session) !== undefined,
300 )
301 : undefined;
302
303/**
304 * A launcher alias a call sets, such as `AD=<launcher>`. Grading drops an assignment it cannot prove
305 * ran, as in `mkdir -p d && AD=<launcher>; $AD open …`, but a call that did not fail ran it.
306 */
307const launcherAliases = (command: string, targets: ReadonlyArray<LauncherTarget>) =>
308 [...command.matchAll(/(?<![\w$])(\w+)=(["']?)([^\s;&|"'()]+)\2(?=[\s;&|)]|$)/g)].flatMap(
309 ([, name = "", , value = ""]) =>
310 targets.some((target) => target.command === value) ? [[name, value] as const] : [],
311 );
312const withAliases = (command: string, aliases: ReadonlyArray<readonly [string, string]>) =>
313 aliases.reduce(
314 (text, [name, value]) =>
315 text.replace(new RegExp(`"?\\$(?:\\{${name}\\}|${name}(?!\\w))"?`, "g"), value),
316 command,
317 );
318
319/** The sessions and variables after these calls. Aliases may span calls, as in grading. */
320const followSessions = (
321 targets: ReadonlyArray<LauncherTarget>,
322 declared: DeviceWorld["agentDeviceSessions"],
323 history: ReadonlyArray<ShellCall>,
324 owners: ReadonlyArray<AgentDeviceOwner>,
325) => {
326 const sessions = new Map<string, AgentDeviceSession | undefined>();
327 for (const target of targets) {
328 const prior = declared?.[target.deviceId];
329 sessions.set(target.session, prior && { app: prior.app !== undefined });
330 }
331 for (const owner of owners) sessions.set(owner.session, { app: true });
332 const variables = new Map<string, string>();
333 for (const { tool, input, output } of history) {
334 const command =
335 tool === "Bash" && typeof input === "object" && input !== null && "command" in input
336 ? input.command
337 : undefined;
338 if (typeof command !== "string") continue;
339 const failed = FAILED.test(output);
340 const aliases = failed ? [] : launcherAliases(command, targets);
341 const actions = shellInvocations(withAliases(command, aliases), variables, false, !failed);
342 for (const [name, value] of aliases) variables.set(name, value);
343 // A lone command's failure is its own. A compound call's output may not say which command
344 // failed, so its certain launcher commands count, as the simulator likely ran them. A call that
345 // did not fail ran the ones the parser could not prove ran, such as `mkdir -p d && $L open; …`.
346 if (actions.length === 1 && failed) continue;
347 for (const action of actions) {
348 const call =
349 (action.executed || !failed) && !action.remote
350 ? launcherCall(targets, action.words)
351 : undefined;
352 if (call?.tracked !== undefined && !holder(call, call.tracked, sessions, owners))
353 sessions.set(call.tracked, afterCall(call, sessions.get(call.tracked)));
354 }
355 }
356 return { sessions, variables };
357};
358
359/** Each session the launchers name after these calls. */
360export const agentDeviceSessions = (
361 targets: ReadonlyArray<LauncherTarget>,
362 declared: DeviceWorld["agentDeviceSessions"],
363 history: ReadonlyArray<ShellCall>,
364 owners: ReadonlyArray<AgentDeviceOwner> = [],
365) => followSessions(targets, declared, history, owners).sessions;
366
367/**
368 * Each launcher `open` a new call makes, read as the tracker reads calls, with the other thread's
369 * session that holds its device and whether another session holds it at all. agent-device refuses
370 * an open that has a holder with DEVICE_IN_USE. Each launcher command runs in order, so an earlier
371 * `close --session <owner>` in the same call frees the device for a later open.
372 */
373const opensIn = (
374 targets: ReadonlyArray<LauncherTarget>,
375 { sessions, variables }: ReturnType<typeof followSessions>,
376 command: string,
377 owners: ReadonlyArray<AgentDeviceOwner>,
378) => {
379 const current = new Map(sessions);
380 return shellInvocations(
381 withAliases(command, launcherAliases(command, targets)),
382 new Map(variables),
383 false,
384 true,
385 ).flatMap((action) => {
386 const call = action.remote ? undefined : launcherCall(targets, action.words);
387 if (call?.tracked === undefined) return [];
388 const owner = holder(call, call.tracked, current, owners);
389 const held = owners.some(
390 (candidate) =>
391 candidate.deviceId === call.deviceId && current.get(candidate.session) !== undefined,
392 );
393 if (owner === undefined) current.set(call.tracked, afterCall(call, current.get(call.tracked)));
394 return call.command === "open" ? [{ call, words: action.words, owner, held }] : [];
395 });
396};
397
398/**
399 * What the shell simulator is told of the sessions the launchers name, and other threads', with
400 * the device each drives where it is known.
401 */
402export const describeSessions = (
403 sessions: ReadonlyMap<string, AgentDeviceSession | undefined>,
404 owners: ReadonlyArray<AgentDeviceOwner> = [],
405 devices: ReadonlyMap<string, string> = new Map(),
406) =>
407 [...sessions]
408 .map(([name, session]) => {
409 const named = devices.has(name) ? `${name} (${devices.get(name)})` : name;
410 const owner = owners.find((candidate) => candidate.session === name);
411 if (owner !== undefined)
412 return session === undefined
413 ? `${named} belonged to another T3 thread (${owner.threadId}) and was closed, which freed its device`
414 : `${named} belongs to another T3 thread (${owner.threadId}) and holds its device with an app open`;
415 return session === undefined
416 ? `${named} has not started`
417 : session.recordingOnly
418 ? `${named} holds only a --scope device recording`
419 : session.app
420 ? `${named} is open with an app`
421 : `${named} is open with no app`;
422 })
423 .join("; ");
424
425/** agent-device's human-readable error, with the diagnostics lines its remote client adds. */
426const agentDeviceError = (
427 home: string,
428 session: string,
429 seed: string,
430 error: { readonly code: string; readonly message: string; readonly hint: string },
431) => {
432 const hash = NodeCrypto.createHash("sha256").update(seed).digest();
433 const id = `${hash.readUIntBE(0, 5).toString(36).padStart(8, "0")}-${hash.toString("hex", 5, 9)}`;
434 return [
435 "Exit code 1",
436 `Error (${error.code}): ${error.message}`,
437 `Hint: ${error.hint}`,
438 `Diagnostic ID: ${id}`,
439 `Diagnostics Log: ${home}/.agent-device/remote-diagnostics/${session}/requests/${hash.toString("hex", 9, 17)}.ndjson`,
440 ].join("\n");
441};
442const NO_SESSION_HINT = "Run open first or pass an explicit device selector.";
443/** agent-device's hint when another session holds the device; after --wait it says it waited. */
444const inUseHint = (owner: string, waitedMs?: number) =>
445 [
446 ...(waitedMs === undefined
447 ? []
448 : [`Waited ${Math.round(waitedMs)}ms for this device and it stayed busy.`]),
449 "Run agent-device session list to inspect active sessions.",
450 `To reuse this device, rerun the command with --session ${owner}.`,
451 `To open a new session on this device, first run agent-device close --session ${owner}.`,
452 ...(waitedMs === undefined
453 ? ["To wait for the device to free up, run the same open with --wait <ms>."]
454 : []),
455 ].join(" ");
456/** agent-device's refusal of an open on a device another session holds, unless --wait is bad. */
457const inUseError = (call: LauncherCall, owner: AgentDeviceOwner) => {
458 const wait = call.flags.get("--wait");
459 return wait === undefined || /^\d+$/.test(wait)
460 ? {
461 code: "DEVICE_IN_USE",
462 message: `Device is already in use by session "${owner.session}".`,
463 hint: inUseHint(owner.session, wait === undefined ? undefined : Number(wait)),
464 }
465 : undefined;
466};
467
468/**
469 * agent-device 0.21.31's error for a plain launcher command that needs its session before
470 * anything starts the session, as recorded on a real Mac. Both refs name the session (the bound
471 * launcher itself, the shared one through the --session flag device_open returned) and the device.
472 */
473export const agentDeviceSessionError = (
474 targets: ReadonlyArray<LauncherTarget>,
475 sessions: ReadonlyMap<string, AgentDeviceSession | undefined>,
476 command: string,
477 owners: ReadonlyArray<AgentDeviceOwner> = [],
478) => {
479 const plain = command.trim().replace(/\s+2>&1$/, "");
480 if (/[\n;&|<>`$()\\]/.test(plain)) return undefined;
481 const [action, ...others] = shellInvocations(plain);
482 const call = action && others.length === 0 ? launcherCall(targets, action.words) : undefined;
483 // Unknown flags and --json or debug output stay with the simulator.
484 const flags = [...(call?.flags.keys() ?? [])].filter((flag) => flag.startsWith("--"));
485 if (call?.session === undefined || flags.some((flag) => !FLAGS.has(flag))) return undefined;
486 if (flags.some((flag) => ["--json", "--debug", "--verbose"].includes(flag))) return undefined;
487 const session = sessions.get(call.session);
488 if (call.command === "record" && call.args[0] === "start" && call.appScope) {
489 if (session === undefined && call.selector)
490 return {
491 session: call.session,
492 code: "INVALID_ARGS",
493 message:
494 "record start with app scope and an explicit session requires an active app session; run open <app> first, or use --scope device to record the full screen",
495 hint: "Check command arguments and run --help for usage examples.",
496 };
497 if (session && !session.app && !session.recordingOnly && call.target.iosSimulator)
498 return {
499 session: call.session,
500 code: "UNKNOWN",
501 message:
502 "App-scoped recording requires an active app session; open the app under test before recording.",
503 hint: "Unexpected internal error. Retry with --debug and report the diagnostics log if it persists.",
504 };
505 return undefined;
506 }
507 const owner = holder(call, call.session, sessions, owners);
508 const inUse = owner && inUseError(call, owner);
509 if (inUse !== undefined) return { session: call.session, ...inUse };
510 if (call.command === "snapshot" && call.target.ios && !session?.app)
511 return {
512 session: call.session,
513 code: "SESSION_NOT_FOUND",
514 message: `iOS snapshot requires an active app session on the target device. Run open first (for example: open --session ${session === undefined ? "sim" : call.session} --platform ios --device "<name>" <app>).`,
515 hint: NO_SESSION_HINT,
516 };
517 if (session !== undefined) return undefined;
518 if (NEEDS_SESSION.test(call.command))
519 return {
520 session: call.session,
521 code: "SESSION_NOT_FOUND",
522 message: "No active session. Run open first.",
523 hint: NO_SESSION_HINT,
524 };
525 if (call.command === "close" && call.args.length === 0 && call.selector)
526 return {
527 session: call.session,
528 code: "SESSION_NOT_FOUND",
529 message: "No active session",
530 hint: NO_SESSION_HINT,
531 };
532 return undefined;
533};
534
535const decodeDeviceError = Schema.decodeUnknownOption(DeviceError);
536/** A device tool mock that is a T3 device error, such as `{"_tag": "DeviceOperationError", ...}`. */
537export const deviceFailure = (mock: unknown) => Option.getOrUndefined(decodeDeviceError(mock));
538
539const Injected = Context.Reference<DeviceError | undefined>(
540 "@t3tools/evals/mcp-tool-use/devices/Injected",
541 { defaultValue: () => undefined },
542);
543const failing = <A, R>(effect: Effect.Effect<A, DeviceError, R>) =>
544 Effect.flatMap(Injected, (failure) => (failure === undefined ? effect : Effect.fail(failure)));
545
546// The hosts' own output, which T3 passes on. Launcher scripts embed these paths, never shown.
547const NODE = "/usr/bin/node";
548const HUB = "http://127.0.0.1:47613";
549const PLATFORM_NAMES = { ios: "iOS Simulators", android: "Android Emulators" } as const;
550/** T3 reads only a capture's PNG header; replay has no screens to send. */
551const capture = (platform: DevicePlatform) => {
552 const png = new Uint8Array(24);
553 const view = new DataView(png.buffer);
554 view.setUint32(0, 0x89504e47);
555 view.setUint32(4, 0x0d0a1a0a);
556 view.setUint32(12, 0x49484452);
557 view.setUint32(16, platform === "ios" ? 1206 : 1080);
558 view.setUint32(20, platform === "ios" ? 2622 : 2400);
559 return png;
560};
561/**
562 * What `adb exec-out screencap -p` prints before the PNG on an emulator with several displays,
563 * such as a foldable. T3 passes the hub's bytes on, and its PNG check finds no header.
564 */
565const SCREENCAP_WARNING = new TextEncoder().encode(
566 "[Warning] Multiple displays were found, but no display id was specified! Defaulting to the first display found, however this default is not guaranteed to be consistent across captures. A display id should be specified.\n",
567);
568
569const fakeHost = (
570 world: DeviceWorld,
571 id: DeviceHostId,
572 label: string,
573 agentEntry: string,
574): DeviceHost.DeviceHost["Service"] => {
575 const devices = world.devices.filter((device) => device.hostId === id);
576 const discovery = world.deviceDiscoveryFailures?.[id];
577 const booted = new Map(devices.map((device) => [device.id, device.booted]));
578 const availability = (platform: DevicePlatform) =>
579 devices.some((device) => device.platform === platform)
580 ? { platform, available: true }
581 : { platform, available: false, reason: `${label} has no ${PLATFORM_NAMES[platform]}.` };
582 const ready: DeviceHost.DeviceHostReady = {
583 nodePath: NODE,
584 hub: { origin: HUB },
585 devices: {
586 // Boot state changes as T3 boots and shuts devices down, and later lists show it.
587 list:
588 discovery === undefined
589 ? Effect.sync(() => ({
590 devices: devices.map(({ hostId: _hostId, ...device }) => ({
591 ...device,
592 booted: booted.get(device.id) ?? device.booted,
593 })),
594 detail: undefined,
595 }))
596 : Effect.fail(discovery),
597 boot: (device) =>
598 Effect.sync(() => {
599 booted.set(device.id, true);
600 return device.id;
601 }),
602 attach: () => Effect.void,
603 shutdown: (device) => Effect.sync(() => void booted.set(device.id, false)),
604 screenshot: (device) =>
605 Effect.succeed(
606 world.deviceInvalidCapture
607 ? new Uint8Array([...SCREENCAP_WARNING, ...capture(device.platform)])
608 : capture(device.platform),
609 ),
610 },
611 // T3 checks an Android emulator's host for ffmpeg's H.264 encoder before streaming it.
612 run: () => Effect.succeed({ stdout: " V....D libx264 H.264\n", stderr: "", code: 0 }),
613 helpers: { serveSimAxSettings: null, serveSimCli: null },
614 };
615 const agentDevice = {
616 baseUrl: `${HUB}/api/device-hub/agent/${id}`,
617 token: id,
618 entryPath: agentEntry,
619 };
620 return {
621 id,
622 label,
623 summary: Effect.sync(() => ({
624 id,
625 label,
626 kind: id === LOCAL_DEVICE_HOST_ID ? ("local" as const) : ("ssh" as const),
627 platforms: [availability("ios"), availability("android")],
628 hubInstalled: true,
629 agentDeviceInstalled: true,
630 })),
631 platformAvailability: (platform) => Effect.sync(() => availability(platform)),
632 ensureReady: () => Effect.succeed(ready),
633 ensureAgentReady: () => Effect.succeed({ ...ready, agentDevice }),
634 current: Effect.succeed(ready),
635 stopAgent: Effect.void,
636 stop: Effect.void,
637 };
638};
639
640// Refs before and after T3 binds a launcher to each session run this same code. Optional
641// exports detect the newer API; makeWithHosts ignores a fifth argument it does not take.
642type BoundTarget = { readonly session: string; readonly targetArgs: ReadonlyArray<string> };
643type Launchers = {
644 readonly bind: (input: BoundTarget) => Effect.Effect<string, DeviceError>;
645 readonly prune: (live: ReadonlySet<string>) => Effect.Effect<void>;
646};
647const launcherApi: {
648 readonly ensureAgentDeviceShim: typeof AgentDeviceShim.ensureAgentDeviceShim;
649 readonly bindAgentDeviceLauncher?: (
650 input: Parameters<typeof AgentDeviceShim.ensureAgentDeviceShim>[0] & BoundTarget,
651 ) => ReturnType<typeof AgentDeviceShim.ensureAgentDeviceShim>;
652 readonly pruneAgentDeviceLaunchers?: (input: {
653 readonly stateDir: string;
654 readonly live: ReadonlySet<string>;
655 }) => Effect.Effect<void, never, FileSystem.FileSystem | Path.Path>;
656} = AgentDeviceShim;
657type MakeWithHosts = typeof DeviceService.makeWithHosts;
658const makeWithHosts: (
659 hosts: Parameters<MakeWithHosts>[0],
660 testHost: Parameters<MakeWithHosts>[1],
661 configureAgent: Parameters<MakeWithHosts>[2],
662 installTool: Parameters<MakeWithHosts>[3],
663 launchers?: Launchers,
664) => ReturnType<MakeWithHosts> = DeviceService.makeWithHosts;
665
666/** T3's device tools for one sample. `cwd` names the home that T3's state lives under. */
667export const makeDeviceTools = Effect.fn("evaluateMcpToolUse.makeDeviceTools")(function* (
668 world: DeviceWorld,
669 environment: { readonly cwd: string; readonly platform: string },
670) {
671 const home = /^\/(?:home|Users)\/[^/]+/.exec(environment.cwd)?.[0] ?? "/home/user";
672 for (const id of Object.keys(world.agentDeviceSessions ?? {})) {
673 if (!world.devices.some((device) => device.id === id)) {
674 return yield* Effect.die(new Error(`world.agentDeviceSessions names no device ${id}.`));
675 }
676 }
677 // T3 names each thread's session on a device; another thread's is named the same way.
678 const owners: Array<AgentDeviceOwner> = [];
679 for (const [deviceId, threadId] of Object.entries(world.deviceOwners ?? {})) {
680 const device = world.devices.find((candidate) => candidate.id === deviceId);
681 if (device === undefined) {
682 return yield* Effect.die(new Error(`world.deviceOwners names no device ${deviceId}.`));
683 }
684 const session = yield* agentDeviceSession(threadId, device.hostId, deviceId);
685 owners.push({ deviceId, threadId, session });
686 }
687 const config = ServerConfig.layerTest(environment.cwd, `${home}/.t3`);
688 const files = FileSystem.layerNoop({
689 makeDirectory: () => Effect.void,
690 writeFileString: () => Effect.void,
691 chmod: () => Effect.void,
692 });
693 const devices = Layer.effect(
694 DeviceService.DeviceService,
695 Effect.gen(function* () {
696 const fs = yield* FileSystem.FileSystem;
697 const path = yield* Path.Path;
698 const digest = yield* Digest.Digest;
699 const { stateDir } = yield* ServerConfig.ServerConfig;
700 const agentEntry = path.join(home, ".t3", "tools", "agent-device", "bin", "agent-device.mjs");
701 const hosts = new Map<DeviceHostId, DeviceHost.DeviceHost["Service"]>([
702 [
703 LOCAL_DEVICE_HOST_ID,
704 fakeHost(world, LOCAL_DEVICE_HOST_ID, LOCAL_DEVICE_HOST_LABEL, agentEntry),
705 ],
706 ]);
707 // Hosts with devices, then configured hosts that may have none, such as a failed one.
708 for (const id of [
709 ...world.devices.map((device) => device.hostId),
710 ...Object.keys(world.deviceHosts ?? {}),
711 ...Object.keys(world.deviceDiscoveryFailures ?? {}),
712 ]) {
713 const label = world.deviceHosts?.[id];
714 if (hosts.has(id)) continue;
715 if (label === undefined) {
716 return yield* Effect.die(new Error(`world.deviceHosts has no label for ${id}.`));
717 }
718 hosts.set(id, fakeHost(world, id, label, agentEntry));
719 }
720 // As DeviceService.make, minus writing the config file the path names.
721 const configureAgent = (hostId: DeviceHostId, _endpoint: unknown, deviceId?: string) =>
722 agentDeviceConfigPath(stateDir, hostId, path, deviceId).pipe(
723 Effect.provideService(Digest.Digest, digest),
724 Effect.mapError(
725 (cause) =>
726 new DeviceOperationError({
727 operation: "configure agent",
728 reason: "settings_failed",
729 cause,
730 }),
731 ),
732 );
733 const withFiles = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
734 effect.pipe(
735 Effect.provideService(FileSystem.FileSystem, fs),
736 Effect.provideService(Path.Path, path),
737 );
738 const { bindAgentDeviceLauncher: bind, pruneAgentDeviceLaunchers: prune } = launcherApi;
739 const launchers: Launchers | undefined =
740 bind && prune
741 ? {
742 bind: (input) =>
743 AgentDeviceShim.ensureAgentDeviceShim({ entryPath: agentEntry, stateDir }).pipe(
744 Effect.andThen(bind({ entryPath: agentEntry, stateDir, ...input })),
745 withFiles,
746 Effect.mapError(
747 (cause) =>
748 new DeviceOperationError({
749 operation: "bind agent launcher",
750 reason: "command_failed",
751 cause,
752 }),
753 ),
754 ),
755 prune: (live) => withFiles(prune({ stateDir, live })),
756 }
757 : undefined;
758 const service = yield* makeWithHosts(hosts, undefined, configureAgent, undefined, launchers);
759 return DeviceService.DeviceService.of({
760 ...service,
761 // Older refs install the CLI through agentCli; makeWithHosts has none to install.
762 ...("agentCli" in service ? { agentCli: Effect.succeed(agentEntry) } : {}),
763 list: failing(service.list),
764 open: Effect.fn("DeviceService.DeviceService.open")((input, context) =>
765 failing(service.open(input, context)),
766 ),
767 screenshot: Effect.fn("DeviceService.DeviceService.screenshot")((input) =>
768 failing(service.screenshot(input)),
769 ),
770 close: Effect.fn("DeviceService.DeviceService.close")((input) =>
771 failing(service.close(input)),
772 ),
773 });
774 }),
775 );
776 const context = yield* Layer.build(
777 McpHttpServer.layerDeviceToolkit.pipe(
778 Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
779 Layer.provideMerge(devices),
780 Layer.provideMerge(
781 Layer.mergeAll(
782 liveThreadsLayer,
783 Layer.mock(ProjectService.ProjectService)({ getById: () => Effect.succeedNone }),
784 ServerSettingsService.layerTest({
785 enableDeviceSupport: true,
786 enableAgentDeviceAccess: world.deviceAgentAccess !== false,
787 deviceOnboardingCompleted: true,
788 }),
789 ),
790 ),
791 Layer.provideMerge(config),
792 Layer.provideMerge(files),
793 ),
794 );
795 type Entry = Parameters<McpServer.McpServer["Service"]["addTool"]>[0];
796 // The registry keeps each tool's handler; its public type lists only definitions.
797 const tools = Context.get(context, McpServer.McpServer).tools as ReadonlyArray<Entry>;
798 // The testkit's live thread runs on the `codex` instance; the caller must match it.
799 const scope: McpInvocationContext.McpThreadInvocationScope = {
800 environmentId: EnvironmentId.make("00000000-0000-4000-8000-0000000000e0"),
801 requestNamespace: "mcp-tool-use-eval",
802 thread: {
803 threadId: ThreadId.make("mcp-tool-use-eval"),
804 providerSessionId: "mcp-tool-use-eval",
805 providerInstanceId: ProviderInstanceId.make("codex"),
806 },
807 client: undefined,
808 // A thread holds `device` when agent device access was on as its session started.
809 capabilities: new Set<McpInvocationContext.McpCapability>(
810 world.deviceAgentAccess === false ? [] : ["device"],
811 ),
812 issuedAt: 0,
813 };
814 const request = McpSchema.McpRequestContext.of({
815 clientId: 0,
816 protocolVersion: "2025-06-18",
817 clientCapabilities: {},
818 });
819 const launchers: Array<string> = [];
820 const targets: Array<LauncherTarget> = [];
821 const call = Effect.fn("evaluateMcpToolUse.deviceCall")(function* (
822 name: string,
823 input: unknown,
824 failure?: DeviceError,
825 ) {
826 const entry = tools.find((tool) => tool.tool.name === name);
827 if (entry === undefined) return yield* Effect.die(new Error(`T3 has no device tool ${name}.`));
828 const result = yield* entry.handle(input).pipe(
829 Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
830 Effect.provideService(McpSchema.McpRequestContext, request),
831 Effect.provideService(HostProcess.ExecutablePath, NODE),
832 Effect.provideService(HostProcess.Platform, environment.platform as NodeJS.Platform),
833 Effect.provideService(Injected, failure),
834 // T3 logs each failed screenshot; the result already says so.
835 Effect.provideService(References.MinimumLogLevel, "Error"),
836 // An MCP client shows a protocol error, such as bad parameters, as the tool's error.
837 Effect.catch((error) =>
838 Effect.succeed(
839 new McpSchema.CallToolResult({
840 isError: true,
841 content: [{ type: "text", text: error.message }],
842 }),
843 ),
844 ),
845 );
846 if (!("content" in result)) return yield* Effect.die(new Error(`${name} asked for input.`));
847 const text = result.content
848 .flatMap((part) => (part.type === "text" ? [part.text] : []))
849 .join("\n");
850 // McpServer turns a defect into this text; here a defect is a bug in the eval's world.
851 if (text === "Tool execution failed due to an internal server error.") {
852 return yield* Effect.die(new Error(`${name} hit a defect in T3 or the eval's world.`));
853 }
854 const opened = result.structuredContent;
855 if (name === "device_open" && !result.isError && isOpened(opened)) {
856 const { command, targetArgs = [], session: bound } = opened.agentDevice;
857 launchers.push(command);
858 // Both refs return the session in the target flags; a bound launcher also names it.
859 const flag = targetArgs.indexOf("--session");
860 const session = bound ?? (flag < 0 ? undefined : targetArgs[flag + 1]);
861 const { device } = opened;
862 if (session !== undefined && device !== undefined)
863 targets.push({
864 command,
865 session,
866 bound: bound !== undefined,
867 deviceId: device.id,
868 ios: device.platform === "ios",
869 iosSimulator: device.platform === "ios" && !device.physical,
870 });
871 }
872 return { text, isError: result.isError === true };
873 });
874 const sessions = (history: ReadonlyArray<ShellCall>) =>
875 agentDeviceSessions(targets, world.agentDeviceSessions, history, owners);
876 /** A device as the simulator knows it from the world: its name, id and host. */
877 const deviceName = (deviceId: string) => {
878 const device = world.devices.find((candidate) => candidate.id === deviceId);
879 if (device === undefined) return deviceId;
880 const host =
881 device.hostId === LOCAL_DEVICE_HOST_ID
882 ? LOCAL_DEVICE_HOST_LABEL
883 : (world.deviceHosts?.[device.hostId] ?? device.hostId);
884 return `${device.name}, ${device.id}, on ${host}`;
885 };
886 const launcherText = (command: string) => {
887 const target = targets.findLast((candidate) => candidate.command === command);
888 if (target === undefined) return command;
889 return target.bound
890 ? `${command} (drives ${deviceName(target.deviceId)}, in session ${target.session})`
891 : `${command} (shared: each command's --udid or --serial and --session name its device and session)`;
892 };
893 const describe = (history: ReadonlyArray<ShellCall>, command: string | undefined) => {
894 const followed = followSessions(targets, world.agentDeviceSessions, history, owners);
895 const sessionDevices = new Map([
896 ...targets.map((target) => [target.session, deviceName(target.deviceId)] as const),
897 ...owners.map((owner) => [owner.session, deviceName(owner.deviceId)] as const),
898 ]);
899 // Code, not the simulator, decides which of the call's opens another thread's session refuses.
900 const opens =
901 command === undefined || owners.length === 0
902 ? []
903 : opensIn(targets, followed, command, owners).flatMap(({ call, words, owner, held }) => {
904 const opened = `\`${words.join(" ")}\` drives ${deviceName(call.deviceId)}`;
905 const error = owner && inUseError(call, owner);
906 if (owner !== undefined && error !== undefined)
907 return [
908 `${opened}, which session ${owner.session} holds: it fails with exit code 1, printing "Error (${error.code}): ${error.message}" and "Hint: ${error.hint}"`,
909 ];
910 return held
911 ? []
912 : [`${opened}, which no other session holds: it prints no DEVICE_IN_USE`];
913 });
914 return [
915 ...(launchers.length > 0
916 ? [
917 `agent-device launchers T3's device_open returned: ${[...new Set(launchers)].map(launcherText).join("; ")}`,
918 ]
919 : []),
920 ...(followed.sessions.size > 0
921 ? [
922 `Their agent-device sessions: ${describeSessions(followed.sessions, owners, sessionDevices)}.`,
923 ]
924 : []),
925 ...(opens.length > 0 ? [`In the new call, ${[...new Set(opens)].join("; ")}.`] : []),
926 ];
927 };
928 return {
929 call,
930 launchers,
931 owners,
932 sessions,
933 sessionError: (history, command) => {
934 const error = agentDeviceSessionError(targets, sessions(history), command, owners);
935 return error && agentDeviceError(home, error.session, `${history.length}\n${command}`, error);
936 },
937 describe,
938 } satisfies DeviceTools;
939});
940
941const isOpened = Schema.is(
942 Schema.Struct({
943 device: Schema.optional(
944 Schema.Struct({ id: Schema.String, platform: Schema.String, physical: Schema.Boolean }),
945 ),
946 agentDevice: Schema.Struct({
947 command: Schema.String,
948 targetArgs: Schema.optional(Schema.Array(Schema.String)),
949 session: Schema.optional(Schema.String),
950 }),
951 }),
952);