1// See docs/internals/mobile-development.md#measuring-renders.
2import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient";
3import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
4import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
5import * as NodeServices from "@effect/platform-node/NodeServices";
6import * as NodeWS from "@effect/platform-node/NodeWS";
7import {
8 CommandId,
9 EnvironmentId,
10 ExecutionEnvironmentDescriptor,
11 ORCHESTRATION_PROTOCOL_HEADER,
12 ORCHESTRATION_PROTOCOL_VERSION,
13 OrchestrationV2ShellSnapshot,
14 ProjectId,
15 ProviderInstanceId,
16 ThreadId,
17 WsRpcGroup,
18 type OrchestrationV2ThreadProjection,
19} from "@t3tools/contracts";
20import * as Console from "effect/Console";
21import * as SecureRandom from "effect/crypto/SecureRandom";
22import * as Effect from "effect/Effect";
23import * as FileSystem from "effect/FileSystem";
24import * as Layer from "effect/Layer";
25import * as Option from "effect/Option";
26import * as Schema from "effect/Schema";
27import * as Stream from "effect/Stream";
28import { Command, Flag } from "effect/cli";
29import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
30import { RpcClient, RpcSerialization } from "effect/rpc";
31import { Socket } from "effect/socket";
32
33const RenderReport = Schema.Struct({
34 commits: Schema.Number,
35 totalRenders: Schema.Number,
36 components: Schema.Array(Schema.Struct({ key: Schema.String, count: Schema.Number })),
37 causes: Schema.Array(Schema.Struct({ key: Schema.String, count: Schema.Number })),
38});
39type RenderReport = typeof RenderReport.Type;
40const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(RenderReport));
41const InspectorTargets = Schema.Array(
42 Schema.Struct({ appId: Schema.String, webSocketDebuggerUrl: Schema.String }),
43);
44const CdpReply = Schema.Struct({
45 id: Schema.Number,
46 result: Schema.optional(
47 Schema.Struct({
48 result: Schema.optional(Schema.Struct({ value: Schema.optional(Schema.Unknown) })),
49 exceptionDetails: Schema.optional(Schema.Unknown),
50 }),
51 ),
52});
53const decodeCdpReply = Schema.decodeUnknownOption(Schema.fromJsonString(CdpReply));
54const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
55
56class MeasureError extends Schema.TaggedError<MeasureError>()("MeasureError", {
57 message: Schema.String,
58}) {}
59
60const STREAM_PROMPT =
61 "Think it through step by step, then write a 3000-word essay on the history of typesetting in plain paragraphs with a few headings. Do not use any tools.";
62
63interface Options {
64 readonly server: string;
65 readonly token: string;
66 readonly metro: string;
67 readonly scheme: string;
68 readonly runs: number;
69 readonly windowSeconds: number;
70 readonly provider: string;
71 readonly model: string;
72 readonly projectRoot: string;
73}
74
75const uuid = Effect.flatMap(SecureRandom.SecureRandom, (random) => random.uuidV4());
76
77const median = (values: ReadonlyArray<number>) => {
78 const sorted = [...values].sort((a, b) => a - b);
79 const middle = Math.floor(sorted.length / 2);
80 return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
81};
82
83const connectInspector = Effect.fn("measure.connectInspector")(function* (metro: string) {
84 const http = yield* HttpClient.HttpClient;
85 const targets = yield* http
86 .get(`${metro}/json/list`)
87 .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(InspectorTargets)));
88 const target = targets.find((entry) => entry.appId.startsWith("com.t3tools.t3code"));
89 if (!target)
90 return yield* new MeasureError({ message: `No T3 Code app is attached to Metro at ${metro}.` });
91 // Expo closes inspector sockets unless Origin matches Metro's advertised URL; pass that URL as --metro.
92 const socket = yield* Socket.makeWebSocket(target.webSocketDebuggerUrl, {
93 headers: { origin: new URL(metro).origin },
94 openTimeout: "10 seconds",
95 });
96 const writer = yield* socket.writer;
97 const reader = yield* socket.reader;
98 let nextId = 0;
99 return Effect.fn("measure.evaluate")(function* (expression: string) {
100 const id = ++nextId;
101 yield* writer.write(
102 encodeJson({ id, method: "Runtime.evaluate", params: { expression, returnByValue: true } }),
103 );
104 while (true) {
105 for (const frame of yield* reader.pull) {
106 const text = typeof frame === "string" ? frame : new TextDecoder().decode(frame);
107 const reply = decodeCdpReply(text);
108 if (Option.isNone(reply) || reply.value.id !== id) continue;
109 if (reply.value.result?.exceptionDetails)
110 return yield* new MeasureError({
111 message: `Evaluation failed: ${encodeJson(reply.value.result.exceptionDetails)}`,
112 });
113 return reply.value.result?.result?.value;
114 }
115 }
116 });
117});
118
119const connectServer = Effect.fn("measure.connectServer")(function* (options: Options) {
120 const http = (yield* HttpClient.HttpClient).pipe(
121 HttpClient.mapRequest((request) =>
122 request.pipe(
123 HttpClientRequest.bearerToken(options.token),
124 HttpClientRequest.setHeader(
125 ORCHESTRATION_PROTOCOL_HEADER,
126 String(ORCHESTRATION_PROTOCOL_VERSION),
127 ),
128 ),
129 ),
130 HttpClient.filterStatusOk,
131 );
132 const descriptor = yield* http
133 .get(`${options.server}/.well-known/t3/environment`)
134 .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)));
135 const shell = yield* http
136 .get(`${options.server}/api/orchestration/shell`)
137 .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(OrchestrationV2ShellSnapshot)));
138 let projectId = shell.projects.find(
139 (project) => project.workspaceRoot === options.projectRoot,
140 )?.id;
141 // A project record outlives its folder, and turns fail without it.
142 yield* (yield* FileSystem.FileSystem).makeDirectory(options.projectRoot, { recursive: true });
143 if (!projectId) {
144 projectId = ProjectId.make(yield* uuid);
145 yield* http.execute(
146 HttpClientRequest.post(`${options.server}/api/projects/mutate`).pipe(
147 HttpClientRequest.bodyJsonUnsafe({
148 type: "project.create",
149 commandId: yield* uuid,
150 projectId,
151 title: "Render measurement",
152 workspaceRoot: options.projectRoot,
153 }),
154 ),
155 );
156 }
157 const ticket = yield* http
158 .execute(HttpClientRequest.post(`${options.server}/api/auth/websocket-ticket`))
159 .pipe(
160 Effect.flatMap(HttpClientResponse.schemaBodyJson(Schema.Struct({ ticket: Schema.String }))),
161 );
162 // Built into the caller's scope: `Effect.provide` would close the socket once `make` returns.
163 const protocol = yield* Layer.build(
164 RpcClient.layerProtocolSocket().pipe(
165 Layer.provide(
166 Socket.layerWebSocket(
167 `${options.server.replace(/^http/, "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL_VERSION}&wsTicket=${encodeURIComponent(ticket.ticket)}`,
168 ),
169 ),
170 Layer.provide(RpcSerialization.layerJson),
171 ),
172 );
173 const client = yield* RpcClient.make(WsRpcGroup).pipe(Effect.provideContext(protocol));
174 return { client, environmentId: EnvironmentId.make(descriptor.environmentId), projectId };
175});
176
177// Some providers stream only reasoning, so any streaming text item counts.
178const streamingItem = (projection: OrchestrationV2ThreadProjection) =>
179 projection.turnItems.find(
180 (item) => (item.type === "assistant_message" || item.type === "reasoning") && item.streaming,
181 );
182
183const measureStreamReply = Effect.fn("measure.streamReply")(function* (
184 options: Options,
185 run: number,
186 route: Option.Option<string>,
187) {
188 if (Option.isSome(route)) yield* reloadAndOpen(options, route.value);
189 const evaluate = yield* connectInspector(options.metro);
190 const { client, environmentId, projectId } = yield* connectServer(options);
191 const threadId = ThreadId.make(yield* uuid);
192 yield* client["orchestration.launchThread"]({
193 commandId: CommandId.make(yield* uuid),
194 threadId,
195 projectId,
196 title: `Render measurement ${run}`,
197 modelSelection: {
198 instanceId: ProviderInstanceId.make(options.provider),
199 model: options.model,
200 },
201 // The prompt needs no tools; approval-required keeps a stray call from acting.
202 runtimeMode: "approval-required",
203 interactionMode: "default",
204 workspaceStrategy: { type: "root" },
205 initialMessage: { text: STREAM_PROMPT, attachments: [] },
206 });
207 // A failed or interrupted window must not leave a provider turn running.
208 yield* Effect.addFinalizer(() =>
209 Effect.gen(function* () {
210 const after = yield* client["orchestration.getThreadProjection"]({ threadId });
211 const activeRun = after.runs.find(
212 (entry) => !["completed", "failed", "interrupted", "cancelled"].includes(entry.status),
213 );
214 if (!activeRun) return;
215 yield* client["orchestration.dispatchCommand"]({
216 type: "run.interrupt",
217 commandId: CommandId.make(yield* uuid),
218 threadId,
219 runId: activeRun.id,
220 });
221 }).pipe(Effect.ignore),
222 );
223 if (Option.isNone(route))
224 yield* evaluate(
225 `__t3RenderCounterOpen(${encodeJson(`${options.scheme}://threads/${environmentId}/${threadId}`)})`,
226 );
227 const projectionOf = (item: { kind: string; projection?: OrchestrationV2ThreadProjection }) =>
228 item.kind === "snapshot" && item.projection
229 ? Effect.succeed(item.projection)
230 : client["orchestration.getThreadProjection"]({ threadId });
231 const streaming = yield* client["orchestration.subscribeThread"]({ threadId }).pipe(
232 Stream.mapEffect(projectionOf),
233 Stream.filter((projection) => streamingItem(projection) !== undefined),
234 Stream.runHead,
235 Effect.timeout("90 seconds"),
236 );
237 if (Option.isNone(streaming))
238 return yield* new MeasureError({ message: `Run ${run}: the reply never started streaming.` });
239 yield* evaluate("__t3RenderCounterStart()");
240 yield* Effect.sleep(`${options.windowSeconds} seconds`);
241 const report = yield* decodeReport(yield* evaluate("JSON.stringify(__t3RenderCounterStop())"));
242 // An in-view streaming thread must commit; out-of-view streaming may commit nothing.
243 if (report.commits === 0 && Option.isNone(route))
244 return yield* new MeasureError({
245 message: `Run ${run}: no React commits in the window. Check the device for a red box.`,
246 });
247 const after = yield* client["orchestration.getThreadProjection"]({ threadId });
248 return { report, stillStreaming: streamingItem(after) !== undefined };
249}, Effect.scoped);
250
251const appLink = (options: Options, route: string) =>
252 `__t3RenderCounterOpen(${encodeJson(`${options.scheme}://${route.replace(/^\//, "")}`)})`;
253
254const reloadAndOpen = Effect.fn("measure.reloadAndOpen")(function* (
255 options: Options,
256 route: string,
257) {
258 // A link to an already-open screen only refocuses it, so each run starts from a reload.
259 yield* Effect.scoped(
260 Effect.flatMap(connectInspector(options.metro), (evaluate) =>
261 evaluate("__t3RenderCounterReload()"),
262 ),
263 );
264 yield* Effect.sleep("15 seconds");
265 if (route === "/" || route === "") return;
266 yield* Effect.scoped(
267 Effect.flatMap(connectInspector(options.metro), (evaluate) =>
268 evaluate(appLink(options, route)),
269 ),
270 );
271 yield* Effect.sleep("5 seconds");
272});
273
274const measureRouteOpen = Effect.fn("measure.routeOpen")(function* (
275 options: Options,
276 route: string,
277 run: number,
278) {
279 yield* Effect.scoped(
280 Effect.flatMap(connectInspector(options.metro), (evaluate) =>
281 evaluate("__t3RenderCounterReload()"),
282 ),
283 );
284 yield* Effect.sleep("15 seconds");
285 const evaluate = yield* connectInspector(options.metro);
286 yield* evaluate("__t3RenderCounterStart()");
287 yield* evaluate(appLink(options, route));
288 yield* Effect.sleep(`${options.windowSeconds} seconds`);
289 const report = yield* decodeReport(yield* evaluate("JSON.stringify(__t3RenderCounterStop())"));
290 if (report.commits === 0)
291 return yield* new MeasureError({
292 message: `Run ${run}: no React commits in the window. Check that ${route} is a valid app path.`,
293 });
294 return { report, stillStreaming: undefined };
295}, Effect.scoped);
296
297const medianTable = (
298 title: string,
299 reports: ReadonlyArray<RenderReport>,
300 pick: (report: RenderReport) => RenderReport["causes"],
301 limit: number,
302) => {
303 const keys = new Set(reports.flatMap((report) => pick(report).map((entry) => entry.key)));
304 const rows = [...keys]
305 .map((key) => ({
306 key,
307 count: median(reports.map((report) => pick(report).find((e) => e.key === key)?.count ?? 0)),
308 }))
309 .filter((row) => row.count > 0)
310 .sort((a, b) => b.count - a.count)
311 .slice(0, limit);
312 return [
313 `| ${title} | Renders |`,
314 "|---|---:|",
315 ...rows.map((r) => `| \`${r.key}\` | ${r.count} |`),
316 ];
317};
318
319const formatTable = (reports: ReadonlyArray<RenderReport>) => {
320 const totals = (pick: (report: RenderReport) => number) => median(reports.map(pick));
321 return [
322 `Median of ${reports.length} runs.`,
323 "",
324 "| | |",
325 "|---|---:|",
326 `| renders | ${totals((r) => r.totalRenders)} |`,
327 `| commits | ${totals((r) => r.commits)} |`,
328 // Provider pace sets how many commits land in the window; per-commit cost compares across runs.
329 `| renders per commit | ${totals((r) => Math.round(r.totalRenders / Math.max(r.commits, 1)))} |`,
330 "",
331 ...medianTable("Component", reports, (r) => r.components, 20),
332 "",
333 ...medianTable("Cause", reports, (r) => r.causes, 15),
334 ].join("\n");
335};
336
337const command = Command.make(
338 "measure-renders",
339 {
340 server: Flag.String("server").pipe(
341 Flag.withDescription("T3 server origin the app is paired with."),
342 Flag.withDefault(""),
343 ),
344 token: Flag.String("token").pipe(
345 Flag.withDescription("Bearer token from `t3 auth session issue --token-only`."),
346 Flag.withDefault(""),
347 ),
348 route: Flag.String("route").pipe(
349 Flag.withDescription(
350 "App path to open and measure. With --server, the path stays open while a new thread streams.",
351 ),
352 Flag.optional,
353 ),
354 metro: Flag.String("metro").pipe(Flag.withDefault("http://127.0.0.1:8081")),
355 scheme: Flag.String("scheme").pipe(Flag.withDefault("t3code-dev")),
356 runs: Flag.Int("runs").pipe(Flag.withDefault(3)),
357 window: Flag.Int("window").pipe(
358 Flag.withDescription("Seconds to record; defaults to 12 for a reply and 6 for --route."),
359 Flag.optional,
360 ),
361 provider: Flag.String("provider").pipe(Flag.withDefault("claudeAgent")),
362 model: Flag.String("model").pipe(Flag.withDefault("claude-haiku-5-5")),
363 projectRoot: Flag.String("project-root").pipe(
364 Flag.withDescription("Scratch directory for the measurement project; created if missing."),
365 Flag.withDefault(""),
366 ),
367 },
368 Effect.fn("measure.main")(function* (flags) {
369 const route = flags.route;
370 const options: Options = {
371 ...flags,
372 windowSeconds: Option.getOrElse(flags.window, () =>
373 Option.isSome(route) && !flags.server ? 6 : 12,
374 ),
375 };
376 const streams = Option.isNone(route) || flags.server !== "";
377 if (streams && (!flags.server || !flags.token || !flags.projectRoot))
378 return yield* new MeasureError({
379 message: "Streaming a reply needs --server, --token and --project-root.",
380 });
381 const reports: RenderReport[] = [];
382 for (let run = 1; run <= flags.runs; run++) {
383 const { report, stillStreaming } =
384 Option.isSome(route) && !streams
385 ? yield* measureRouteOpen(options, route.value, run)
386 : yield* measureStreamReply(options, run, route);
387 yield* Console.error(
388 `run ${run}: ${report.totalRenders} renders in ${report.commits} commits${stillStreaming === false ? " (warning: streaming ended inside the window)" : ""}`,
389 );
390 reports.push(report);
391 // Let the interrupted turn settle so the next run starts from a quiet app.
392 yield* Effect.sleep("3 seconds");
393 }
394 yield* Console.log(formatTable(reports));
395 }),
396);
397
398Command.run(command, { version: "0.0.0" }).pipe(
399 Effect.provide(
400 Layer.mergeAll(
401 NodeServices.layer,
402 NodeCrypto.layer,
403 NodeHttpClient.layerUndici,
404 NodeWS.layerWebSocketConstructor,
405 ),
406 ),
407 NodeRuntime.runMain,
408);