1// @effect-diagnostics nodeBuiltinImport:off - Host-side video analysis shells out to ffmpeg.
2
3import * as NodeChildProcess from "node:child_process";
4import * as NodeFSP from "node:fs/promises";
5
6import { check, type Check, type Rect } from "./checks.ts";
7import { run } from "./device.ts";
8
9/** Pixel width of an image or video, to map snapshot coordinates onto its frames. */
10async function pixelWidth(path: string): Promise<number> {
11 const { stdout } = await run("ffprobe", [
12 "-v",
13 "error",
14 "-select_streams",
15 "v:0",
16 "-show_entries",
17 "stream=width",
18 "-of",
19 "csv=p=0",
20 path,
21 ]);
22 return Number(stdout.trim());
23}
24
25/** Downsampled grayscale crops of one region, one per video frame (or one for a PNG). */
26async function grayFrames(input: string, region: Rect, scale: number): Promise<Uint8Array[]> {
27 const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
28 const [x, y, width, height] = [
29 Math.round(region.x * scale),
30 Math.round(region.y * scale),
31 even(region.width * scale),
32 even(region.height * scale),
33 ];
34 const [outWidth, outHeight] = [even(region.width / 3), even(region.height / 3)];
35 const bytes = await new Promise<Buffer>((resolve, reject) => {
36 const child = NodeChildProcess.spawn(
37 "ffmpeg",
38 [
39 "-v",
40 "error",
41 "-i",
42 input,
43 // One output frame per recorded frame, so indices match select=eq(n,…) for evidence.
44 "-fps_mode",
45 "passthrough",
46 "-vf",
47 `crop=${width}:${height}:${x}:${y},scale=${outWidth}:${outHeight},format=gray`,
48 "-f",
49 "rawvideo",
50 "-",
51 ],
52 { stdio: ["ignore", "pipe", "pipe"] },
53 );
54 const chunks: Buffer[] = [];
55 let stderr = "";
56 child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk));
57 child.stderr.on("data", (chunk) => (stderr += chunk));
58 child.on("error", reject);
59 child.on("close", (code) =>
60 code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg failed: ${stderr}`)),
61 );
62 });
63 const size = outWidth * outHeight;
64 return Array.from({ length: Math.floor(bytes.length / size) }, (_, index) =>
65 bytes.subarray(index * size, (index + 1) * size),
66 );
67}
68
69const distance = (a: Uint8Array, b: Uint8Array) => {
70 let sum = 0;
71 for (let index = 0; index < a.length; index++) sum += Math.abs(a[index]! - b[index]!);
72 return sum / a.length;
73};
74
75/** How far a region has moved from its origin toward its destination look, 0 to 1. */
76export interface FrameProgress {
77 readonly nav: number;
78 readonly chrome: number;
79}
80
81/**
82 * A frame lags when the top of the screen is well into the transition while the tab bar
83 * region still looks like the origin: tabs lingering over an entering thread, or a returned
84 * list drawn before its tabs.
85 */
86const chromeLags = (frame: FrameProgress) => frame.nav > 0.3 && frame.chrome < 0.4;
87// One lagging frame is capture jitter; a regression lasts several.
88const LAG_FRAME_LIMIT = 2;
89
90const progress = (origin: number, destination: number) =>
91 origin + destination === 0 ? 0 : origin / (origin + destination);
92
93/**
94 * Checks that the tab bar moves with the screen during a recorded transition. The first and
95 * last frames are the before and after looks, so nothing is compared across capture sources.
96 */
97export async function lagCheck(
98 name: string,
99 input: {
100 readonly video: string;
101 readonly nav: Rect;
102 readonly chrome: Rect;
103 /** Snapshot units across the screen, to scale regions onto the video's pixels. */
104 readonly viewportWidth: number;
105 },
106 evidence: string,
107): Promise<{ check: Check; frame?: string }> {
108 // A recording the device failed to export is a gap in evidence, not a lag.
109 if (!(await NodeFSP.stat(input.video).catch(() => null)))
110 return { check: check(name, true, "no recording; not checked") };
111 const scale = (await pixelWidth(input.video)) / input.viewportWidth;
112 const [navFrames, chromeFrames] = await Promise.all([
113 grayFrames(input.video, input.nav, scale),
114 grayFrames(input.video, input.chrome, scale),
115 ]);
116 const [navOrigin, navDestination] = [navFrames[0], navFrames.at(-1)];
117 const [chromeOrigin, chromeDestination] = [chromeFrames[0], chromeFrames.at(-1)];
118 if (!navOrigin || !navDestination || !chromeOrigin || !chromeDestination)
119 throw new Error(`Could not read frames of ${input.video}.`);
120 const lagging = navFrames.flatMap((nav, index) => {
121 const frame = {
122 nav: progress(distance(nav, navOrigin), distance(nav, navDestination)),
123 chrome: progress(
124 distance(chromeFrames[index]!, chromeOrigin),
125 distance(chromeFrames[index]!, chromeDestination),
126 ),
127 };
128 return chromeLags(frame) ? [index] : [];
129 });
130 if (lagging.length < LAG_FRAME_LIMIT)
131 return { check: check(name, true, `${lagging.length} lagging video frames`) };
132 await run("ffmpeg", [
133 "-v",
134 "error",
135 "-y",
136 "-i",
137 input.video,
138 "-vf",
139 `select=eq(n\\,${lagging[0]})`,
140 "-frames:v",
141 "1",
142 evidence,
143 ]);
144 return {
145 check: check(name, false, `${lagging.length} video frames (first #${lagging[0]})`),
146 frame: evidence,
147 };
148}