Open History to see the last commit
1/** Pure grading for the mobile behavior eval: frame and scroll math, checks, and the summary. */
2import type { FeedSample } from "./device.ts";
3
4export type Metrics = Readonly<Record<string, number | boolean | string | null>>;
5
6export interface Check {
7 readonly metric: string;
8 readonly max?: number;
9 readonly min?: number;
10 readonly equals?: number | boolean | string | null;
11}
12
13const percentile = (sorted: ReadonlyArray<number>, p: number) =>
14 sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]!;
15
16export function median(values: ReadonlyArray<number>): number {
17 const sorted = [...values].sort((a, b) => a - b);
18 const middle = Math.floor(sorted.length / 2);
19 return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
20}
21
22const round = (value: number, digits = 1) => Number(value.toFixed(digits));
23
24/**
25 * Frame times from rAF timestamps, graded against the display's budget: `refreshMs` when given,
26 * else 120 Hz when the fastest frames come under 12 ms and 60 Hz otherwise. A steady low frame
27 * rate still counts as dropped frames. `dropped` counts the frames that should have run inside
28 * each long gap.
29 */
30export function frameStats(
31 timestamps: ReadonlyArray<number>,
32 prefix: string,
33 refreshMs?: number,
34): Metrics {
35 const gaps = timestamps.slice(1).map((time, index) => time - timestamps[index]!);
36 if (gaps.length < 10) return { [`${prefix}Frames`]: gaps.length };
37 const sorted = [...gaps].sort((a, b) => a - b);
38 const refresh = refreshMs ?? (percentile(sorted, 0.05) < 12 ? 1000 / 120 : 1000 / 60);
39 const dropped = gaps.reduce(
40 (total, gap) => total + Math.max(0, Math.round(gap / refresh) - 1),
41 0,
42 );
43 return {
44 [`${prefix}Frames`]: gaps.length,
45 [`${prefix}RefreshHz`]: Math.round(1000 / refresh),
46 [`${prefix}P50Ms`]: round(percentile(sorted, 0.5)),
47 [`${prefix}P95Ms`]: round(percentile(sorted, 0.95)),
48 [`${prefix}P99Ms`]: round(percentile(sorted, 0.99)),
49 [`${prefix}MaxMs`]: round(sorted.at(-1)!),
50 [`${prefix}DroppedPct`]: round((100 * dropped) / (gaps.length + dropped)),
51 };
52}
53
54/** Fastest real fling step: about 8000 pt/s at 60 Hz. A bigger one-frame move is a jump. */
55export const TELEPORT_PX = 400;
56
57/**
58 * How visible rows moved between JS frames. A row's screen y is its list position minus the
59 * scroll offset, so a list that shifts its offset to hold content still reads as no motion.
60 * `direction` is the way rows should move: +1 while dragging content down (scrolling up).
61 */
62export function feedMotion(samples: ReadonlyArray<FeedSample>, direction: 1 | -1 | 0): Metrics {
63 let backward = 0;
64 let largest = 0;
65 let teleports = 0;
66 let moved = 0;
67 for (let index = 1; index < samples.length; index++) {
68 const before = samples[index - 1]!;
69 const after = samples[index]!;
70 const shared = Object.keys(before.rows ?? {}).filter((key) => after.rows?.[key] !== undefined);
71 let step = 0;
72 for (const key of shared) {
73 const rowStep = after.rows![key]! - before.rows![key]!;
74 if (Math.abs(rowStep) > Math.abs(step)) step = rowStep;
75 }
76 // A jump can replace every visible row; with the same content, the offset change is the move.
77 if (shared.length === 0 && before.content === after.content)
78 step = (before.scroll ?? 0) - (after.scroll ?? 0);
79 moved += Math.abs(step);
80 largest = Math.max(largest, Math.abs(step));
81 if (Math.abs(step) > TELEPORT_PX) teleports++;
82 if (direction !== 0) backward = Math.max(backward, -step * direction);
83 }
84 const fromEnd = samples.flatMap((sample) =>
85 sample.fromEnd === undefined ? [] : [sample.fromEnd],
86 );
87 return {
88 samples: samples.length,
89 movedPx: Math.round(moved),
90 backwardPx: Math.round(backward),
91 largestStepPx: Math.round(largest),
92 teleports,
93 maxFromEndPx: fromEnd.length ? Math.max(...fromEnd) : null,
94 finalFromEndPx: fromEnd.at(-1) ?? null,
95 };
96}
97
98export function checkFails(check: Check, metrics: Metrics): string | null {
99 const value = metrics[check.metric];
100 if (value === undefined) return `${check.metric} missing`;
101 if (check.equals !== undefined)
102 return value === check.equals ? null : `${check.metric}=${value}, want ${check.equals}`;
103 if (typeof value !== "number") return `${check.metric}=${value}, want a number`;
104 if (check.max !== undefined && value > check.max)
105 return `${check.metric}=${value} > ${check.max}`;
106 if (check.min !== undefined && value < check.min)
107 return `${check.metric}=${value} < ${check.min}`;
108 return null;
109}
110
111export interface SampleRow {
112 readonly id: string;
113 readonly device: string;
114 readonly sample: number;
115 readonly pass: boolean | null;
116 readonly failures: ReadonlyArray<string>;
117 readonly metrics: Metrics;
118 /** Why the sample has no grade: the scenario failed before it could measure. */
119 readonly error?: string;
120}
121
122export interface CaseResult {
123 readonly id: string;
124 readonly device: string;
125 readonly samples: number;
126 readonly passed: number;
127 readonly errors: number;
128 /**
129 * `null` when too few samples were graded: errors outnumber graded samples, or none ran.
130 * Otherwise a case fails when half or more of its graded samples fail.
131 */
132 readonly pass: boolean | null;
133 readonly key: number | boolean | string | null;
134}
135
136export function caseResult(
137 id: string,
138 device: string,
139 key: string,
140 rows: ReadonlyArray<SampleRow>,
141): CaseResult {
142 const graded = rows.filter((row) => row.pass !== null);
143 const errors = rows.filter((row) => row.error !== undefined).length;
144 const passed = graded.filter((row) => row.pass).length;
145 const values = graded.map((row) => row.metrics[key]);
146 const numbers = values.filter((value): value is number => typeof value === "number");
147 return {
148 id,
149 device,
150 samples: graded.length,
151 passed,
152 errors,
153 pass: graded.length === 0 || errors > graded.length ? null : passed > graded.length / 2,
154 key:
155 numbers.length === values.length && numbers.length > 0
156 ? round(median(numbers))
157 : (values[Math.floor(values.length / 2)] ?? null),
158 };
159}
160
161const mark = (result: CaseResult) =>
162 `${result.pass === null ? "incomplete" : result.pass ? "pass" : "FAIL"} ${result.passed}/${result.samples + result.errors}`;
163
164/** Markdown scoreboard. With a baseline, each row shows the earlier result. */
165export function summarize(input: {
166 readonly cases: ReadonlyArray<{
167 readonly id: string;
168 readonly kind: string;
169 readonly source: string;
170 readonly key: string;
171 }>;
172 readonly results: ReadonlyArray<CaseResult>;
173 readonly baseline?: ReadonlyArray<CaseResult>;
174 readonly header: string;
175}): string {
176 const devices = [...new Set(input.results.map((result) => result.device))];
177 const cell = (result: CaseResult | undefined) =>
178 result ? `${mark(result)} (${result.key ?? "-"})` : "-";
179 const lines = [
180 input.header,
181 "",
182 `| Case | Kind | Source | Key metric | ${devices.join(" | ")} |`,
183 `| --- | --- | --- | --- | ${devices.map(() => "---").join(" | ")} |`,
184 ];
185 for (const testCase of input.cases) {
186 const cells = devices.map((device) => {
187 const now = input.results.find((r) => r.id === testCase.id && r.device === device);
188 const before = input.baseline?.find((r) => r.id === testCase.id && r.device === device);
189 return before ? `${cell(now)} was ${cell(before)}` : cell(now);
190 });
191 if (cells.every((value) => value === "-")) continue;
192 lines.push(
193 `| \`${testCase.id}\` | ${testCase.kind} | ${testCase.source} | ${testCase.key} | ${cells.join(" | ")} |`,
194 );
195 }
196 const ran = input.results.filter((result) => result.pass !== null);
197 const incomplete = input.results.length - ran.length;
198 lines.push(
199 "",
200 `${ran.filter((r) => r.pass).length} of ${ran.length} case runs pass${incomplete ? `, ${incomplete} incomplete` : ""}.`,
201 );
202 return `${lines.join("\n")}\n`;
203}
204
205export function resultsFor(
206 rows: ReadonlyArray<SampleRow>,
207 cases: ReadonlyArray<{ readonly id: string; readonly key: string }>,
208): ReadonlyArray<CaseResult> {
209 const keys = [...new Set(rows.map((row) => `${row.id}\u0000${row.device}`))];
210 return keys.map((key) => {
211 const [id, device] = key.split("\u0000") as [string, string];
212 return caseResult(
213 id,
214 device,
215 cases.find((entry) => entry.id === id)?.key ?? "",
216 rows.filter((row) => row.id === id && row.device === device),
217 );
218 });
219}
220
221/**
222 * Results that should stop a release: a guard or control that failed or could not be graded.
223 * `bad` cases reproduce known bugs and fail until those are fixed.
224 */
225export function blockingResults(
226 results: ReadonlyArray<CaseResult>,
227 cases: ReadonlyArray<{ readonly id: string; readonly kind: string }>,
228): ReadonlyArray<CaseResult> {
229 return results.filter(
230 (result) =>
231 result.pass !== true && cases.find((entry) => entry.id === result.id)?.kind !== "bad",
232 );
233}