Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

cases.ts

338 lines10.0 KB

1/**
2 * Cases for the mobile behavior eval. `bad` fails on main today and must pass once fixed;
3 * `guard` passes on main and must keep passing (its issue is fixed, or does not show on iOS);
4 * `control` is ordinary use that must keep passing. Each names the scenario that produces its metrics and the checks on them.
5 */
6import type { Check } from "./grade.ts";
7import type { Pose, ScenarioName } from "./scenarios.ts";
8​
9export interface MobileBehaviorCase {
10 readonly id: string;
11 readonly area: "scroll" | "jank" | "composer" | "status" | "wrong-thread";
12 readonly kind: "bad" | "guard" | "control";
13 /** The issue or PR it came from. */
14 readonly source: string;
15 /** The real bad experience, or the good one a guard protects. */
16 readonly experience: string;
17 readonly scenario: ScenarioName;
18 /** The metric the scoreboard shows. */
19 readonly key: string;
20 readonly checks: ReadonlyArray<Check>;
21 /** Omit to run on every pose. */
22 readonly poses?: ReadonlyArray<Pose>;
24​
25/**
26 * Frame budget checks for jank cases: at most 5% of UI frames and 10% of JS frames dropped.
27 * Swipe-driven cases check JS frames only: the injected swipes cost the simulator's UI thread.
28 */
29const smooth: ReadonlyArray<Check> = [
30 { metric: "uiDroppedPct", max: 5 },
31 { metric: "jsDroppedPct", max: 10 },
32];
33​
34export const CASES: ReadonlyArray<MobileBehaviorCase> = [
35 {
36 id: "scroll-up-never-moves-backward",
37 area: "scroll",
38 kind: "guard",
39 source: "#22350",
40 experience: "Flicking up a long thread, content jerks back down toward the latest message.",
41 scenario: "scrollUpLongThread",
42 key: "backwardPx",
43 checks: [
44 { metric: "movedPx", min: 300 },
45 { metric: "backwardPx", max: 24 },
46 ],
47 },
48 {
49 id: "scroll-up-never-teleports",
50 area: "scroll",
51 kind: "guard",
52 source: "#12072",
53 experience: "Dragging up a little after scroll-to-end, the viewport jumps far away.",
54 scenario: "scrollUpLongThread",
55 key: "largestStepPx",
56 checks: [
57 { metric: "movedPx", min: 300 },
58 { metric: "teleports", equals: 0 },
59 ],
60 },
61 {
62 id: "streaming-stays-at-latest",
63 area: "scroll",
64 kind: "bad",
65 source: "#15469",
66 experience:
67 "While a reply streams at the bottom, the thread jumps far above the latest message.",
68 scenario: "streamInView",
69 key: "settledFromEndPx",
70 checks: [
71 { metric: "settledFromEndPx", max: 64 },
72 { metric: "followingAtEnd", equals: true },
73 ],
74 },
75 {
76 id: "send-in-long-thread-stays-at-end",
77 area: "scroll",
78 kind: "guard",
79 source: "#13706",
80 experience:
81 "Sending in a long thread, the timeline jumps far up instead of showing the new message.",
82 scenario: "sendInLongThread",
83 key: "maxFromEndPx",
84 checks: [
85 { metric: "maxFromEndPx", max: 1500 },
86 { metric: "finalFromEndPx", max: 24 },
87 ],
88 },
89 {
90 id: "background-finish-keeps-scroll",
91 area: "scroll",
92 kind: "guard",
93 source: "#15165",
94 experience: "Another thread finishing resets the open thread's scroll position.",
95 scenario: "backgroundFinish",
96 key: "largestStepPx",
97 checks: [{ metric: "largestStepPx", max: 4 }],
98 },
99 {
100 id: "jank-streaming-long-reply",
101 area: "jank",
102 kind: "bad",
103 source: "#15641, #14010, #19120",
104 experience: "A long reply streaming into a long thread freezes or stutters the app.",
105 scenario: "streamInView",
106 key: "jsDroppedPct",
107 checks: smooth,
108 },
109 {
110 id: "jank-scrolling-settled-thread",
111 area: "jank",
112 kind: "control",
113 source: "#13925 (Android)",
114 experience: "Scrolling a long settled thread drops frames.",
115 scenario: "scrollUpLongThread",
116 key: "jsP99Ms",
117 checks: [
118 { metric: "movedPx", min: 300 },
119 { metric: "jsDroppedPct", max: 10 },
120 ],
121 },
122 {
123 id: "jank-scrolling-subagent-thread",
124 area: "jank",
125 kind: "control",
126 source: "#18376 (Android)",
127 experience: "Scrolling a subagent's thread is far less smooth than its parent.",
128 scenario: "openSubagentThread",
129 key: "jsP99Ms",
130 checks: [
131 { metric: "movedPx", min: 300 },
132 { metric: "jsDroppedPct", max: 10 },
133 ],
134 },
135 {
136 id: "draft-survives-thread-switch",
137 area: "composer",
138 kind: "guard",
139 source: "#17422",
140 experience: "A typed draft is gone after switching threads and coming back.",
141 scenario: "draftThreadSwitch",
142 key: "draftKept",
143 checks: [
144 { metric: "draftKept", equals: true },
145 { metric: "editorShowsDraft", equals: true },
146 ],
147 },
148 {
149 id: "draft-never-leaks-to-other-thread",
150 area: "composer",
151 kind: "guard",
152 source: "#7390",
153 experience: "Text typed in a streaming thread shows up in another thread's composer.",
154 scenario: "draftThreadSwitch",
155 key: "leakedIntoOtherEditor",
156 checks: [
157 { metric: "leakedIntoOtherDraft", equals: false },
158 { metric: "leakedIntoOtherEditor", equals: false },
159 ],
160 },
161 {
162 id: "draft-survives-background",
163 area: "composer",
164 kind: "guard",
165 source: "#17422, #7390",
166 experience: "A typed draft is gone after the app goes to the background and back.",
167 scenario: "draftBackground",
168 key: "draftKept",
169 checks: [
170 { metric: "draftKept", equals: true },
171 { metric: "editorShowsDraft", equals: true },
172 ],
173 },
174 {
175 id: "composer-clears-after-send",
176 area: "composer",
177 kind: "guard",
178 source: "#20982",
179 experience: "After sending, the message text stays in the composer.",
180 scenario: "sendInLongThread",
181 key: "editorCleared",
182 checks: [
183 { metric: "draftCleared", equals: true },
184 { metric: "editorCleared", equals: true },
185 ],
186 },
187 {
188 id: "composer-tap-focuses-text",
189 area: "composer",
190 kind: "guard",
191 source: "#24303",
192 experience: "Tapping the composer's text area opens the model picker instead of the keyboard.",
193 scenario: "tapComposer",
194 key: "editorFocused",
195 checks: [
196 { metric: "stayedOnThread", equals: true },
197 { metric: "editorFocused", equals: true },
198 ],
199 },
200 {
201 id: "composer-keeps-focus-after-command-pick",
202 area: "composer",
203 kind: "guard",
204 source: "#10956",
205 experience: "Picking a skill or command from the / menu closes the keyboard.",
206 scenario: "skillPick",
207 key: "editorFocused",
208 checks: [{ metric: "editorFocused", equals: true }],
209 },
210 {
211 id: "failed-send-is-never-silent",
212 area: "composer",
213 kind: "guard",
214 source: "#15801",
215 experience: "A send the server will reject looks sent, with no error.",
216 scenario: "failedSend",
217 // The cell shows which path ran: Send blocked, or a rejected send with an error.
218 key: "sendBlocked",
219 checks: [{ metric: "silentFailure", equals: false }],
220 },
221 {
222 id: "working-shelf-clears-when-done",
223 area: "status",
224 kind: "bad",
225 source: "#24901",
226 experience: "A finished thread stays in the Working shelf.",
227 scenario: "workingShelfFinish",
228 key: "shelfCountAfter",
229 checks: [
230 { metric: "shelfShownWhileWorking", equals: true },
231 { metric: "shelfCountAfter", equals: 0 },
232 { metric: "statusAfter", equals: "ready" },
233 ],
234 },
235 {
236 id: "pending-question-shows-input",
237 area: "status",
238 kind: "guard",
239 source: "#24901",
240 experience: "A thread waiting on a question shows as Working instead of needing input.",
241 scenario: "pendingQuestion",
242 key: "status",
243 checks: [
244 { metric: "status", equals: "input" },
245 { metric: "shelfCount", equals: 0 },
246 ],
247 },
248 {
249 id: "running-subagents-not-idle",
250 area: "status",
251 kind: "guard",
252 source: "#16464, #4962",
253 experience: "A thread with subagents still running looks idle in the list and the thread.",
254 scenario: "subagentsRunning",
255 key: "status",
256 checks: [
257 { metric: "status", equals: "working" },
258 { metric: "threadShowsStatus", equals: true },
259 ],
260 },
261 {
262 id: "long-press-pin-pins-that-row",
263 area: "wrong-thread",
264 kind: "guard",
265 source: "#20428",
266 experience: "Pin from a row's long-press menu pins a different thread.",
267 scenario: "pinFromLongPress",
268 key: "targetPinned",
269 checks: [
270 { metric: "targetPinned", equals: true },
271 { metric: "otherPinned", equals: 0 },
272 ],
273 },
274 {
275 id: "link-opens-new-thread",
276 area: "wrong-thread",
277 kind: "guard",
278 source: "#14689 (web)",
279 experience: "A link to a just-created thread redirects to another thread.",
280 scenario: "linkToNewThread",
281 key: "opensTarget",
282 checks: [{ metric: "opensTarget", equals: true }],
283 },
284 {
285 id: "link-opens-archived-thread",
286 area: "wrong-thread",
287 kind: "guard",
288 source: "#16443",
289 experience: "A link to an archived thread sends you Home.",
290 scenario: "linkToArchivedThread",
291 key: "opensTarget",
292 checks: [{ metric: "opensTarget", equals: true }],
293 },
294 {
295 id: "folded-duo-back-opens-right-thread",
296 area: "wrong-thread",
297 kind: "guard",
298 source: "#20633",
299 experience: "On a folded iPhone Duo, after going back, the next thread opens off screen.",
300 scenario: "duoBackThenOpen",
301 key: "titleOnScreen",
302 poses: ["duo-closed"],
303 checks: [
304 { metric: "opensTarget", equals: true },
305 { metric: "titleOnScreen", equals: true },
306 ],
307 },
308 {
309 id: "control-open-thread-from-list",
310 area: "wrong-thread",
311 kind: "control",
312 source: "control",
313 experience: "Tapping a row opens that thread.",
314 scenario: "openFromList",
315 key: "opensTarget",
316 checks: [{ metric: "opensTarget", equals: true }],
317 },
318 {
319 id: "control-send-reaches-server",
320 area: "composer",
321 kind: "control",
322 source: "control",
323 experience: "A message sent from the composer reaches the server once.",
324 scenario: "sendInLongThread",
325 key: "serverMessages",
326 checks: [{ metric: "serverMessages", equals: 1 }],
327 },
328 {
329 id: "control-streaming-reply-renders",
330 area: "scroll",
331 kind: "control",
332 source: "control",
333 experience: "A streaming reply grows in the open thread.",
334 scenario: "streamInView",
335 key: "contentGrewPx",
336 checks: [{ metric: "contentGrewPx", min: 500 }],
337 },
338];