Open History to see the last commit
1// @effect-diagnostics globalTimers:off globalFetch:off - This browser and WebView transport runs without an Effect runtime.
2import { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts";
3import {
4 PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE,
5 type PreviewStreamHostSetup,
6 type PreviewViewportSetting,
7} from "@t3tools/contracts";
8
9export const PREVIEW_STREAM_BASE_PATH = "/api/preview-stream";
10
11export interface PreviewStreamViewport {
12 readonly width: number;
13 readonly height: number;
14}
15
16export type PreviewStreamMouseButton = "none" | "left" | "middle" | "right";
17
18export interface PreviewStreamControl {
19 readonly canOperate: boolean;
20 readonly controller: "agent" | "you" | "another-viewer" | "unclaimed";
21 readonly generation: number;
22 readonly dialog: null | {
23 readonly type: string;
24 readonly message: string;
25 readonly defaultValue: string;
26 };
27}
28
29export const previewStreamControlLabel = (control: PreviewStreamControl | null): string =>
30 !control
31 ? "Connecting..."
32 : !control.canOperate
33 ? "Read-only"
34 : control.controller === "you"
35 ? "You have control"
36 : control.controller === "agent"
37 ? "Agent has control"
38 : control.controller === "another-viewer"
39 ? "Another viewer has control"
40 : "Watching";
41
42const isPreviewStreamDialog = (value: unknown): value is PreviewStreamControl["dialog"] =>
43 value === null ||
44 (typeof value === "object" &&
45 "type" in value &&
46 typeof value.type === "string" &&
47 "message" in value &&
48 typeof value.message === "string" &&
49 "defaultValue" in value &&
50 typeof value.defaultValue === "string");
51
52/** Coordinates are page CSS px. */
53export type PreviewStreamInput =
54 | { readonly type: "takeControl" }
55 | { readonly type: "releaseControl" }
56 | { readonly type: "dialog"; readonly accept: boolean; readonly promptText?: string }
57 | { readonly type: "viewport"; readonly setting: PreviewViewportSetting }
58 | {
59 readonly type: "mouse";
60 readonly action: "move" | "down" | "up";
61 readonly x: number;
62 readonly y: number;
63 readonly button: PreviewStreamMouseButton;
64 /** Pressed buttons: left 1, right 2, middle 4. */
65 readonly buttons: number;
66 readonly clickCount: number;
67 readonly modifiers: number;
68 }
69 | {
70 readonly type: "wheel";
71 readonly x: number;
72 readonly y: number;
73 readonly deltaX: number;
74 readonly deltaY: number;
75 readonly modifiers: number;
76 }
77 | {
78 readonly type: "key";
79 readonly action: "down" | "up";
80 readonly key: string;
81 readonly code: string;
82 /** Windows virtual key code (DOM `keyCode`); Enter, Backspace, and arrows need it. */
83 readonly keyCode?: number;
84 /** Only for printable keys without Ctrl or Meta. */
85 readonly text?: string;
86 readonly modifiers: number;
87 }
88 | { readonly type: "text"; readonly text: string }
89 | { readonly type: "resize"; readonly width: number; readonly height: number }
90 | { readonly type: "navigate"; readonly url: string }
91 | { readonly type: "history"; readonly delta: -1 | 1 }
92 | { readonly type: "reload"; readonly ignoreCache?: boolean }
93 | { readonly type: "probe"; readonly x: number; readonly y: number };
94
95export interface PreviewStreamDownload {
96 readonly fileName: string;
97 readonly sizeBytes: number;
98 /** Authenticated with the stream's own access; cookie sessions must send credentials. */
99 readonly url: string;
100}
101
102export interface PreviewStreamFileChooser {
103 readonly multiple: boolean;
104 readonly accept: string;
105 /** POST multipart `file` parts here; an empty form cancels the page's picker. */
106 readonly uploadUrl: string;
107 readonly credentials: boolean;
108}
109
110const previewStreamUploadUrl = (
111 target: Pick<PreviewStreamTarget, "access" | "threadId" | "tabId">,
112 chooser: string,
113): string =>
114 withDeviceHubQuery(
115 `${target.access.httpBase}/upload?${new URLSearchParams({
116 threadId: target.threadId,
117 tabId: target.tabId,
118 chooser,
119 }).toString()}`,
120 target.access,
121 );
122
123export async function uploadPreviewStreamFiles(
124 chooser: PreviewStreamFileChooser,
125 files: ReadonlyArray<Blob & { readonly name?: string }>,
126): Promise<void> {
127 const body = new FormData();
128 for (const file of files) body.append("file", file, file.name ?? "file");
129 const response = await fetch(chooser.uploadUrl, {
130 method: "POST",
131 body,
132 credentials: chooser.credentials ? "include" : "omit",
133 });
134 if (!response.ok) throw new Error((await response.text()) || "The upload was refused.");
135}
136
137export const previewStreamDownloadUrl = (
138 target: Pick<PreviewStreamTarget, "access" | "threadId" | "tabId">,
139 id: string,
140): string =>
141 withDeviceHubQuery(
142 `${target.access.httpBase}/download?${new URLSearchParams({
143 threadId: target.threadId,
144 tabId: target.tabId,
145 id,
146 }).toString()}`,
147 target.access,
148 );
149
150export interface PreviewStreamProbe {
151 readonly x: number;
152 readonly y: number;
153 readonly editable: boolean;
154}
155
156export interface PreviewStreamPointer {
157 readonly phase: "move" | "click";
158 readonly x: number;
159 readonly y: number;
160 readonly sequence: number;
161}
162
163export const previewStreamModifiers = (event: {
164 readonly altKey: boolean;
165 readonly ctrlKey: boolean;
166 readonly metaKey: boolean;
167 readonly shiftKey: boolean;
168}): number =>
169 (event.altKey ? 1 : 0) |
170 (event.ctrlKey ? 2 : 0) |
171 (event.metaKey ? 4 : 0) |
172 (event.shiftKey ? 8 : 0);
173
174export interface PreviewStreamTarget {
175 readonly access: DeviceHubAccess;
176 readonly threadId: string;
177 readonly tabId: string;
178 /** Viewer backing-store size in device px. The server never sends larger frames. */
179 readonly maxWidth: number;
180 readonly maxHeight: number;
181 /** Passive viewers reduce their own access, including automatic control grants. */
182 readonly interactive?: boolean;
183}
184
185export interface PreviewStreamEvents {
186 readonly onFrame: (jpeg: ArrayBuffer) => void;
187 readonly onViewport: (viewport: PreviewStreamViewport) => void;
188 readonly onProbe?: (probe: PreviewStreamProbe) => void;
189 readonly onPointer?: (pointer: PreviewStreamPointer) => void;
190 readonly onControl?: (control: PreviewStreamControl) => void;
191 readonly onClipboard?: (text: string) => void;
192 readonly onDownload?: (download: PreviewStreamDownload) => void;
193 readonly onPopup?: (tabId: string) => void;
194 readonly onFileChooser?: (chooser: PreviewStreamFileChooser | null) => void;
195 readonly onConnectedChange: (connected: boolean) => void;
196 /** The upgrade was refused; refresh access and start a new client. */
197 readonly onUnauthorized: () => void;
198 readonly onGone?: () => void;
199 readonly onHostSetup?: (setup: PreviewStreamHostSetup) => void;
200}
201
202export interface PreviewStreamClient {
203 readonly send: (input: PreviewStreamInput) => boolean;
204 readonly stop: () => void;
205}
206
207const ACK_MESSAGE = JSON.stringify({ type: "ack" });
208export function createPreviewStreamClient(
209 target: PreviewStreamTarget,
210 events: PreviewStreamEvents,
211): PreviewStreamClient {
212 const query = new URLSearchParams({
213 threadId: target.threadId,
214 tabId: target.tabId,
215 maxWidth: String(Math.max(1, Math.round(target.maxWidth))),
216 maxHeight: String(Math.max(1, Math.round(target.maxHeight))),
217 });
218 if (target.interactive === false) query.set("interactive", "false");
219 const url = withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access);
220 let stopped = false;
221 let socket: WebSocket | null = null;
222 let retryTimer: ReturnType<typeof setTimeout> | null = null;
223 let failures = 0;
224 let control: PreviewStreamControl | null = null;
225 let fileChooser: string | null = null;
226
227 // The server re-offers a picker still open, under a new id, when this viewer reattaches.
228 const withdrawFileChooser = () => {
229 if (fileChooser === null) return;
230 fileChooser = null;
231 events.onFileChooser?.(null);
232 };
233
234 const connect = () => {
235 if (stopped) return;
236 const ws = new WebSocket(url);
237 ws.binaryType = "arraybuffer";
238 socket = ws;
239 let opened = false;
240 ws.addEventListener("open", () => {
241 if (socket !== ws) return;
242 opened = true;
243 events.onConnectedChange(true);
244 });
245 ws.addEventListener("message", (event) => {
246 if (socket !== ws) return;
247 if (event.data instanceof ArrayBuffer) {
248 failures = 0;
249 ws.send(ACK_MESSAGE);
250 events.onFrame(event.data);
251 return;
252 }
253 if (typeof event.data !== "string") return;
254 let message: unknown;
255 try {
256 message = JSON.parse(event.data);
257 } catch {
258 return;
259 }
260 if (typeof message !== "object" || message === null) return;
261 const {
262 type,
263 x,
264 y,
265 width,
266 height,
267 editable,
268 canOperate,
269 controller,
270 generation,
271 dialog,
272 text,
273 id,
274 fileName,
275 sizeBytes,
276 multiple,
277 accept,
278 phase,
279 sequence,
280 tabId,
281 } = message as Record<string, unknown>;
282 if (
283 type === "fileChooser" &&
284 typeof id === "string" &&
285 typeof multiple === "boolean" &&
286 typeof accept === "string"
287 ) {
288 fileChooser = id;
289 events.onFileChooser?.({
290 multiple,
291 accept,
292 uploadUrl: previewStreamUploadUrl(target, id),
293 credentials: target.access.credentials,
294 });
295 } else if (type === "fileChooserClosed" && typeof id === "string") {
296 if (fileChooser !== id) return;
297 fileChooser = null;
298 events.onFileChooser?.(null);
299 } else if (
300 type === "pointer" &&
301 (phase === "move" || phase === "click") &&
302 typeof x === "number" &&
303 typeof y === "number" &&
304 typeof sequence === "number"
305 ) {
306 events.onPointer?.({ phase, x, y, sequence });
307 } else if (type === "popup" && typeof tabId === "string") {
308 events.onPopup?.(tabId);
309 } else if (type === "clipboard" && typeof text === "string") {
310 events.onClipboard?.(text);
311 } else if (
312 type === "download" &&
313 typeof id === "string" &&
314 typeof fileName === "string" &&
315 typeof sizeBytes === "number"
316 ) {
317 events.onDownload?.({
318 fileName,
319 sizeBytes,
320 url: previewStreamDownloadUrl(target, id),
321 });
322 } else if (type === "viewport" && typeof width === "number" && typeof height === "number") {
323 failures = 0;
324 events.onViewport({ width, height });
325 } else if (
326 type === "probe" &&
327 typeof x === "number" &&
328 typeof y === "number" &&
329 typeof editable === "boolean"
330 ) {
331 events.onProbe?.({ x, y, editable });
332 } else if (
333 type === "control" &&
334 typeof canOperate === "boolean" &&
335 (controller === "agent" ||
336 controller === "you" ||
337 controller === "another-viewer" ||
338 controller === "unclaimed") &&
339 typeof generation === "number" &&
340 isPreviewStreamDialog(dialog)
341 ) {
342 const nextControl: PreviewStreamControl = {
343 canOperate,
344 controller,
345 generation,
346 dialog,
347 };
348 control = nextControl;
349 events.onControl?.(nextControl);
350 }
351 });
352 ws.addEventListener("close", (event) => {
353 if (socket !== ws) return;
354 socket = null;
355 control = null;
356 withdrawFileChooser();
357 if (opened) events.onConnectedChange(false);
358 if (stopped) return;
359 if (event.code === 4404) {
360 stopped = true;
361 events.onGone?.();
362 return;
363 }
364 if (event.code === PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE) {
365 stopped = true;
366 events.onHostSetup?.(decodeHostSetup(event.reason));
367 return;
368 }
369 // Rejected upgrades surface as 1006 before open for both cookies and tickets.
370 if (event.code === 1008 || event.code === 4401 || (!opened && event.code === 1006)) {
371 stopped = true;
372 events.onUnauthorized();
373 return;
374 }
375 retryTimer = setTimeout(connect, Math.min(500 * 2 ** failures++, 10_000));
376 });
377 ws.addEventListener("error", () => ws.close());
378 };
379
380 connect();
381
382 return {
383 send: (input) => {
384 if (socket?.readyState !== WebSocket.OPEN) return false;
385 if (!control?.canOperate) return false;
386 if (input.type !== "takeControl" && control.controller !== "you") return false;
387 socket.send(JSON.stringify(input));
388 return true;
389 },
390 stop: () => {
391 stopped = true;
392 if (retryTimer !== null) clearTimeout(retryTimer);
393 const ws = socket;
394 socket = null;
395 control = null;
396 withdrawFileChooser();
397 ws?.close();
398 },
399 };
400}
401
402export interface PreviewFramePainter {
403 readonly paint: (jpeg: ArrayBuffer) => void;
404 readonly stop: () => void;
405}
406
407/** One decode at a time, keeping only the latest waiting frame for slow viewers. */
408export function createPreviewFramePainter(
409 canvas: HTMLCanvasElement,
410 onPainted?: () => void,
411): PreviewFramePainter {
412 const context = canvas.getContext("2d");
413 let stopped = false;
414 let decoding = false;
415 let waiting: ArrayBuffer | null = null;
416 const draw = (jpeg: ArrayBuffer) => {
417 decoding = true;
418 void createImageBitmap(new Blob([jpeg], { type: "image/jpeg" }))
419 .then((bitmap) => {
420 if (stopped || !context) {
421 bitmap.close();
422 return;
423 }
424 if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) {
425 canvas.width = bitmap.width;
426 canvas.height = bitmap.height;
427 }
428 context.drawImage(bitmap, 0, 0);
429 bitmap.close();
430 onPainted?.();
431 })
432 .catch(() => undefined)
433 .finally(() => {
434 decoding = false;
435 const next = waiting;
436 waiting = null;
437 if (next && !stopped) draw(next);
438 });
439 };
440 return {
441 paint: (jpeg) => {
442 if (stopped) return;
443 if (decoding) waiting = jpeg;
444 else draw(jpeg);
445 },
446 stop: () => {
447 stopped = true;
448 waiting = null;
449 },
450 };
451}
452
453const decodeHostSetup = (reason: string): PreviewStreamHostSetup => {
454 try {
455 const value = JSON.parse(reason) as Partial<PreviewStreamHostSetup>;
456 if (
457 (value.need === "sandbox" || value.need === "libraries") &&
458 typeof value.command === "string"
459 ) {
460 return { need: value.need, command: value.command };
461 }
462 } catch {}
463 return { need: "sandbox", command: "sudo npx t3 browser setup" };
464};
465
466export const previewStreamHostSetupMessage = (setup: PreviewStreamHostSetup) =>
467 setup.need === "sandbox"
468 ? "This server's host blocks the sandbox its browser runs in. Run this once on the host, then try again:"
469 : "This server's host is missing libraries its browser needs. Run this once on the host, then try again:";