Open History to see the last commit
1import {
2 createPreviewFramePainter,
3 createPreviewStreamClient,
4 previewStreamControlLabel,
5 previewStreamModifiers,
6 type PreviewStreamClient,
7 type PreviewStreamControl,
8 type PreviewStreamDownload,
9 type PreviewStreamFileChooser,
10 uploadPreviewStreamFiles,
11 type PreviewStreamInput,
12 type PreviewStreamMouseButton,
13 type PreviewStreamPointer,
14 type PreviewStreamViewport,
15 previewStreamHostSetupMessage,
16} from "@t3tools/client-runtime/preview/server-browser-stream";
17import type {
18 EnvironmentId,
19 PreviewStreamHostSetup,
20 PreviewViewportSetting,
21} from "@t3tools/contracts";
22import {
23 type ClipboardEvent,
24 type FormEvent,
25 type KeyboardEvent,
26 type MouseEvent,
27 type PointerEvent,
28 type Ref,
29 useEffect,
30 useEffectEvent,
31 useImperativeHandle,
32 useRef,
33 useState,
34} from "react";
35
36import { AgentCursorMark } from "~/components/preview/AgentBrowserCursor";
37import { CommandBlock } from "~/components/CommandBlock";
38import { Button } from "~/components/ui/button";
39import { Input } from "~/components/ui/input";
40import { toastManager } from "~/components/ui/toast";
41import { cn } from "~/lib/utils";
42import { refreshPreviewStreamAccess, usePreviewStreamAccess } from "~/state/previewStream";
43import { observeResize } from "~/lib/observeResize";
44
45export interface ServerBrowserHandle {
46 readonly navigate: (url: string) => void;
47 readonly history: (delta: -1 | 1) => void;
48 readonly reload: (options?: { readonly ignoreCache?: boolean }) => void;
49 readonly stopLoading: () => void;
50 readonly viewport: (setting: PreviewViewportSetting) => void;
51 readonly canvas: () => HTMLCanvasElement | null;
52}
53
54const RESIZE_THROTTLE_MS = 150;
55// Reuse nearby probes when a slow link would return the tap's answer after it ends.
56const PROBE_REUSE_PX = 24;
57const PROBE_REUSE_MS = 10_000;
58// Android keyboards report Backspace as text deletion, so keep a deletable sentinel.
59const INPUT_SENTINEL = "\u200b";
60const BACKSPACE = { key: "Backspace", code: "Backspace", keyCode: 8 } as const;
61const DELETE = { key: "Delete", code: "Delete", keyCode: 46 } as const;
62const COPY = { key: "c", code: "KeyC", keyCode: 67 } as const;
63const CUT = { key: "x", code: "KeyX", keyCode: 88 } as const;
64const CTRL = 2;
65const MAX_UNAUTHORIZED_REFUSALS = 3;
66const TAP_SLOP_PX = 8;
67const MULTI_CLICK_MS = 500;
68const MULTI_CLICK_SLOP_PX = 4;
69const WHEEL_LINE_PX = 16;
70
71type MouseInput = Extract<PreviewStreamInput, { type: "mouse" }>;
72type WheelInput = Extract<PreviewStreamInput, { type: "wheel" }>;
73
74interface PagePoint {
75 readonly x: number;
76 readonly y: number;
77 readonly scale: number;
78}
79
80interface TouchGesture {
81 readonly pointerId: number;
82 readonly startX: number;
83 readonly startY: number;
84 lastX: number;
85 lastY: number;
86 panning: boolean;
87}
88
89interface TouchProbe {
90 readonly x: number;
91 readonly y: number;
92 editable: boolean | null;
93 answered: boolean;
94 tapped: boolean;
95}
96
97const buttonOf = (button: number): PreviewStreamMouseButton =>
98 button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none";
99
100const pressedButtonOf = (buttons: number): PreviewStreamMouseButton =>
101 buttons & 1 ? "left" : buttons & 2 ? "right" : buttons & 4 ? "middle" : "none";
102
103// Mouse thumb buttons: `button === 3` is Back, `button === 4` is Forward.
104const historyDeltaOf = (button: number): -1 | 1 | null =>
105 button === 3 ? -1 : button === 4 ? 1 : null;
106
107/** Browsers may refuse a write that arrives after the key press; a toast button is a fresh gesture. */
108async function copyPageText(text: string) {
109 try {
110 await navigator.clipboard.writeText(text);
111 } catch {
112 const id = toastManager.add({
113 type: "info",
114 title: "The page copied text",
115 actionProps: {
116 children: "Copy",
117 onClick: () => {
118 toastManager.close(id);
119 void navigator.clipboard.writeText(text).catch(() => undefined);
120 },
121 },
122 });
123 }
124}
125
126function offerDownload(download: PreviewStreamDownload) {
127 const id = toastManager.add({
128 type: "info",
129 title: `Downloaded ${download.fileName}`,
130 actionProps: {
131 children: "Save",
132 onClick: () => {
133 toastManager.close(id);
134 const anchor = document.createElement("a");
135 anchor.href = download.url;
136 anchor.download = download.fileName;
137 anchor.click();
138 },
139 },
140 });
141}
142
143export function ServerBrowserSurface(props: {
144 readonly environmentId: EnvironmentId;
145 readonly threadId: string;
146 readonly tabId: string;
147 readonly visible: boolean;
148 readonly followSize?: boolean;
149 readonly controlPosition?: "top" | "bottom";
150 readonly onFirstFrame?: () => void;
151 readonly onViewport?: (viewport: PreviewStreamViewport) => void;
152 readonly onControl?: (control: PreviewStreamControl | null) => void;
153 readonly onPopup?: (tabId: string) => void;
154 readonly className?: string;
155 readonly pdfUrl?: string | null;
156 readonly ref?: Ref<ServerBrowserHandle>;
157}) {
158 const {
159 environmentId,
160 threadId,
161 tabId,
162 visible,
163 followSize = true,
164 controlPosition = "top",
165 onFirstFrame,
166 onViewport,
167 onControl,
168 onPopup,
169 className,
170 pdfUrl = null,
171 ref,
172 } = props;
173 const canvasRef = useRef<HTMLCanvasElement | null>(null);
174 const inputRef = useRef<HTMLTextAreaElement | null>(null);
175 const keySentRef = useRef(false);
176 const clientRef = useRef<PreviewStreamClient | null>(null);
177 const viewportRef = useRef<PreviewStreamViewport | null>(null);
178 const sizeRef = useRef<{ width: number; height: number } | null>(null);
179 const hasFrameRef = useRef(false);
180 const controlRef = useRef<PreviewStreamControl | null>(null);
181 const [control, setControl] = useState<PreviewStreamControl | null>(null);
182 const [promptText, setPromptText] = useState("");
183 const [fileChooser, setFileChooser] = useState<PreviewStreamFileChooser | null>(null);
184 const [agentCursor, setAgentCursor] = useState<AgentCursorPlacement | null>(null);
185 const fileInputRef = useRef<HTMLInputElement | null>(null);
186 const answerFileChooser = (files: ReadonlyArray<File>) => {
187 const chooser = fileChooser;
188 if (!chooser) return;
189 setFileChooser(null);
190 void uploadPreviewStreamFiles(chooser, files).catch((cause: unknown) =>
191 toastManager.add({
192 type: "error",
193 title: "Could not send the files to the page",
194 description: cause instanceof Error ? cause.message : undefined,
195 }),
196 );
197 };
198 const unauthorizedRef = useRef(0);
199 const [accessDenied, setAccessDenied] = useState(false);
200 const access = usePreviewStreamAccess(environmentId, { retry: visible && !accessDenied });
201 const [hostSetup, setHostSetup] = useState<PreviewStreamHostSetup | null>(null);
202 const pendingMoveRef = useRef<MouseInput | null>(null);
203 const pendingWheelRef = useRef<WheelInput | null>(null);
204 const inputFrameRef = useRef<number | null>(null);
205 const mouseButtonsRef = useRef(0);
206 const mouseClicksRef = useRef({ left: 1, middle: 1, right: 1, none: 1 });
207 const lastClickRef = useRef<{
208 button: PreviewStreamMouseButton;
209 time: number;
210 x: number;
211 y: number;
212 count: number;
213 } | null>(null);
214 const touchRef = useRef<TouchGesture | null>(null);
215 const probeRef = useRef<TouchProbe | null>(null);
216 const lastProbeRef = useRef<{ x: number; y: number; editable: boolean; at: number } | null>(null);
217 const firstFrame = useEffectEvent(() => onFirstFrame?.());
218 const viewportChanged = useEffectEvent((viewport: PreviewStreamViewport) =>
219 onViewport?.(viewport),
220 );
221 const controlChanged = useEffectEvent((next: PreviewStreamControl | null) => onControl?.(next));
222 const popupOpened = useEffectEvent((popupTabId: string) => onPopup?.(popupTabId));
223 const [cap, setCap] = useState<{ width: number; height: number } | null>(null);
224
225 // Keep clearInput stable: a new identity reconnects the stream.
226 const send = (input: PreviewStreamInput) => {
227 clientRef.current?.send(input);
228 };
229
230 const clearInput = () => {
231 if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current);
232 inputFrameRef.current = null;
233 pendingMoveRef.current = pendingWheelRef.current = null;
234 mouseButtonsRef.current = 0;
235 mouseClicksRef.current = { left: 1, middle: 1, right: 1, none: 1 };
236 lastClickRef.current = null;
237 touchRef.current = probeRef.current = lastProbeRef.current = null;
238 keySentRef.current = false;
239 if (inputRef.current) inputRef.current.value = INPUT_SENTINEL;
240 inputRef.current?.blur();
241 };
242
243 const flushInput = () => {
244 if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current);
245 inputFrameRef.current = null;
246 const move = pendingMoveRef.current;
247 const wheel = pendingWheelRef.current;
248 pendingMoveRef.current = null;
249 pendingWheelRef.current = null;
250 if (move) send(move);
251 if (wheel) send(wheel);
252 };
253
254 const scheduleFlush = () => {
255 if (inputFrameRef.current === null) inputFrameRef.current = requestAnimationFrame(flushInput);
256 };
257
258 const queueWheel = (point: PagePoint, deltaX: number, deltaY: number, modifiers: number) => {
259 const pending = pendingWheelRef.current;
260 pendingWheelRef.current = {
261 type: "wheel",
262 x: point.x,
263 y: point.y,
264 deltaX: (pending?.deltaX ?? 0) + deltaX,
265 deltaY: (pending?.deltaY ?? 0) + deltaY,
266 modifiers,
267 };
268 scheduleFlush();
269 };
270
271 const pagePoint = (clientX: number, clientY: number, clamp: boolean): PagePoint | null => {
272 const canvas = canvasRef.current;
273 const viewport = viewportRef.current;
274 if (!canvas || !viewport || !hasFrameRef.current || controlRef.current?.controller !== "you")
275 return null;
276 const rect = canvas.getBoundingClientRect();
277 const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height);
278 const width = canvas.width * fit;
279 const height = canvas.height * fit;
280 if (!(width > 0 && height > 0)) return null;
281 const scale = viewport.width / width;
282 const x = (clientX - rect.left - (rect.width - width) / 2) * scale;
283 const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height);
284 const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height;
285 if (!inside && !clamp) return null;
286 return {
287 x: Math.min(Math.max(x, 0), viewport.width),
288 y: Math.min(Math.max(y, 0), viewport.height),
289 scale,
290 };
291 };
292
293 const countClick = (button: PreviewStreamMouseButton, x: number, y: number, time: number) => {
294 const last = lastClickRef.current;
295 const count =
296 last &&
297 last.button === button &&
298 time - last.time < MULTI_CLICK_MS &&
299 Math.hypot(x - last.x, y - last.y) < MULTI_CLICK_SLOP_PX
300 ? last.count + 1
301 : 1;
302 lastClickRef.current = { button, time, x, y, count };
303 return count;
304 };
305
306 const focusInput = () => inputRef.current?.focus({ preventScroll: true });
307
308 useImperativeHandle(
309 ref,
310 () => ({
311 navigate: (url) => send({ type: "navigate", url }),
312 history: (delta) => send({ type: "history", delta }),
313 reload: (options) =>
314 send({ type: "reload", ...(options?.ignoreCache ? { ignoreCache: true } : {}) }),
315 stopLoading: () => send({ type: "stopLoading" }),
316 viewport: (setting) => send({ type: "viewport", setting }),
317 canvas: () => (!pdfUrl && hasFrameRef.current ? canvasRef.current : null),
318 }),
319 [pdfUrl, send],
320 );
321
322 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- observes the canvas container's size, sends resizes to the stream and grows the frame cap
323 useEffect(() => {
324 const element = canvasRef.current?.parentElement;
325 if (!element) return;
326 let timer: ReturnType<typeof setTimeout> | null = null;
327 let capTimer: ReturnType<typeof setTimeout> | null = null;
328 const growCap = () => {
329 capTimer = null;
330 const size = sizeRef.current;
331 if (!size) return;
332 const ratio = window.devicePixelRatio || 1;
333 const width = Math.round(size.width * ratio);
334 const height = Math.round(size.height * ratio);
335 setCap((current) =>
336 current !== null && current.width >= width && current.height >= height
337 ? current
338 : {
339 width: Math.max(width, current?.width ?? 0),
340 height: Math.max(height, current?.height ?? 0),
341 },
342 );
343 };
344 const measure = () => {
345 timer = null;
346 const rect = element.getBoundingClientRect();
347 if (rect.width < 1 || rect.height < 1) return;
348 const size = { width: Math.round(rect.width), height: Math.round(rect.height) };
349 const previous = sizeRef.current;
350 if (previous?.width === size.width && previous.height === size.height) return;
351 sizeRef.current = size;
352 if (followSize) clientRef.current?.send({ type: "resize", ...size });
353 // Growing the cap reconnects the stream; wait until resizing settles.
354 if (capTimer !== null) clearTimeout(capTimer);
355 capTimer = setTimeout(growCap, previous === null ? 0 : RESIZE_THROTTLE_MS * 2);
356 };
357 const stopObserving = observeResize(element, () => {
358 if (timer !== null) return;
359 timer = setTimeout(measure, sizeRef.current === null ? 0 : RESIZE_THROTTLE_MS);
360 });
361 return () => {
362 stopObserving();
363 if (timer !== null) clearTimeout(timer);
364 if (capTimer !== null) clearTimeout(capTimer);
365 };
366 }, [followSize]);
367
368 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- opens the preview stream client and paints its frames onto the canvas while visible
369 useEffect(() => {
370 const canvas = canvasRef.current;
371 if (!visible || accessDenied || hostSetup || !access || !cap || !canvas) return;
372 let refreshTimer: ReturnType<typeof setTimeout> | null = null;
373 const painter = createPreviewFramePainter(canvas, () => {
374 if (hasFrameRef.current) return;
375 hasFrameRef.current = true;
376 firstFrame();
377 });
378 const client = createPreviewStreamClient(
379 { access, threadId, tabId, maxWidth: cap.width, maxHeight: cap.height },
380 {
381 onFrame: (jpeg) => {
382 unauthorizedRef.current = 0;
383 painter.paint(jpeg);
384 },
385 onProbe: (result) => {
386 if (controlRef.current?.controller !== "you") return;
387 lastProbeRef.current = { ...result, at: performance.now() };
388 const probe = probeRef.current;
389 if (!probe || probe.x !== result.x || probe.y !== result.y) return;
390 probe.editable = result.editable;
391 probe.answered = true;
392 if (!probe.tapped) return;
393 probeRef.current = null;
394 if (result.editable) inputRef.current?.focus({ preventScroll: true });
395 else inputRef.current?.blur();
396 },
397 onPointer: (pointer) => {
398 const placed = placeAgentCursor(pointer, canvasRef.current, viewportRef.current);
399 if (placed) setAgentCursor(placed);
400 },
401 onClipboard: (text) => void copyPageText(text),
402 onDownload: offerDownload,
403 onPopup: popupOpened,
404 onFileChooser: setFileChooser,
405 onViewport: (viewport) => {
406 viewportRef.current = viewport;
407 viewportChanged(viewport);
408 },
409 onControl: (next) => {
410 const previous = controlRef.current;
411 controlRef.current = next;
412 setControl(next);
413 controlChanged(next);
414 if (
415 next.dialog?.message !== previous?.dialog?.message ||
416 next.dialog?.defaultValue !== previous?.dialog?.defaultValue
417 )
418 setPromptText(next.dialog?.defaultValue ?? "");
419 if (next.controller !== "you") {
420 clearInput();
421 } else if (previous?.controller !== "you") {
422 const size = sizeRef.current;
423 if (followSize && size) client.send({ type: "resize", ...size });
424 }
425 },
426 onConnectedChange: (connected) => {
427 if (connected) return;
428 setFileChooser(null);
429 controlRef.current = null;
430 setControl(null);
431 controlChanged(null);
432 clearInput();
433 },
434 onHostSetup: setHostSetup,
435 onUnauthorized: () => {
436 const refusals = ++unauthorizedRef.current;
437 if (refusals >= MAX_UNAUTHORIZED_REFUSALS) {
438 inputRef.current?.blur();
439 setAccessDenied(true);
440 return;
441 }
442 refreshTimer = setTimeout(
443 () => refreshPreviewStreamAccess(environmentId),
444 refusals === 1 ? 0 : 1_000 * 2 ** (refusals - 1),
445 );
446 },
447 },
448 );
449 clientRef.current = client;
450 return () => {
451 painter.stop();
452 if (refreshTimer !== null) clearTimeout(refreshTimer);
453 client.stop();
454 if (clientRef.current === client) clientRef.current = null;
455 controlRef.current = null;
456 setControl(null);
457 controlChanged(null);
458 clearInput();
459 };
460 }, [
461 access,
462 accessDenied,
463 cap,
464 clearInput,
465 environmentId,
466 followSize,
467 hostSetup,
468 tabId,
469 threadId,
470 visible,
471 ]);
472
473 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- registers a non-passive wheel listener on the canvas so it can stop the panel scrolling
474 useEffect(() => {
475 const canvas = canvasRef.current;
476 if (!canvas) return;
477 // Registered natively: React's wheel listener is passive and cannot stop the panel scrolling.
478 const onWheel = (event: WheelEvent) => {
479 const point = pagePoint(event.clientX, event.clientY, false);
480 if (!point) return;
481 event.preventDefault();
482 const unit =
483 event.deltaMode === WheelEvent.DOM_DELTA_LINE
484 ? WHEEL_LINE_PX
485 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE
486 ? (viewportRef.current?.height ?? 0)
487 : 1;
488 queueWheel(point, event.deltaX * unit, event.deltaY * unit, previewStreamModifiers(event));
489 };
490 canvas.addEventListener("wheel", onWheel, { passive: false });
491 return () => canvas.removeEventListener("wheel", onWheel);
492 }, [pagePoint, queueWheel]);
493
494 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- cancels the pending input animation frame when the component unmounts
495 useEffect(
496 () => () => {
497 if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current);
498 },
499 [],
500 );
501
502 const handlePointer = (event: PointerEvent<HTMLCanvasElement>) => {
503 if (controlRef.current?.controller !== "you") return;
504 // Thumb buttons drive this tab's history in handleMouseUp, never the page.
505 if (historyDeltaOf(event.button) !== null) return;
506 const { type, pointerId, clientX, clientY } = event;
507 const cancelled = type === "pointercancel";
508 const touch = touchRef.current;
509 if (cancelled && touch?.pointerId === pointerId) {
510 touchRef.current = probeRef.current = null;
511 return;
512 }
513 if (event.pointerType === "touch" && !cancelled) {
514 const point = pagePoint(clientX, clientY, type === "pointermove");
515 if (type === "pointerdown") {
516 if (!event.isPrimary) return;
517 touchRef.current = {
518 pointerId,
519 startX: clientX,
520 startY: clientY,
521 lastX: clientX,
522 lastY: clientY,
523 panning: false,
524 };
525 const last = lastProbeRef.current;
526 const reusable =
527 point &&
528 last &&
529 performance.now() - last.at < PROBE_REUSE_MS &&
530 Math.hypot(point.x - last.x, point.y - last.y) < PROBE_REUSE_PX;
531 probeRef.current = point
532 ? {
533 x: point.x,
534 y: point.y,
535 editable: reusable ? last.editable : null,
536 answered: false,
537 tapped: false,
538 }
539 : null;
540 if (point) send({ type: "probe", x: point.x, y: point.y });
541 return;
542 }
543 if (!touch || touch.pointerId !== pointerId) return;
544 if (type === "pointermove") {
545 if (
546 !touch.panning &&
547 Math.hypot(clientX - touch.startX, clientY - touch.startY) < TAP_SLOP_PX
548 )
549 return;
550 touch.panning = true;
551 if (point)
552 queueWheel(
553 point,
554 (touch.lastX - clientX) * point.scale,
555 (touch.lastY - clientY) * point.scale,
556 0,
557 );
558 touch.lastX = clientX;
559 touch.lastY = clientY;
560 return;
561 }
562 touchRef.current = null;
563 const probe = probeRef.current;
564 if (touch.panning) {
565 probeRef.current = null;
566 return;
567 }
568 if (!point) return;
569 // iOS requires focus during the tap; late probe replies only raise Android's keyboard.
570 if (probe?.editable === true) focusInput();
571 else if (probe?.editable === false) inputRef.current?.blur();
572 if (probe && !probe.answered) probe.tapped = true;
573 else probeRef.current = null;
574 flushInput();
575 const clickCount = countClick("left", clientX, clientY, event.timeStamp);
576 const at = { x: point.x, y: point.y, modifiers: 0 };
577 send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 });
578 send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount });
579 send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount });
580 return;
581 }
582 const previous = mouseButtonsRef.current;
583 if ((cancelled || type === "pointerup") && previous === 0) return;
584 if (cancelled || type === "pointerup")
585 mouseButtonsRef.current = cancelled ? 0 : event.buttons & 7;
586 const point = pagePoint(
587 clientX,
588 clientY,
589 type !== "pointerdown" &&
590 (cancelled || type === "pointerup" || previous !== 0 || event.buttons !== 0),
591 );
592 if (!point) return;
593 const at = { x: point.x, y: point.y, modifiers: previewStreamModifiers(event) };
594 const mouse = (action: "down" | "up", button: PreviewStreamMouseButton, buttons: number) => {
595 if (action === "down")
596 mouseClicksRef.current[button] = countClick(button, clientX, clientY, event.timeStamp);
597 send({
598 type: "mouse",
599 action,
600 ...at,
601 button,
602 buttons,
603 clickCount: mouseClicksRef.current[button],
604 });
605 };
606 if (type === "pointerdown") {
607 focusInput();
608 event.currentTarget.setPointerCapture(pointerId);
609 flushInput();
610 mouseButtonsRef.current = event.buttons & 7;
611 mouse("down", buttonOf(event.button), event.buttons);
612 } else if (type === "pointerup") {
613 flushInput();
614 mouse("up", buttonOf(event.button), event.buttons);
615 } else {
616 // Chords arrive as pointermove. Cancellation releases every held button.
617 const next = cancelled ? 0 : event.buttons & 7;
618 const changed = previous ^ next;
619 if (previous !== 0 && changed !== 0) {
620 flushInput();
621 let buttons = previous;
622 for (const bit of [1, 2, 4]) {
623 if (!(changed & bit)) continue;
624 buttons ^= bit;
625 mouse(next & bit ? "down" : "up", pressedButtonOf(bit), buttons);
626 }
627 mouseButtonsRef.current = next;
628 }
629 if (cancelled) return;
630 pendingMoveRef.current = {
631 type: "mouse",
632 action: "move",
633 ...at,
634 button: pressedButtonOf(event.buttons),
635 buttons: event.buttons,
636 clickCount: 0,
637 };
638 scheduleFlush();
639 }
640 };
641
642 // Chromium runs a thumb button's back/forward on mouseup in this document, which
643 // would move the app's router. Cancel it and send the press to the preview, as
644 // the toolbar does.
645 const handleMouseUp = (event: MouseEvent<HTMLDivElement>) => {
646 const delta = historyDeltaOf(event.button);
647 if (delta === null) return;
648 event.preventDefault();
649 send({ type: "history", delta });
650 };
651
652 const resetInput = (textarea: HTMLTextAreaElement) => {
653 keySentRef.current = false;
654 textarea.value = INPUT_SENTINEL;
655 textarea.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length);
656 };
657
658 const sendKeyPress = (
659 key: typeof BACKSPACE | typeof DELETE | typeof COPY | typeof CUT,
660 modifiers = 0,
661 ) => {
662 send({ type: "key", action: "down", ...key, modifiers });
663 send({ type: "key", action: "up", ...key, modifiers });
664 };
665
666 // Edit menu Copy and Cut fire no key, so send the shortcut the server copies on.
667 const handleClipboard = (key: typeof COPY | typeof CUT, event: ClipboardEvent) => {
668 // Keep the sentinel off the clipboard.
669 event.preventDefault();
670 if (controlRef.current?.controller !== "you") return;
671 sendKeyPress(key, CTRL);
672 };
673
674 const handleKey = (action: "down" | "up", event: KeyboardEvent<HTMLTextAreaElement>) => {
675 if (controlRef.current?.controller !== "you") return;
676 keySentRef.current = false;
677 if (
678 event.nativeEvent.isComposing ||
679 event.keyCode === 229 ||
680 event.key === "Process" ||
681 event.key === "Unidentified"
682 ) {
683 return;
684 }
685 if (event.key === "Escape" && event.shiftKey) {
686 event.preventDefault();
687 event.stopPropagation();
688 if (action === "down") event.currentTarget.blur();
689 return;
690 }
691 const shortcut = event.ctrlKey || event.metaKey;
692 if (shortcut && event.key.toLowerCase() === "v") return;
693 if (event.shiftKey && event.key === "Insert") return;
694 // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines.
695 const text = shortcut
696 ? undefined
697 : [...event.key].length === 1
698 ? event.key
699 : event.key === "Enter"
700 ? "\r"
701 : undefined;
702 send({
703 type: "key",
704 action,
705 key: event.key,
706 code: event.code,
707 keyCode: event.keyCode,
708 ...(action === "down" && text !== undefined ? { text } : {}),
709 modifiers: previewStreamModifiers(event),
710 });
711 // Some Android keyboards edit the textarea even when keydown is prevented.
712 keySentRef.current =
713 action === "down" &&
714 !shortcut &&
715 (text !== undefined || event.key === "Backspace" || event.key === "Delete");
716 // The key copies or cuts in the page, so stop the browser's own copy or cut repeating it.
717 if (
718 event.ctrlKey !== event.metaKey &&
719 !event.altKey &&
720 !event.shiftKey &&
721 ["c", "x"].includes(event.key.toLowerCase())
722 ) {
723 event.preventDefault();
724 }
725 if (shortcut) return;
726 event.preventDefault();
727 event.stopPropagation();
728 };
729
730 const handleInput = (event: FormEvent<HTMLTextAreaElement>) => {
731 const native = event.nativeEvent;
732 if (native instanceof InputEvent && native.isComposing) return;
733 const textarea = event.currentTarget;
734 const inputType = native instanceof InputEvent ? native.inputType : "";
735 if (keySentRef.current) keySentRef.current = false;
736 else if (inputType === "deleteContentBackward") sendKeyPress(BACKSPACE);
737 else if (inputType === "deleteContentForward") sendKeyPress(DELETE);
738 else {
739 const text = textarea.value.replace(/^\u200b/, "");
740 if (text) send({ type: "text", text });
741 }
742 resetInput(textarea);
743 };
744
745 return (
746 <div
747 className={cn("relative flex flex-col overflow-hidden", className)}
748 data-server-browser-surface={tabId}
749 onMouseUp={handleMouseUp}
750 >
751 <div
752 className={cn(
753 "visible relative z-20 flex shrink-0 items-center justify-between gap-2 border-border bg-background px-2 py-1",
754 controlPosition === "bottom" ? "order-last border-t" : "border-b",
755 )}
756 >
757 <span role="status" className="text-xs text-muted-foreground">
758 {previewStreamControlLabel(control)}
759 </span>
760 {control?.canOperate ? (
761 <Button
762 variant="outline"
763 size="xs"
764 disabled={control.controller === "another-viewer"}
765 onClick={() =>
766 send({ type: control.controller === "you" ? "releaseControl" : "takeControl" })
767 }
768 >
769 {control.controller === "you" ? "Release control" : "Take control"}
770 </Button>
771 ) : null}
772 </div>
773 <div className="relative min-h-0 flex-1">
774 {pdfUrl ? (
775 <embed
776 src={pdfUrl}
777 type="application/pdf"
778 title="PDF document"
779 className="absolute inset-0 z-10 size-full border-0"
780 />
781 ) : null}
782 <canvas
783 ref={canvasRef}
784 className="block size-full touch-none object-contain"
785 onPointerDown={handlePointer}
786 onPointerMove={handlePointer}
787 onPointerUp={handlePointer}
788 onPointerCancel={handlePointer}
789 onMouseDown={(event) => event.preventDefault()}
790 onContextMenu={(event) => event.preventDefault()}
791 />
792 {!pdfUrl && agentCursor && control?.controller !== "you" ? (
793 <AgentCursorMark
794 {...agentCursor}
795 controller={control?.controller === "agent" ? "agent" : "none"}
796 />
797 ) : null}
798 {/* Pin top-left to avoid focus scrolling; 16px prevents iOS focus zoom. */}
799 <textarea
800 ref={inputRef}
801 aria-label="Browser page"
802 aria-description="Press Shift+Escape to leave the browser page."
803 autoCapitalize="off"
804 autoComplete="off"
805 autoCorrect="off"
806 spellCheck={false}
807 disabled={pdfUrl !== null || control?.controller !== "you"}
808 defaultValue={INPUT_SENTINEL}
809 onFocus={(event) => resetInput(event.currentTarget)}
810 className="sr-only top-0 left-0 text-base"
811 onKeyDown={(event) => handleKey("down", event)}
812 onKeyUp={(event) => handleKey("up", event)}
813 onInput={handleInput}
814 onCompositionEnd={(event) => {
815 const text = event.data.replace(/^\u200b/, "");
816 if (text) send({ type: "text", text });
817 resetInput(event.currentTarget);
818 }}
819 onCopy={(event) => handleClipboard(COPY, event)}
820 onCut={(event) => handleClipboard(CUT, event)}
821 onPaste={(event) => {
822 event.preventDefault();
823 const text = event.clipboardData.getData("text/plain");
824 if (text) send({ type: "text", text });
825 }}
826 />
827 {fileChooser && control?.controller === "you" ? (
828 <div
829 className="absolute inset-x-2 top-2 z-10 flex flex-col gap-2 rounded-lg border border-border bg-background p-3 shadow-lg"
830 role="dialog"
831 aria-label="Choose files for the page"
832 >
833 <p className="text-sm">
834 The page asks for {fileChooser.multiple ? "files" : "a file"}.
835 </p>
836 <input
837 ref={fileInputRef}
838 type="file"
839 className="hidden"
840 multiple={fileChooser.multiple}
841 accept={fileChooser.accept}
842 onChange={(event) => {
843 const files = [...(event.currentTarget.files ?? [])];
844 event.currentTarget.value = "";
845 if (files.length > 0) answerFileChooser(files);
846 }}
847 />
848 <div className="flex justify-end gap-2">
849 <Button variant="outline" size="sm" onClick={() => answerFileChooser([])}>
850 Cancel
851 </Button>
852 <Button size="sm" onClick={() => fileInputRef.current?.click()}>
853 Choose {fileChooser.multiple ? "files" : "file"}
854 </Button>
855 </div>
856 </div>
857 ) : null}
858 {control?.dialog ? (
859 <div
860 className="absolute inset-x-2 top-2 z-10 flex flex-col gap-2 rounded-lg border border-border bg-background p-3 shadow-lg"
861 role="dialog"
862 aria-label="Browser dialog"
863 >
864 <p className="break-words text-sm">{control.dialog.message}</p>
865 {control.controller === "you" ? (
866 <>
867 {control.dialog.type === "prompt" ? (
868 <Input
869 aria-label="Dialog response"
870 value={promptText}
871 onChange={(event) => setPromptText(event.target.value)}
872 />
873 ) : null}
874 <div className="flex justify-end gap-2">
875 <Button
876 variant="outline"
877 size="sm"
878 onClick={() => send({ type: "dialog", accept: false })}
879 >
880 Dismiss
881 </Button>
882 <Button
883 size="sm"
884 onClick={() =>
885 send({
886 type: "dialog",
887 accept: true,
888 ...(control.dialog?.type === "prompt" ? { promptText } : {}),
889 })
890 }
891 >
892 Accept
893 </Button>
894 </div>
895 </>
896 ) : (
897 <p className="text-xs text-muted-foreground">Take control to respond.</p>
898 )}
899 </div>
900 ) : null}
901 </div>
902 {visible && hostSetup ? (
903 <div className="visible absolute inset-0 z-20 flex flex-col items-center justify-center gap-3 bg-background p-4 text-center">
904 <p role="alert" className="max-w-sm text-sm text-muted-foreground">
905 {previewStreamHostSetupMessage(hostSetup)}
906 </p>
907 <CommandBlock command={hostSetup.command} className="w-full max-w-md text-left" />
908 <Button variant="outline" size="sm" onClick={() => setHostSetup(null)}>
909 Try again
910 </Button>
911 </div>
912 ) : null}
913 {visible && accessDenied ? (
914 // The page can be invisible beneath an empty or unreachable state; reconnect must remain reachable.
915 <div className="visible absolute inset-0 z-20 flex flex-col items-center justify-center gap-2 bg-background p-3 text-center">
916 <p role="alert" className="text-xs text-muted-foreground">
917 Browser connection was refused.
918 </p>
919 <Button
920 variant="outline"
921 size="sm"
922 onClick={() => {
923 unauthorizedRef.current = 0;
924 setAccessDenied(false);
925 refreshPreviewStreamAccess(environmentId);
926 }}
927 >
928 Reconnect
929 </Button>
930 </div>
931 ) : null}
932 </div>
933 );
934}
935
936interface AgentCursorPlacement {
937 readonly phase: PreviewStreamPointer["phase"];
938 readonly sequence: number;
939 readonly left: number;
940 readonly top: number;
941}
942
943function placeAgentCursor(
944 pointer: PreviewStreamPointer,
945 canvas: HTMLCanvasElement | null,
946 viewport: PreviewStreamViewport | null,
947): AgentCursorPlacement | null {
948 if (!canvas || !viewport || canvas.width < 1 || canvas.height < 1) return null;
949 const box = { width: canvas.clientWidth, height: canvas.clientHeight };
950 const fit = Math.min(box.width / canvas.width, box.height / canvas.height);
951 const width = canvas.width * fit;
952 const height = canvas.height * fit;
953 return {
954 phase: pointer.phase,
955 sequence: pointer.sequence,
956 left: (box.width - width) / 2 + (pointer.x * width) / viewport.width,
957 top: (box.height - height) / 2 + (pointer.y * height) / viewport.height,
958 };
959}