1import {
2 createPreviewFramePainter,
3 createPreviewStreamClient,
4 previewStreamModifiers,
5 type PreviewStreamClient,
6 type PreviewStreamControl,
7 type PreviewStreamInput,
8 type PreviewStreamPointer,
9 type PreviewStreamViewport,
10} from "@t3tools/client-runtime/preview/server-browser-stream";
11
12import type { PreviewStreamConfiguration, PreviewStreamMessage } from "./preview-stream-document";
13
14declare global {
15 interface Window {
16 ReactNativeWebView: { postMessage: (message: string) => void };
17 }
18}
19
20interface PresentationVideo extends HTMLVideoElement {
21 webkitSetPresentationMode?: (mode: "inline" | "picture-in-picture") => void;
22 webkitPresentationMode?: string;
23}
24
25type WheelInput = Extract<PreviewStreamInput, { type: "wheel" }>;
26type MouseInput = Extract<PreviewStreamInput, { type: "mouse" }>;
27const mouseButton = (button: number): MouseInput["button"] =>
28 button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none";
29
30const RESIZE_DEBOUNCE_MS = 150;
31const TAP_SLOP_PX = 8;
32const MULTI_CLICK_MS = 500;
33const MULTI_CLICK_SLOP_PX = 4;
34const WHEEL_LINE_PX = 16;
35const MAX_PIXEL_RATIO = 2;
36// Reuse nearby probe answers: slow replies arrive too late after touch-end to raise the iOS keyboard.
37const PROBE_REUSE_PX = 24;
38const PROBE_REUSE_MS = 10_000;
39// A sentinel makes backspace fire input; Gboard keydown supplies only keyCode 229.
40const SENTINEL = "\u200b";
41const AGENT_CURSOR_ACTIVE_MS = 700;
42const AGENT_CURSOR_SVG =
43 '<svg width="20" height="20" viewBox="0 0 24 24" fill="#fff" stroke="#3b82f6" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/></svg>';
44
45interface Viewer {
46 readonly stop: () => void;
47 readonly command: (input: PreviewStreamInput) => void;
48 readonly togglePictureInPicture: () => void;
49}
50
51let activeViewer: Viewer | null = null;
52
53export function stop() {
54 activeViewer?.stop();
55 activeViewer = null;
56}
57
58export function command(input: PreviewStreamInput) {
59 activeViewer?.command(input);
60}
61
62export function pictureInPicture() {
63 activeViewer?.togglePictureInPicture();
64}
65
66export function start(configuration: PreviewStreamConfiguration) {
67 stop();
68 const post = (message: PreviewStreamMessage) => {
69 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string.
70 window.ReactNativeWebView.postMessage(JSON.stringify(message));
71 };
72 const { interactive } = configuration;
73 document.body.style.background = configuration.background;
74 const container = document.createElement("div");
75 const canvas = document.createElement("canvas");
76 canvas.setAttribute("role", "img");
77 canvas.setAttribute("aria-label", "Browser page");
78 const agentCursor = document.createElement("div");
79 agentCursor.className = "agent-cursor";
80 agentCursor.setAttribute("aria-hidden", "true");
81 agentCursor.innerHTML = AGENT_CURSOR_SVG;
82 container.append(canvas, agentCursor);
83 const input = document.createElement("textarea");
84 input.setAttribute("aria-label", "Browser page input");
85 input.autocapitalize = "off";
86 input.autocomplete = "off";
87 input.spellcheck = false;
88 input.setAttribute("autocorrect", "off");
89 input.disabled = true;
90 document.body.replaceChildren(container, ...(interactive ? [input] : []));
91
92 let stopped = false;
93 let streaming = false;
94 let viewport: PreviewStreamViewport | null = null;
95 let size: { width: number; height: number } | null = null;
96 let cap: { width: number; height: number } | null = null;
97 let client: PreviewStreamClient | null = null;
98 let control: PreviewStreamControl | null = null;
99 let resizeTimer: ReturnType<typeof setTimeout> | null = null;
100 let wheelFrame: number | null = null;
101 let pendingWheel: WheelInput | null = null;
102 let mouseFrame: number | null = null;
103 let pendingMouse: MouseInput | null = null;
104 let mouseButtons = 0;
105 const mouseClicks = { left: 1, middle: 1, right: 1, none: 1 };
106 let probe: {
107 readonly x: number;
108 readonly y: number;
109 readonly clientX: number;
110 readonly clientY: number;
111 editable: boolean | null;
112 end: "touching" | "acted" | "late" | "panned";
113 } | null = null;
114 let probeCache: {
115 readonly clientX: number;
116 readonly clientY: number;
117 readonly editable: boolean;
118 readonly time: number;
119 } | null = null;
120 let keySent = false;
121 let touch: {
122 start: PointerEvent;
123 last: PointerEvent;
124 panning: boolean;
125 } | null = null;
126 let lastTap: {
127 time: number;
128 x: number;
129 y: number;
130 count: number;
131 button: MouseInput["button"];
132 } | null = null;
133
134 const reportStatus = (status: "connecting" | "streaming") => {
135 streaming = status === "streaming";
136 post({ type: "status", status });
137 };
138 const painter = createPreviewFramePainter(canvas, () => {
139 if (!streaming && !stopped) reportStatus("streaming");
140 sendPictureInPictureFrame();
141 });
142 const send = (message: PreviewStreamInput) => client?.send(message) ?? false;
143 const clearInput = () => {
144 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
145 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
146 wheelFrame = mouseFrame = null;
147 pendingWheel = pendingMouse = null;
148 mouseButtons = 0;
149 Object.assign(mouseClicks, { left: 1, middle: 1, right: 1, none: 1 });
150 probe = probeCache = touch = lastTap = null;
151 keySent = false;
152 input.value = SENTINEL;
153 input.disabled = true;
154 input.blur();
155 };
156
157 const connect = () => {
158 if (client) reportStatus("connecting");
159 client?.stop();
160 control = null;
161 clearInput();
162 if (!cap || stopped) return;
163 const next = createPreviewStreamClient(
164 {
165 access: configuration.access,
166 threadId: configuration.threadId,
167 tabId: configuration.tabId,
168 maxWidth: cap.width,
169 maxHeight: cap.height,
170 interactive,
171 },
172 {
173 onFrame: (jpeg) => painter.paint(jpeg),
174 onPointer: showAgentCursor,
175 onClipboard: (text) => post({ type: "clipboard", text }),
176 onDownload: (download) => post({ type: "download", ...download }),
177 onFileChooser: (chooser) => post({ type: "fileChooser", chooser }),
178 onViewport: (page) => {
179 if (viewport?.width === page.width && viewport.height === page.height) return;
180 viewport = page;
181 post({ type: "viewport", width: page.width, height: page.height });
182 },
183 onProbe: (result) => {
184 if (control?.controller !== "you") return;
185 const current = probe;
186 if (!current || current.x !== result.x || current.y !== result.y) return;
187 probeCache = {
188 clientX: current.clientX,
189 clientY: current.clientY,
190 editable: result.editable,
191 time: performance.now(),
192 };
193 if (current.end === "touching") {
194 current.editable = result.editable;
195 return;
196 }
197 probe = null;
198 if (current.end === "panned") return;
199 if (current.end === "acted" && current.editable === result.editable) return;
200 // Late answers can raise Android's keyboard; iOS needs another tap.
201 if (result.editable) input.focus({ preventScroll: true });
202 else input.blur();
203 },
204 onControl: (nextControl) => {
205 const previous = control;
206 control = nextControl;
207 post({ type: "control", ...nextControl });
208 if (nextControl.controller === "you") agentCursor.style.opacity = "0";
209 if (nextControl.controller !== "you") clearInput();
210 else {
211 input.disabled = !interactive;
212 if (interactive && size && previous?.controller !== "you")
213 next.send({ type: "resize", ...size });
214 }
215 },
216 onConnectedChange: (connected) => {
217 if (!connected) {
218 control = null;
219 clearInput();
220 if (streaming) reportStatus("connecting");
221 return;
222 }
223 },
224 onUnauthorized: () => post({ type: "unauthorized" }),
225 onGone: () => post({ type: "gone" }),
226 onHostSetup: (setup) => post({ type: "hostSetup", ...setup }),
227 },
228 );
229 client = next;
230 };
231
232 const measure = () => {
233 resizeTimer = null;
234 const rect = container.getBoundingClientRect();
235 if (rect.width < 1 || rect.height < 1) return;
236 const next = { width: Math.round(rect.width), height: Math.round(rect.height) };
237 if (size?.width === next.width && size.height === next.height) return;
238 size = next;
239 if (interactive) send({ type: "resize", ...next });
240 const ratio = Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO);
241 const side = Math.max(next.width, next.height);
242 const width = Math.round((interactive ? next.width : side) * ratio);
243 const height = Math.round((interactive ? next.height : side) * ratio);
244 if (cap && cap.width >= width && cap.height >= height) return;
245 cap = { width: Math.max(width, cap?.width ?? 0), height: Math.max(height, cap?.height ?? 0) };
246 connect();
247 };
248 const observer = new ResizeObserver(() => {
249 if (resizeTimer !== null) clearTimeout(resizeTimer);
250 resizeTimer = setTimeout(measure, size === null ? 0 : RESIZE_DEBOUNCE_MS);
251 });
252 observer.observe(container);
253
254 // iOS keeps its layout viewport under the keyboard; resize to the visual viewport.
255 const visualViewport = window.visualViewport;
256 const followVisualViewport = () => {
257 if (!visualViewport) return;
258 container.style.top = `${visualViewport.offsetTop}px`;
259 container.style.height = `${visualViewport.height}px`;
260 };
261 if (interactive) {
262 visualViewport?.addEventListener("resize", followVisualViewport);
263 visualViewport?.addEventListener("scroll", followVisualViewport);
264 }
265
266 let agentCursorTimer: ReturnType<typeof setTimeout> | null = null;
267 const showAgentCursor = (pointer: PreviewStreamPointer) => {
268 if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller === "you")
269 return;
270 const box = { width: canvas.clientWidth, height: canvas.clientHeight };
271 const fit = Math.min(box.width / canvas.width, box.height / canvas.height);
272 const width = canvas.width * fit;
273 const height = canvas.height * fit;
274 const left = (box.width - width) / 2 + (pointer.x * width) / viewport.width;
275 const top = (box.height - height) / 2 + (pointer.y * height) / viewport.height;
276 agentCursor.style.transform = `translate3d(${left}px, ${top}px, 0)`;
277 agentCursor.style.opacity = "1";
278 if (pointer.phase === "click") {
279 agentCursor.querySelector(".ping")?.remove();
280 const ping = document.createElement("span");
281 ping.className = "ping";
282 agentCursor.prepend(ping);
283 }
284 if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
285 agentCursorTimer = setTimeout(() => {
286 agentCursor.style.opacity = control?.controller === "agent" ? "0.35" : "0";
287 }, AGENT_CURSOR_ACTIVE_MS);
288 };
289
290 const pagePoint = (clientX: number, clientY: number, clamp: boolean) => {
291 if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller !== "you")
292 return null;
293 const rect = canvas.getBoundingClientRect();
294 const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height);
295 const width = canvas.width * fit;
296 const height = canvas.height * fit;
297 if (!(width > 0 && height > 0)) return null;
298 const scale = viewport.width / width;
299 const x = (clientX - rect.left - (rect.width - width) / 2) * scale;
300 const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height);
301 const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height;
302 if (!inside && !clamp) return null;
303 return {
304 x: Math.min(Math.max(x, 0), viewport.width),
305 y: Math.min(Math.max(y, 0), viewport.height),
306 scale,
307 };
308 };
309
310 const flushWheel = () => {
311 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
312 wheelFrame = null;
313 const wheel = pendingWheel;
314 pendingWheel = null;
315 if (wheel) send(wheel);
316 };
317 const queueWheel = (point: { x: number; y: number }, deltaX: number, deltaY: number) => {
318 pendingWheel = {
319 type: "wheel",
320 x: point.x,
321 y: point.y,
322 deltaX: (pendingWheel?.deltaX ?? 0) + deltaX,
323 deltaY: (pendingWheel?.deltaY ?? 0) + deltaY,
324 modifiers: 0,
325 };
326 wheelFrame ??= requestAnimationFrame(flushWheel);
327 };
328
329 const flushMouse = () => {
330 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
331 mouseFrame = null;
332 if (pendingMouse) send(pendingMouse);
333 pendingMouse = null;
334 };
335 const countClick = (event: PointerEvent, button: MouseInput["button"]) => {
336 const last = lastTap;
337 const count =
338 last &&
339 last.button === button &&
340 event.timeStamp - last.time < MULTI_CLICK_MS &&
341 Math.hypot(event.clientX - last.x, event.clientY - last.y) < MULTI_CLICK_SLOP_PX
342 ? last.count + 1
343 : 1;
344 lastTap = { time: event.timeStamp, x: event.clientX, y: event.clientY, count, button };
345 return count;
346 };
347 const mouseInput = (
348 event: PointerEvent,
349 point: { x: number; y: number },
350 action: MouseInput["action"],
351 button: MouseInput["button"],
352 buttons = event.buttons,
353 clickCount = mouseClicks[button],
354 ): MouseInput => ({
355 type: "mouse",
356 action,
357 x: point.x,
358 y: point.y,
359 button,
360 buttons,
361 clickCount,
362 modifiers: previewStreamModifiers(event),
363 });
364 const onPointerDown = (event: PointerEvent) => {
365 if (control?.controller !== "you") return;
366 if (!event.isPrimary) return;
367 event.preventDefault();
368 if (event.pointerType !== "touch") {
369 const point = pagePoint(event.clientX, event.clientY, false);
370 if (!point) return;
371 canvas.setPointerCapture(event.pointerId);
372 input.focus({ preventScroll: true });
373 flushMouse();
374 flushWheel();
375 mouseButtons = event.buttons & 7;
376 const button = mouseButton(event.button);
377 mouseClicks[button] = countClick(event, button);
378 send(mouseInput(event, point, "down", button));
379 return;
380 }
381 touch = { start: event, last: event, panning: false };
382 const point = pagePoint(event.clientX, event.clientY, false);
383 if (!point) {
384 probe = null;
385 return;
386 }
387 const cached = probeCache;
388 const reuse =
389 cached !== null &&
390 performance.now() - cached.time < PROBE_REUSE_MS &&
391 Math.hypot(event.clientX - cached.clientX, event.clientY - cached.clientY) <= PROBE_REUSE_PX;
392 probe = {
393 x: point.x,
394 y: point.y,
395 clientX: event.clientX,
396 clientY: event.clientY,
397 editable: reuse ? cached.editable : null,
398 end: "touching",
399 };
400 send({ type: "probe", x: point.x, y: point.y });
401 };
402 const onPointerMove = (event: PointerEvent) => {
403 if (control?.controller !== "you") return;
404 if (event.pointerType !== "touch") {
405 const point = pagePoint(event.clientX, event.clientY, mouseButtons !== 0);
406 if (!point) return;
407 // Chorded presses and releases arrive as pointermove while another button is held.
408 const changed = mouseButtons ^ (event.buttons & 7);
409 if (mouseButtons !== 0 && changed !== 0) {
410 flushMouse();
411 flushWheel();
412 for (const bit of [1, 2, 4]) {
413 if (!(changed & bit)) continue;
414 const button = bit === 1 ? "left" : bit === 2 ? "right" : "middle";
415 const down = (event.buttons & bit) !== 0;
416 mouseButtons ^= bit;
417 if (down) mouseClicks[button] = countClick(event, button);
418 send(mouseInput(event, point, down ? "down" : "up", button, mouseButtons));
419 }
420 }
421 const button =
422 mouseButtons & 1
423 ? "left"
424 : mouseButtons & 2
425 ? "right"
426 : mouseButtons & 4
427 ? "middle"
428 : "none";
429 pendingMouse = mouseInput(event, point, "move", button, event.buttons, 0);
430 mouseFrame ??= requestAnimationFrame(flushMouse);
431 return;
432 }
433 if (!touch || touch.start.pointerId !== event.pointerId) return;
434 if (
435 !touch.panning &&
436 Math.hypot(event.clientX - touch.start.clientX, event.clientY - touch.start.clientY) <
437 TAP_SLOP_PX
438 ) {
439 return;
440 }
441 touch.panning = true;
442 const point = pagePoint(event.clientX, event.clientY, true);
443 if (point) {
444 queueWheel(
445 point,
446 (touch.last.clientX - event.clientX) * point.scale,
447 (touch.last.clientY - event.clientY) * point.scale,
448 );
449 }
450 touch.last = event;
451 };
452 const releaseMouse = (event: PointerEvent, cancelled: boolean) => {
453 let buttons = mouseButtons;
454 if (buttons === 0) return;
455 mouseButtons = cancelled ? 0 : event.buttons & 7;
456 flushMouse();
457 flushWheel();
458 const point = pagePoint(event.clientX, event.clientY, true);
459 if (!point) return;
460 for (const bit of [1, 2, 4]) {
461 if (!(buttons & bit) || (mouseButtons & bit) !== 0) continue;
462 buttons &= ~bit;
463 const button = bit === 1 ? "left" : bit === 2 ? "right" : "middle";
464 send(mouseInput(event, point, "up", button, buttons));
465 }
466 };
467 const onPointerEnd = (event: PointerEvent) => {
468 const cancelled = event.type === "pointercancel";
469 if (event.pointerType !== "touch") {
470 releaseMouse(event, cancelled);
471 return;
472 }
473 const ended = touch;
474 if (!ended || ended.start.pointerId !== event.pointerId) return;
475 touch = null;
476 const answered = probe;
477 if (cancelled || ended.panning) {
478 if (answered) answered.end = "panned";
479 return;
480 }
481 const point = pagePoint(event.clientX, event.clientY, false);
482 if (!point) return;
483 // Focusing inside the tap's user activation is what lets iOS raise the keyboard.
484 if (answered?.editable === true) input.focus({ preventScroll: true });
485 else if (answered?.editable === false) input.blur();
486 if (answered) answered.end = answered.editable === null ? "late" : "acted";
487 flushWheel();
488 const clickCount = countClick(event, "left");
489 const at = { x: point.x, y: point.y, modifiers: 0 };
490 send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 });
491 send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount });
492 send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount });
493 };
494 const onWheel = (event: WheelEvent) => {
495 const point = pagePoint(event.clientX, event.clientY, false);
496 if (!point) return;
497 event.preventDefault();
498 const unit =
499 event.deltaMode === WheelEvent.DOM_DELTA_LINE
500 ? WHEEL_LINE_PX
501 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE
502 ? (viewport?.height ?? 0)
503 : 1;
504 queueWheel(point, event.deltaX * unit, event.deltaY * unit);
505 };
506 const preventDefault = (event: Event) => event.preventDefault();
507
508 const onKey = (action: "down" | "up", event: KeyboardEvent) => {
509 if (control?.controller !== "you") return;
510 if (
511 event.isComposing ||
512 event.keyCode === 229 ||
513 event.key === "Process" ||
514 event.key === "Unidentified"
515 ) {
516 return;
517 }
518 const shortcut = event.ctrlKey || event.metaKey;
519 if (shortcut && event.key.toLowerCase() === "v") return;
520 if (event.shiftKey && event.key === "Insert") return;
521 // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines.
522 const text = shortcut
523 ? undefined
524 : [...event.key].length === 1
525 ? event.key
526 : event.key === "Enter"
527 ? "\r"
528 : undefined;
529 send({
530 type: "key",
531 action,
532 key: event.key,
533 code: event.code,
534 keyCode: event.keyCode,
535 ...(action === "down" && text !== undefined ? { text } : {}),
536 modifiers: previewStreamModifiers(event),
537 });
538 // Some Android keyboards edit the textarea even when keydown is prevented.
539 keySent =
540 action === "down" &&
541 !shortcut &&
542 (text !== undefined || event.key === "Backspace" || event.key === "Delete");
543 if (!shortcut) event.preventDefault();
544 };
545 const resetInput = () => {
546 input.value = SENTINEL;
547 input.setSelectionRange(SENTINEL.length, SENTINEL.length);
548 };
549 const pressKey = (key: "Backspace" | "Delete", keyCode: number) => {
550 for (const action of ["down", "up"] as const) {
551 send({ type: "key", action, key, code: key, keyCode, modifiers: 0 });
552 }
553 };
554 const onInput = (event: Event) => {
555 if (event instanceof InputEvent && event.isComposing) return;
556 const inputType = event instanceof InputEvent ? event.inputType : "";
557 if (keySent) keySent = false;
558 else if (inputType === "deleteContentBackward") pressKey("Backspace", 8);
559 else if (inputType === "deleteContentForward") pressKey("Delete", 46);
560 else {
561 const value = input.value;
562 const text = value.startsWith(SENTINEL) ? value.slice(SENTINEL.length) : value;
563 if (text) send({ type: "text", text });
564 }
565 resetInput();
566 };
567 const onCompositionEnd = (event: CompositionEvent) => {
568 const text = event.data.startsWith(SENTINEL) ? event.data.slice(SENTINEL.length) : event.data;
569 if (text) send({ type: "text", text });
570 resetInput();
571 };
572
573 if (interactive) {
574 canvas.addEventListener("pointerdown", onPointerDown);
575 canvas.addEventListener("pointermove", onPointerMove);
576 canvas.addEventListener("pointerup", onPointerEnd);
577 canvas.addEventListener("pointercancel", onPointerEnd);
578 canvas.addEventListener("wheel", onWheel, { passive: false });
579 canvas.addEventListener("mousedown", preventDefault);
580 canvas.addEventListener("contextmenu", preventDefault);
581 input.addEventListener("keydown", (event) => onKey("down", event));
582 input.addEventListener("keyup", (event) => onKey("up", event));
583 input.addEventListener("input", onInput);
584 input.addEventListener("compositionend", onCompositionEnd);
585 input.addEventListener("focus", resetInput);
586 // Copying the input would put its sentinel on this device's clipboard.
587 input.addEventListener("copy", preventDefault);
588 input.addEventListener("cut", preventDefault);
589 resetInput();
590 }
591
592 // WebKit reports no support on a video that has not loaded yet; check the prototype.
593 const videoPrototype: PresentationVideo = HTMLVideoElement.prototype;
594 const pictureInPictureSupported =
595 interactive &&
596 typeof canvas.captureStream === "function" &&
597 typeof CanvasCaptureMediaStreamTrack === "function" &&
598 ((document.pictureInPictureEnabled === true &&
599 typeof videoPrototype.requestPictureInPicture === "function") ||
600 typeof videoPrototype.webkitSetPresentationMode === "function");
601 // WebKit revokes activation after return; start picture in picture before awaiting. Prepare paused video before the tap.
602 let video: PresentationVideo | null = null;
603 let track: CanvasCaptureMediaStreamTrack | null = null;
604 const pictureInPictureActive = () =>
605 video !== null &&
606 (document.pictureInPictureElement === video ||
607 video.webkitPresentationMode === "picture-in-picture");
608 const reportPictureInPicture = (detail?: string) =>
609 post({
610 type: "pictureInPicture",
611 supported: pictureInPictureSupported,
612 active: pictureInPictureActive(),
613 ...(detail ? { detail } : {}),
614 });
615 // Called after each paint. A frame drawn before the player attached is lost, so retry.
616 const sendPictureInPictureFrame = () => {
617 if (video && (video.readyState === HTMLMediaElement.HAVE_NOTHING || pictureInPictureActive())) {
618 track?.requestFrame();
619 }
620 };
621 if (pictureInPictureSupported) {
622 const stream = canvas.captureStream(0);
623 const [captured] = stream.getVideoTracks();
624 const element: PresentationVideo = document.createElement("video");
625 element.muted = true;
626 element.playsInline = true;
627 // Under the canvas at full size: WebKit pauses muted video it considers off screen.
628 element.srcObject = stream;
629 for (const name of [
630 "enterpictureinpicture",
631 "leavepictureinpicture",
632 "webkitpresentationmodechanged",
633 ]) {
634 element.addEventListener(name, () => {
635 if (!pictureInPictureActive()) element.pause();
636 reportPictureInPicture();
637 });
638 }
639 container.prepend(element);
640 canvas.style.background = configuration.background;
641 video = element;
642 track = captured instanceof CanvasCaptureMediaStreamTrack ? captured : null;
643 }
644 const togglePictureInPicture = () => {
645 const element = video;
646 if (!element) return;
647 if (pictureInPictureActive()) {
648 if (document.pictureInPictureElement) void document.exitPictureInPicture().catch(() => {});
649 else element.webkitSetPresentationMode?.("inline");
650 return;
651 }
652 if (element.readyState === HTMLMediaElement.HAVE_NOTHING) {
653 reportPictureInPicture("The page has not shown yet. Try again in a moment.");
654 return;
655 }
656 // The page may have changed since the video's last frame. WebKit captures only on a draw.
657 track?.requestFrame();
658 canvas.getContext("2d")?.drawImage(canvas, 0, 0);
659 const fail = (error: unknown) => {
660 element.pause();
661 reportPictureInPicture(
662 error instanceof Error ? error.message : "Picture in picture is unavailable.",
663 );
664 };
665 if (document.pictureInPictureEnabled) {
666 const entering = element.requestPictureInPicture();
667 void Promise.all([entering, element.play()]).catch(fail);
668 } else {
669 element.webkitSetPresentationMode?.("picture-in-picture");
670 void element.play().catch(fail);
671 }
672 };
673
674 activeViewer = {
675 stop: () => {
676 stopped = true;
677 observer.disconnect();
678 visualViewport?.removeEventListener("resize", followVisualViewport);
679 visualViewport?.removeEventListener("scroll", followVisualViewport);
680 if (resizeTimer !== null) clearTimeout(resizeTimer);
681 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
682 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
683 if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
684 painter.stop();
685 client?.stop();
686 client = null;
687 track?.stop();
688 if (video) video.srcObject = null;
689 },
690 command: (message) => {
691 send(message);
692 },
693 togglePictureInPicture,
694 };
695 window.addEventListener("pagehide", stop, { once: true });
696 reportStatus("connecting");
697 if (interactive) reportPictureInPicture();
698}