1import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2import { useAtomValue } from "@effect/atom-react";
3import { AsyncResult, Atom } from "effect/reactivity";
4import { useEffect, useId, useRef, useState, type ReactNode } from "react";
5
6import { cn } from "~/lib/utils";
7import { Button } from "~/components/ui/button";
8import { renewDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
9import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile";
10import { deviceKeyboard, deviceModel } from "./deviceModels";
11import { fitDeviceFrame } from "./deviceFrameLayout";
12import { DeviceDuoViewport } from "./DeviceDuoViewport";
13import { DeviceDuoControls } from "./DeviceDuoControls";
14import { DeviceAndroidFoldControls } from "./DeviceAndroidFoldControls";
15import { DevicePhoneViewport } from "./DevicePhoneViewport";
16import { DeviceLoadingView } from "./DeviceLoadingView";
17import type { DeviceAxElement } from "./deviceHubApi";
18import { androidFoldViewAtom, deviceAxElementsAtom } from "./deviceHubAtoms";
19import {
20 deviceScreenKey,
21 deviceStreamSessionAtom,
22 type DeviceStreamSession,
23 IDLE_DEVICE_STREAM_STATE,
24} from "./deviceStreamSession";
25import type { DeviceHardwareButton, DeviceScreenSize } from "@t3tools/client-runtime/device/stream";
26import { observeResize } from "~/lib/observeResize";
27
28const NO_SESSION_ATOM = Atom.make<DeviceStreamSession | null>(null).pipe(
29 Atom.withLabel("device-stream:none"),
30);
31const NO_AX_ELEMENTS: ReadonlyArray<DeviceAxElement> = [];
32const NO_AX_ATOM = Atom.make(AsyncResult.initial<ReadonlyArray<DeviceAxElement>, never>()).pipe(
33 Atom.withLabel("device-ax-elements:none"),
34);
35const CONTROLS_RAIL_WIDTH = 56;
36
37function screenAspect(platform: DevicePlatform, screen: DeviceScreenSize | null): number {
38 if (!screen) return platform === "ios" ? 9 / 19.5 : 9 / 20;
39 const landscape =
40 screen.orientation === "landscape_left" || screen.orientation === "landscape_right";
41 const w = landscape
42 ? Math.max(screen.width, screen.height)
43 : Math.min(screen.width, screen.height);
44 const h = landscape
45 ? Math.min(screen.width, screen.height)
46 : Math.max(screen.width, screen.height);
47 return w / h;
48}
49
50// serve-sim sends raw portrait framebuffers even when iOS reports landscape; rotate the display.
51function screenRotation(platform: DevicePlatform, screen: DeviceScreenSize | null): number {
52 if (platform !== "ios" || !screen || screen.width > screen.height) return 0;
53 switch (screen.orientation) {
54 case "landscape_left":
55 return 90;
56 case "landscape_right":
57 return -90;
58 case "portrait_upside_down":
59 return 180;
60 default:
61 return 0;
62 }
63}
64
65export interface DeviceViewControls {
66 readonly phone: boolean;
67 readonly streaming: boolean;
68 readonly phoneUnavailableReason: string | null;
69 readonly showPhone: () => void;
70 readonly showFlat: () => void;
71 readonly resetView: () => void;
72 readonly foldingControls?: ReactNode;
73 readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null;
74 readonly handle: DeviceStreamHandle | null;
75}
76
77export interface DeviceStreamHandle {
78 readonly pressButton: (button: DeviceHardwareButton) => void;
79 readonly rotate: () => void;
80 readonly inputConnected: boolean;
81}
82
83export function DeviceStreamView(props: {
84 readonly environmentId: EnvironmentId;
85 readonly platform: DevicePlatform;
86 readonly deviceId: string;
87 readonly deviceName?: string;
88 readonly deviceDescription?: string;
89 readonly visible: boolean;
90 readonly hostId: string;
91 readonly allowPhoneView?: boolean;
92 readonly renderControls?: (view: DeviceViewControls) => ReactNode;
93 readonly axOverlay?: boolean;
94 /** Lets a parent read this view's own stream state, such as its screen size. */
95 readonly viewId?: string;
96 /** The host serves only serve-sim's stream and input socket (Simlock). */
97 readonly serveSimOnly?: boolean;
98}) {
99 const model = deviceModel(props.platform, props.deviceName ?? "");
100 const isDuo = model?.id === "iphone-duo";
101 const [presentation, setPresentation] = useState<"phone" | "flat">("phone");
102 const [keyboardAttached, setKeyboardAttached] = useState(false);
103 const cancelPhoneInputRef = useRef<(() => void) | null>(null);
104 const cancelPhoneInput = () => cancelPhoneInputRef.current?.();
105 const resetViewRef = useRef<(() => void) | null>(null);
106 const onResetReady = (reset: (() => void) | null) => {
107 resetViewRef.current = reset;
108 };
109 const onInputCancel = (cancel: (() => void) | null) => {
110 cancelPhoneInputRef.current = cancel;
111 };
112 const access = useDeviceHubAccess(props.environmentId, props.hostId);
113 const ownViewId = useId();
114 const viewId = props.viewId ?? ownViewId;
115 const session = useAtomValue(
116 access && props.visible
117 ? deviceStreamSessionAtom({
118 environmentId: props.environmentId,
119 hostId: props.hostId,
120 platform: props.platform,
121 deviceId: props.deviceId,
122 access,
123 serveSimOnly: props.serveSimOnly === true,
124 viewId,
125 })
126 : NO_SESSION_ATOM,
127 );
128 const client = session?.stream ?? null;
129 const canvas = session?.canvas ?? null;
130 const { status, detail, hasFrame, screen, mjpegUrl, mjpegGeneration, duoControl } =
131 session?.state ?? IDLE_DEVICE_STREAM_STATE;
132 const inputState = session?.state.input ?? IDLE_DEVICE_STREAM_STATE.input;
133 const phoneUnavailable = session?.state.phoneUnavailable ?? false;
134 const foldAngle = useAtomValue(
135 androidFoldViewAtom(deviceScreenKey(props.environmentId, props.hostId, props.deviceId)),
136 ).angle;
137 const attachMjpegImage = (image: HTMLImageElement | null) => {
138 if (!client || !image) return;
139 client.setMjpegImage(image);
140 return () => client.setMjpegImage(null);
141 };
142 const mountCanvas = (node: HTMLDivElement | null) => {
143 if (!node || !canvas) return;
144 node.append(canvas);
145 return () => canvas.remove();
146 };
147
148 const aspect = screenAspect(props.platform, screen);
149
150 const retainingFrame =
151 status === "connecting" && hasFrame && inputState.connected && screen !== null;
152 const showPhone =
153 props.allowPhoneView &&
154 (status === "streaming" || retainingFrame) &&
155 props.visible &&
156 presentation === "phone" &&
157 !phoneUnavailable &&
158 !mjpegUrl &&
159 !props.axOverlay &&
160 (!isDuo || screen?.supportsHingeAngle === true);
161 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
162
163 // Fit the frame in both dimensions; pinned CSS height plus a width clamp distorts aspect-ratio.
164 const [host, setHost] = useState({ width: 0, height: 0 });
165 const hostRef = (element: HTMLDivElement | null) => {
166 if (!element) return;
167 const update = () => {
168 const rect = element.getBoundingClientRect();
169 setHost((current) =>
170 current.width === rect.width && current.height === rect.height
171 ? current
172 : { width: rect.width, height: rect.height },
173 );
174 };
175 update();
176 return observeResize(element, update);
177 };
178 const frame = fitDeviceFrame(aspect, host.width, host.height, controlsInset);
179
180 const rotation = screenRotation(props.platform, screen);
181
182 const sideways = rotation === 90 || rotation === -90;
183 const mediaStyle: React.CSSProperties = sideways
184 ? {
185 width: frame.height,
186 height: frame.width,
187 left: (frame.width - frame.height) / 2,
188 top: (frame.height - frame.width) / 2,
189 transform: `rotate(${rotation}deg)`,
190 }
191 : {
192 width: frame.width,
193 height: frame.height,
194 ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}),
195 };
196
197 const axResult = useAtomValue(
198 props.axOverlay && access && props.visible
199 ? deviceAxElementsAtom({ access, platform: props.platform, deviceId: props.deviceId })
200 : NO_AX_ATOM,
201 );
202 const axElements = AsyncResult.getOrElse(axResult, () => NO_AX_ELEMENTS);
203
204 const activePointer = useRef<{ id: number; x: number; y: number } | null>(null);
205 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- clears pointer ownership when its device input connection disconnects or is replaced
206 useEffect(() => {
207 if (!client || !inputState.connected) return;
208 return () => {
209 activePointer.current = null;
210 };
211 }, [client, inputState.connected]);
212 const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
213 const rect = event.currentTarget.getBoundingClientRect();
214 const x = (event.clientX - rect.left) / rect.width;
215 const y = (event.clientY - rect.top) / rect.height;
216 return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
217 };
218
219 const endPointer = (event: React.PointerEvent<HTMLElement>) => {
220 const pointer = activePointer.current;
221 if (pointer?.id !== event.pointerId) return;
222 activePointer.current = null;
223 const { x, y } = event.type === "lostpointercapture" ? pointer : normalizedPoint(event);
224 client?.sendTouch("end", x, y);
225 };
226
227 const phoneUnavailableReason =
228 isDuo && !screen?.supportsHingeAngle
229 ? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
230 : phoneUnavailable
231 ? "3D is unavailable on this browser"
232 : mjpegUrl
233 ? "3D requires the H.264 stream"
234 : props.axOverlay
235 ? "Turn off accessibility frames to use 3D"
236 : null;
237
238 const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? "");
239 const resetView = () => {
240 if (!isDuo) {
241 const orientation = keyboardAttached ? "landscape_right" : "portrait";
242 if (screen?.orientation !== orientation) client?.setOrientation(orientation);
243 }
244 resetViewRef.current?.();
245 };
246 const profile = resolveDeviceShape({
247 platform: props.platform,
248 name: props.deviceName ?? "",
249 portraitAspect: Math.min(aspect, 1 / aspect),
250 });
251
252 return (
253 <div
254 className={cn(
255 "relative flex size-full min-h-0 min-w-0",
256 props.allowPhoneView ? "bg-background" : "bg-black/90",
257 )}
258 >
259 {props.renderControls ? (
260 <DeviceControlsSlot
261 renderControls={props.renderControls}
262 view={{
263 phone: !!showPhone,
264 streaming: status === "streaming",
265 phoneUnavailableReason,
266 foldingControls:
267 props.platform === "android" && access ? (
268 <DeviceAndroidFoldControls
269 key={`${props.hostId}:${props.deviceId}`}
270 access={access}
271 deviceId={props.deviceId}
272 visible={props.visible}
273 enabled={status === "streaming"}
274 screenWidth={screen?.width}
275 screenHeight={screen?.height}
276 deviceKey={deviceScreenKey(props.environmentId, props.hostId, props.deviceId)}
277 renewAccess={() => renewDeviceHubAccess(props.environmentId, props.hostId)}
278 />
279 ) : showPhone && isDuo && screen?.supportsHingeAngle ? (
280 <DeviceDuoControls
281 screen={screen}
282 state={duoControl}
283 enabled={inputState.connected}
284 onCommand={(command) => {
285 cancelPhoneInput();
286 client?.controlDuo(command);
287 }}
288 />
289 ) : null,
290 keyboard:
291 showPhone && keyboardSource
292 ? {
293 attached: keyboardAttached,
294 toggle: () => {
295 cancelPhoneInput();
296 if (!keyboardAttached && screen?.orientation !== "landscape_right")
297 client?.setOrientation("landscape_right");
298 setKeyboardAttached(!keyboardAttached);
299 },
300 }
301 : null,
302 showPhone: () => setPresentation("phone"),
303 showFlat: () => setPresentation("flat"),
304 handle: client
305 ? {
306 pressButton: client.pressButton,
307 rotate: client.rotate,
308 inputConnected: inputState.connected,
309 }
310 : null,
311 }}
312 onResetView={resetView}
313 />
314 ) : null}
315 <div
316 ref={hostRef}
317 className={cn(
318 "relative flex min-h-0 min-w-0 flex-1 items-center justify-center overflow-hidden outline-none",
319 // Overlay so the ring paints above the stream canvas.
320 "after:pointer-events-none after:absolute after:inset-0 after:z-10 focus-visible:after:ring-2 focus-visible:after:ring-inset focus-visible:after:ring-ring",
321 controlsInset && "pr-14",
322 )}
323 tabIndex={0}
324 role="application"
325 aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`}
326 onKeyDown={(event) => {
327 if (event.target !== event.currentTarget) return;
328 if (event.metaKey && !["r", "R"].includes(event.key)) return;
329 // Let browser paste chords fire the paste event.
330 if (
331 (event.ctrlKey && (event.key === "v" || event.key === "V")) ||
332 (event.shiftKey && event.key === "Insert")
333 ) {
334 return;
335 }
336 event.preventDefault();
337 client?.sendKey(event.nativeEvent, "down");
338 }}
339 onPaste={(event) => {
340 if (event.target !== event.currentTarget) return;
341 event.preventDefault();
342 client?.sendText(event.clipboardData.getData("text/plain"));
343 }}
344 onKeyUp={(event) => {
345 if (event.target !== event.currentTarget) return;
346 client?.sendKey(event.nativeEvent, "up");
347 }}
348 >
349 <div
350 className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
351 style={{ width: frame.width, height: frame.height }}
352 onPointerDown={(event) => {
353 if (event.button !== 0 || !inputState.connected || activePointer.current !== null)
354 return;
355 event.currentTarget.setPointerCapture(event.pointerId);
356 (event.currentTarget.parentElement as HTMLElement | null)?.focus();
357 const { x, y } = normalizedPoint(event);
358 activePointer.current = { id: event.pointerId, x, y };
359 client?.sendTouch("begin", x, y);
360 }}
361 onPointerMove={(event) => {
362 if (activePointer.current?.id !== event.pointerId) return;
363 const { x, y } = normalizedPoint(event);
364 activePointer.current = { id: event.pointerId, x, y };
365 client?.sendTouch("move", x, y);
366 }}
367 onPointerUp={endPointer}
368 onPointerCancel={endPointer}
369 onLostPointerCapture={endPointer}
370 >
371 <div
372 ref={mountCanvas}
373 className={cn("absolute top-0 left-0", mjpegUrl && "hidden")}
374 style={mediaStyle}
375 />
376 {props.visible && access && mjpegUrl ? (
377 <img
378 key={mjpegGeneration}
379 ref={attachMjpegImage}
380 alt=""
381 draggable={false}
382 className="absolute top-0 left-0 object-contain"
383 style={mediaStyle}
384 />
385 ) : null}
386 {axElements.length > 0 ? (
387 <div className="pointer-events-none absolute inset-0" aria-hidden>
388 {axElements.map((element) => (
389 <div
390 key={element.id}
391 className="absolute border border-info/80 bg-info/10"
392 style={{
393 left: `${element.x * 100}%`,
394 top: `${element.y * 100}%`,
395 width: `${element.width * 100}%`,
396 height: `${element.height * 100}%`,
397 }}
398 >
399 {element.label ? (
400 <span className="absolute -top-3.5 left-0 max-w-full truncate rounded-sm bg-info px-1 text-3xs leading-3.5 text-white">
401 {element.label}
402 </span>
403 ) : null}
404 </div>
405 ))}
406 </div>
407 ) : null}
408 </div>
409 {session && showPhone && isDuo && model ? (
410 <DeviceDuoViewport
411 onFrameListener={session.setFrameListener}
412 onControlRejectedListener={session.setControlRejectedListener}
413 model={model}
414 hingePreview={
415 duoControl.requested?.control === "angle" ? duoControl.requested.value : null
416 }
417 source={session.canvas}
418 client={session.stream}
419 onInputCancel={onInputCancel}
420 onResetReady={onResetReady}
421 screen={screen}
422 onUnavailable={session.markPhoneUnavailable}
423 />
424 ) : session && showPhone ? (
425 <DevicePhoneViewport
426 profile={profile}
427 model={deviceModel(props.platform, props.deviceName ?? "")}
428 accessory={keyboardAttached ? keyboardSource : null}
429 source={session.canvas}
430 onFrameListener={session.setFrameListener}
431 client={session.stream}
432 onInputCancel={onInputCancel}
433 onResetReady={onResetReady}
434 screen={screen}
435 foldAngle={foldAngle}
436 onUnavailable={session.markPhoneUnavailable}
437 />
438 ) : null}
439 {props.allowPhoneView && !props.renderControls && status === "streaming" ? (
440 <div className="absolute top-3 left-3 flex gap-1 rounded-lg border border-border/50 bg-background/90 p-1 shadow-sm">
441 <Button
442 variant={showPhone ? "secondary" : "ghost"}
443 size="xs"
444 aria-pressed={!!showPhone}
445 disabled={!!phoneUnavailableReason}
446 title={phoneUnavailableReason ?? "Show 3D phone"}
447 onClick={() => setPresentation("phone")}
448 >
449 3D
450 </Button>
451 <Button
452 variant={!showPhone ? "secondary" : "ghost"}
453 size="xs"
454 aria-pressed={!showPhone}
455 onClick={() => setPresentation("flat")}
456 >
457 Flat
458 </Button>
459 </div>
460 ) : null}
461 {status === "streaming" && !inputState.connected ? (
462 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
463 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
464 Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting…
465 </span>
466 </div>
467 ) : null}
468 {retainingFrame && showPhone ? (
469 // Shown only once the device has been silent for two seconds.
470 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2 transition-opacity delay-2000 duration-0 starting:opacity-0">
471 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
472 Waiting for device video…
473 </span>
474 </div>
475 ) : null}
476 {status !== "streaming" && !(retainingFrame && showPhone) ? (
477 <div className="absolute inset-0">
478 <DeviceLoadingView
479 name={props.deviceName ?? "Device"}
480 description={props.deviceDescription ?? ""}
481 stage="stream"
482 message={status === "error" ? (detail ?? "Stream failed.") : "Connecting video…"}
483 error={status === "error"}
484 >
485 {status === "error" ? (
486 <Button
487 size="sm"
488 variant="outline"
489 onClick={() => {
490 // An expired ticket surfaces as unauthorized on restart and
491 // refreshes access, which starts a new session.
492 session?.reconnect();
493 }}
494 >
495 Reconnect
496 </Button>
497 ) : null}
498 </DeviceLoadingView>
499 </div>
500 ) : null}
501 </div>
502 </div>
503 );
504}
505
506function DeviceControlsSlot(props: {
507 renderControls: (view: DeviceViewControls) => ReactNode;
508 view: Omit<DeviceViewControls, "resetView">;
509 onResetView: () => void;
510}) {
511 return props.renderControls({ ...props.view, resetView: props.onResetView });
512}