1// @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime.
2
3import {
4 createDuoControl,
5 type DuoCommand,
6 type DuoControlState,
7 type DuoPose,
8} from "./duoControl.ts";
9import * as Schema from "effect/Schema";
10import * as Option from "effect/Option";
11import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
12import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
13import type { DevicePlatform } from "@t3tools/contracts";
14
15export type DeviceStreamStatus = "connecting" | "streaming" | "error";
16
17export interface DeviceScreenSize {
18 readonly width: number;
19 readonly height: number;
20 readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right";
21 readonly screenId?: number;
22 readonly supportsHingeAngle?: boolean;
23 readonly supportsPhysicalOrientation?: boolean;
24 readonly hingeAngle?: number;
25 readonly hingePose?: DuoPose | null;
26 readonly tableMode?: boolean;
27 readonly tableModeAvailable?: boolean;
28}
29
30const screenConfigSchema = Schema.Struct({
31 width: Schema.Finite.check(Schema.isGreaterThan(0)),
32 height: Schema.Finite.check(Schema.isGreaterThan(0)),
33 orientation: Schema.Literals([
34 "portrait",
35 "portrait_upside_down",
36 "landscape_left",
37 "landscape_right",
38 ]),
39 inputUnavailable: Schema.optionalKey(Schema.Boolean),
40 screenId: Schema.optionalKey(Schema.Number),
41 supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
42 supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
43 hingeAngle: Schema.optionalKey(
44 Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })),
45 ),
46 hingePose: Schema.optionalKey(
47 Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])),
48 ),
49 tableMode: Schema.optionalKey(Schema.Boolean),
50 tableModeAvailable: Schema.optionalKey(Schema.Boolean),
51});
52const controlReplySchema = Schema.Struct({
53 requestId: Schema.Int,
54 ok: Schema.Boolean,
55 error: Schema.optionalKey(Schema.String),
56});
57const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
58const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
59const iceServerSchema = Schema.Struct({
60 urls: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
61 username: Schema.optionalKey(Schema.String),
62 credential: Schema.optionalKey(Schema.String),
63});
64const webRtcSettingsSchema = Schema.Struct({
65 transport: Schema.Literal("webrtc"),
66 iceServers: Schema.optionalKey(Schema.Array(iceServerSchema)),
67 iceTransportPolicy: Schema.optionalKey(Schema.Literals(["all", "relay"])),
68});
69// serve-sim reports its stream settings as `streamSettings` (EAS) or `stream`.
70const serveSimApiSchema = Schema.Struct({
71 streamSettings: Schema.optionalKey(Schema.Unknown),
72 stream: Schema.optionalKey(Schema.Unknown),
73});
74const decodeServeSimApi = Schema.decodeUnknownOption(serveSimApiSchema);
75const decodeWebRtcSettings = Schema.decodeUnknownOption(webRtcSettingsSchema);
76type WebRtcSettings = typeof webRtcSettingsSchema.Type;
77const decodeTransportLocked = Schema.decodeUnknownOption(
78 Schema.Struct({
79 error: Schema.Literal("stream_transport_locked"),
80 transport: Schema.Literal("webrtc"),
81 }),
82);
83const decodeWebRtcAnswer = Schema.decodeUnknownOption(
84 Schema.Struct({ type: Schema.Literal("answer"), sdp: Schema.String }),
85);
86
87export interface DuoPanelSinks {
88 readonly cover: DeviceFrameSink;
89 readonly inner: DeviceFrameSink;
90 /** Invalidate captured input synchronously, before React can commit the new layout. */
91 readonly onScreen?: (screen: DeviceScreenSize) => void;
92}
93
94export interface DeviceStreamEvents {
95 readonly onDuoControl?: (state: DuoControlState) => void;
96 readonly onDuoUnavailable?: (detail?: string) => void;
97 readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;
98 readonly onScreen: (screen: DeviceScreenSize) => void;
99 readonly onUnauthorized: () => void;
100 /**
101 * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is
102 * unsupported); the owner should show an `<img>` instead of the canvas
103 * and attach it with `setMjpegImage` so the client can observe real frames.
104 */
105 readonly onMjpegFallback: (url: string) => void;
106 /**
107 * The stream moved from MJPEG back to painting the canvas, because the
108 * device serves WebRTC only; the owner should show the canvas again.
109 */
110 readonly onCanvasVideo?: () => void;
111 readonly onInputConnected: (connected: boolean, detail?: string) => void;
112}
113
114export interface DeviceStreamTarget {
115 readonly platform: DevicePlatform;
116 readonly deviceId: string;
117 readonly access: DeviceHubAccess;
118 /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */
119 readonly preferMjpeg?: boolean;
120 readonly panelId?: 1 | 3;
121 readonly videoOnly?: boolean;
122 /** The host speaks only serve-sim's AVCC stream and input socket, on both platforms (Simlock). */
123 readonly serveSimOnly?: boolean;
124}
125
126export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher";
127
128const RETRY_DELAY_MS = 1_000;
129const FIRST_FRAME_TIMEOUT_MS = 15_000;
130// A WebRTC feed that shows no frame for this long is reconnected; checked
131// every WEBRTC_STALL_CHECK_MS, so a transient `disconnected` peer that
132// recovers is left alone.
133const WEBRTC_STALL_TIMEOUT_MS = FIRST_FRAME_TIMEOUT_MS;
134const WEBRTC_STALL_CHECK_MS = 5_000;
135const TRANSPORT_PROBE_TIMEOUT_MS = 5_000;
136// Offers carry every candidate: serve-sim does not accept trickled ICE.
137const ICE_GATHERING_TIMEOUT_MS = 2_000;
138// A gathering cut short without a reflexive or relay candidate is retried
139// with twice the deadline, up to this; then the offer goes as gathered. The
140// whole sequence stays inside the first-frame deadline, which retries do
141// not extend.
142const ICE_GATHERING_MAX_TIMEOUT_MS = 4_000;
143const MJPEG_FRAME_CHECK_MS = 250;
144const FRAME_DURATION_US = 16_667;
145const SEMU_MAGIC = 0x53454d55;
146const SEMU_HEADER_BYTES = 16;
147const SEMU_FLAG_KEY = 1;
148const SOFT_DECODE_QUEUE = 8;
149// expo-device-hub 0.15.3 serve-emu input.js MAX_TEXT_BYTES limits packets to 300 UTF-8 bytes.
150const ANDROID_MAX_TEXT_BYTES = 300;
151// The pinned hub queue depth is 128; the client does not read queue rejections.
152const ANDROID_MAX_TEXT_CHUNKS = 64;
153
154const IOS_MSG_TOUCH = 0x03;
155const IOS_MSG_BUTTON = 0x04;
156const IOS_MSG_KEY = 0x06;
157const IOS_MSG_ORIENTATION = 0x07;
158const IOS_TAG_SCREEN_CONFIG = 0x82;
159const IOS_TAG_INPUT_ADMITTED = 0x83;
160
161const encoder = new TextEncoder();
162const decoder = new TextDecoder();
163
164/** serve-sim requires a v4 UUID; `crypto.randomUUID` is missing over plain-HTTP LAN origins. */
165function webRtcSessionId(): string {
166 const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
167 bytes[6] = (bytes[6]! & 0x0f) | 0x40;
168 bytes[8] = (bytes[8]! & 0x3f) | 0x80;
169 const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
170 return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
171}
172
173const isWebCodecsSupported = (): boolean =>
174 typeof globalThis !== "undefined" &&
175 "VideoDecoder" in globalThis &&
176 "EncodedVideoChunk" in globalThis;
177
178function utf8ByteLength(codePoint: number): number {
179 if (codePoint <= 0x7f) return 1;
180 if (codePoint <= 0x7ff) return 2;
181 if (codePoint <= 0xffff) return 3;
182 return 4;
183}
184
185function taggedJson(tag: number, payload: unknown): Uint8Array<ArrayBuffer> {
186 const json = encoder.encode(JSON.stringify(payload));
187 const out = new Uint8Array(1 + json.length);
188 out[0] = tag;
189 out.set(json, 1);
190 return out;
191}
192
193export function avcCodecString(bytes: Uint8Array): string {
194 if (bytes.length < 4) return "avc1.42E01E";
195 const hex = (byte: number) => byte.toString(16).padStart(2, "0");
196 return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`;
197}
198
199export function parseSemuPacket(raw: ArrayBuffer): {
200 readonly data: Uint8Array;
201 readonly isKey: boolean | null;
202 readonly timestamp: number | null;
203} {
204 const bytes = new Uint8Array(raw);
205 if (bytes.byteLength > SEMU_HEADER_BYTES) {
206 const view = new DataView(raw, 0, SEMU_HEADER_BYTES);
207 if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) {
208 const pts = view.getBigUint64(8, false);
209 return {
210 data: bytes.subarray(SEMU_HEADER_BYTES),
211 isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0,
212 timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null,
213 };
214 }
215 }
216 return { data: bytes, isKey: null, timestamp: null };
217}
218
219const isVideoSessionMessage = (text: string) => {
220 try {
221 const message = JSON.parse(text) as { type?: unknown };
222 return message.type === "video-session";
223 } catch {
224 return false;
225 }
226};
227
228export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } {
229 let isKey = false;
230 let sps: Uint8Array | null = null;
231 const len = buf.length;
232 let i = 0;
233 while (i + 2 < len) {
234 if (buf[i] === 0 && buf[i + 1] === 0) {
235 let codeLen = 0;
236 if (buf[i + 2] === 1) codeLen = 3;
237 else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4;
238 if (codeLen) {
239 const nalType = buf[i + codeLen]! & 0x1f;
240 if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen);
241 if (nalType === 5) isKey = true;
242 i += codeLen + 1;
243 continue;
244 }
245 }
246 i++;
247 }
248 return { isKey, sps };
249}
250
251export type AvccChunk = {
252 readonly type: "description" | "keyframe" | "delta" | "seed";
253 readonly payload: Uint8Array;
254};
255
256const AVCC_TAGS: Record<number, AvccChunk["type"] | undefined> = {
257 1: "description",
258 2: "keyframe",
259 3: "delta",
260 4: "seed",
261};
262
263export class AvccDemuxer {
264 private buffer = new Uint8Array(64 * 1024);
265 private length = 0;
266
267 push(bytes: Uint8Array): AvccChunk[] {
268 if (this.length + bytes.length > this.buffer.length) {
269 let capacity = this.buffer.length;
270 while (capacity < this.length + bytes.length) capacity *= 2;
271 const grown = new Uint8Array(capacity);
272 grown.set(this.buffer.subarray(0, this.length));
273 this.buffer = grown;
274 }
275 this.buffer.set(bytes, this.length);
276 this.length += bytes.length;
277
278 const chunks: AvccChunk[] = [];
279 let offset = 0;
280 while (this.length - offset >= 4) {
281 const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4);
282 const frameLength = view.getUint32(0, false);
283 if (this.length - offset - 4 < frameLength) break;
284 if (frameLength >= 1) {
285 const type = AVCC_TAGS[this.buffer[offset + 4]!];
286 if (type) {
287 chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) });
288 }
289 }
290 offset += 4 + frameLength;
291 }
292 if (offset > 0) {
293 this.buffer.copyWithin(0, offset, this.length);
294 this.length -= offset;
295 }
296 return chunks;
297 }
298
299 reset(): void {
300 this.length = 0;
301 }
302}
303
304export interface DeviceStreamClient {
305 readonly start: () => void;
306 readonly stop: () => void;
307 /**
308 * `stop()` detaches it; attach a fresh image for each restart.
309 */
310 readonly setMjpegImage: (image: HTMLImageElement | null) => void;
311 /** Normalized 0..1 coordinates in the displayed frame. */
312 readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
313 readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void;
314 /** Android only: iOS input is HID key codes, which cannot carry arbitrary text. */
315 readonly sendText: (text: string) => void;
316 readonly pressButton: (button: DeviceHardwareButton) => void;
317 readonly rotate: () => void;
318 readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
319 readonly controlDuo: (command: DuoCommand) => void;
320 readonly setDuoPanels: (panels: DuoPanelSinks | null) => void;
321 readonly sendRawTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
322}
323
324const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
325 Enter: 0x28,
326 Escape: 0x29,
327 Backspace: 0x2a,
328 Tab: 0x2b,
329 Space: 0x2c,
330 Minus: 0x2d,
331 Equal: 0x2e,
332 BracketLeft: 0x2f,
333 BracketRight: 0x30,
334 Backslash: 0x31,
335 Semicolon: 0x33,
336 Quote: 0x34,
337 Backquote: 0x35,
338 Comma: 0x36,
339 Period: 0x37,
340 Slash: 0x38,
341 Delete: 0x4c,
342 ArrowRight: 0x4f,
343 ArrowLeft: 0x50,
344 ArrowDown: 0x51,
345 ArrowUp: 0x52,
346 NumpadDivide: 0x54,
347 NumpadMultiply: 0x55,
348 NumpadSubtract: 0x56,
349 NumpadAdd: 0x57,
350 NumpadEnter: 0x58,
351 Numpad1: 0x59,
352 Numpad2: 0x5a,
353 Numpad3: 0x5b,
354 Numpad4: 0x5c,
355 Numpad5: 0x5d,
356 Numpad6: 0x5e,
357 Numpad7: 0x5f,
358 Numpad8: 0x60,
359 Numpad9: 0x61,
360 Numpad0: 0x62,
361 NumpadDecimal: 0x63,
362 NumpadEqual: 0x67,
363 ControlLeft: 0xe0,
364 ShiftLeft: 0xe1,
365 AltLeft: 0xe2,
366 MetaLeft: 0xe3,
367 ControlRight: 0xe4,
368 ShiftRight: 0xe5,
369 AltRight: 0xe6,
370 MetaRight: 0xe7,
371};
372
373function hidUsageForCode(code: string): number | null {
374 if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65);
375 if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49);
376 if (code === "Digit0") return 0x27;
377 return HID_USAGE_BY_CODE[code] ?? null;
378}
379
380const ANDROID_KEYCODE_BY_KEY: Readonly<Record<string, number>> = {
381 ArrowUp: 19,
382 ArrowDown: 20,
383 ArrowLeft: 21,
384 ArrowRight: 22,
385 Tab: 61,
386 Enter: 66,
387 Backspace: 67,
388 Delete: 112,
389 Home: 122,
390 End: 123,
391 PageUp: 92,
392 PageDown: 93,
393};
394
395const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [
396 "portrait",
397 "landscape_left",
398 "portrait_upside_down",
399 "landscape_right",
400];
401
402export function createDeviceStreamClient(
403 target: DeviceStreamTarget,
404 output: HTMLCanvasElement | DeviceFrameSink,
405 events: DeviceStreamEvents,
406): DeviceStreamClient {
407 const { access, platform, deviceId } = target;
408 const sink = "present" in output ? output : createCanvasFrameSink(output);
409 // Android normally streams over serve-emu's socket; a serve-sim-only host serves it like iOS.
410 const serveSimOnly = target.serveSimOnly === true;
411 const avcc = platform === "ios" || serveSimOnly;
412 const vendor = avcc ? "/vendor/serve-sim" : "/vendor/serve-emu";
413 const device = encodeURIComponent(deviceId);
414 const httpUrl = (path: string) =>
415 withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access);
416 const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access);
417 const useWebCodecs =
418 isWebCodecsSupported() && !(platform === "ios" && target.preferMjpeg && !serveSimOnly);
419
420 let stopped = true;
421 let socket: WebSocket | null = null;
422 let controller: AbortController | null = null;
423 const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();
424 let primeController: AbortController | null = null;
425 let videoDecoder: VideoDecoder | null = null;
426 let decodedVideoGeneration: number | null = null;
427 let iosInputAdmitted = false;
428 let iosInputUnavailable = false;
429 let timestamp = 0;
430 let awaitingKeyframe = true;
431 let screen: DeviceScreenSize | null = null;
432 let firstFrame = false;
433 let configuring = false;
434 let mjpeg = false;
435 let generation = 0;
436 let decoderEpoch = 0;
437 let frameTimer: ReturnType<typeof setTimeout> | null = null;
438 let videoReadTimer: ReturnType<typeof setTimeout> | null = null;
439 let mjpegImage: HTMLImageElement | null = null;
440 let releaseImage: (() => void) | null = null;
441 let videoGeneration = 0;
442 let panelClients: DeviceStreamClient[] = [];
443 let panelSinks: DuoPanelSinks | null = null;
444 let rotationCursor: DeviceScreenSize["orientation"] | null = null;
445 let pendingOrientation: { requestId: number } | null = null;
446 let iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
447 let peer: { connection: RTCPeerConnection; sessionId: string; video: HTMLVideoElement } | null =
448 null;
449 const duoControl = createDuoControl({
450 send(request) {
451 if (
452 !iosInputAdmitted ||
453 iosInputUnavailable ||
454 socket?.readyState !== WebSocket.OPEN ||
455 !screen?.supportsHingeAngle
456 )
457 return false;
458 if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
459 return false;
460 try {
461 pendingOrientation = null;
462 if (request.command.control === "orientation") {
463 const value = request.command.value;
464 pendingOrientation = { requestId: request.requestId };
465 rotationCursor = value;
466 // Upstream serializes orientation with hinge commands, then broadcasts config.
467 // An orientation-locked app can keep its framebuffer orientation after the sensor rotates.
468 socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value }));
469 } else socket.send(taggedJson(0x10, request));
470 return true;
471 } catch {
472 return false;
473 }
474 },
475 onChange(state) {
476 if (!state.pending) pendingOrientation = null;
477 events.onDuoControl?.(state);
478 },
479 });
480 const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`;
481
482 const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
483
484 const setStatus = (status: DeviceStreamStatus, detail?: string) => {
485 if (!stopped) events.onStatus(status, detail);
486 };
487
488 const clearFrameTimer = () => {
489 if (frameTimer !== null) clearTimeout(frameTimer);
490 frameTimer = null;
491 };
492
493 const fail = (detail: string) => {
494 if (stopped) return;
495 stop();
496 events.onInputConnected(false, detail);
497 events.onStatus("error", detail);
498 };
499
500 const connecting = (detail?: string) => {
501 firstFrame = false;
502 if (frameTimer === null) {
503 frameTimer = setTimeout(
504 () => fail("No video received from the device. Reconnect to try again."),
505 FIRST_FRAME_TIMEOUT_MS,
506 );
507 }
508 setStatus("connecting", detail);
509 };
510
511 const frameReceived = () => {
512 if (firstFrame) return;
513 firstFrame = true;
514 clearFrameTimer();
515 setStatus("streaming");
516 };
517
518 const observeMjpegImage = () => {
519 releaseImage?.();
520 releaseImage = null;
521 const image = mjpegImage;
522 if (!image || stopped || !mjpeg) return;
523 let timer: ReturnType<typeof setTimeout> | null = null;
524 let released = false;
525 const check = () => {
526 if (released || stopped) return;
527 if (timer !== null) clearTimeout(timer);
528 timer = null;
529 if (image.naturalWidth > 0 && image.naturalHeight > 0) frameReceived();
530 else {
531 // Multipart images may not emit load until the response ends.
532 timer = setTimeout(check, MJPEG_FRAME_CHECK_MS);
533 }
534 };
535 const error = () => {
536 if (released) return;
537 const failure = "Could not receive the device stream. Reconnect to try again.";
538 // An <img> cannot read the 409 that a WebRTC-only serve-sim answers with.
539 if (typeof RTCPeerConnection === "undefined") fail(failure);
540 else void switchToWebRtc(failure, false);
541 };
542 image.addEventListener("load", check);
543 image.addEventListener("error", error);
544 releaseImage = () => {
545 released = true;
546 if (timer !== null) clearTimeout(timer);
547 image.removeEventListener("load", check);
548 image.removeEventListener("error", error);
549 image.removeAttribute("src");
550 };
551 image.src = mjpegUrl();
552 check();
553 };
554
555 const setMjpegImage = (image: HTMLImageElement | null) => {
556 if (mjpegImage === image) return;
557 releaseImage?.();
558 releaseImage = null;
559 mjpegImage = image;
560 observeMjpegImage();
561 };
562
563 const fallBackToMjpeg = () => {
564 if (stopped || mjpeg) return;
565 mjpeg = true;
566 controller?.abort();
567 controller = null;
568 closeDecoder();
569 connecting();
570 events.onMjpegFallback(mjpegUrl());
571 if (!releaseImage) observeMjpegImage();
572 };
573
574 const paint = (source: CanvasImageSource, width: number, height: number) => {
575 if (stopped) return;
576 if (!avcc && (screen?.width !== width || screen.height !== height)) {
577 screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" };
578 events.onScreen(screen);
579 }
580 if (!sink.present(source, width, height)) {
581 fail("Could not display the device stream. Reconnect to try again.");
582 return;
583 }
584 frameReceived();
585 };
586
587 const closeDecoder = () => {
588 decoderEpoch++;
589 try {
590 videoDecoder?.close();
591 } catch {}
592 videoDecoder = null;
593 awaitingKeyframe = true;
594 };
595
596 const recoverDecoder = () => {
597 if (serveSimOnly) {
598 // A new AVCC request starts a fresh capture with a keyframe; there is no MJPEG.
599 closeDecoder();
600 controller?.abort();
601 } else if (platform === "ios") fallBackToMjpeg();
602 else {
603 closeDecoder();
604 connecting("Video decoder restarted.");
605 requestKeyframe();
606 }
607 };
608
609 const makeDecoder = () => {
610 const feedGeneration = videoGeneration;
611 const decoder = new VideoDecoder({
612 output: (frame) => {
613 try {
614 if (videoDecoder === decoder && (!avcc || feedGeneration === videoGeneration)) {
615 decodedVideoGeneration = feedGeneration;
616 paint(frame, frame.displayWidth, frame.displayHeight);
617 }
618 } finally {
619 frame.close();
620 }
621 },
622 error: () => {
623 if (stopped || videoDecoder !== decoder || (avcc && feedGeneration !== videoGeneration))
624 return;
625 recoverDecoder();
626 },
627 });
628 return decoder;
629 };
630
631 const configureDecoder = async (
632 config: VideoDecoderConfig,
633 isCurrent = () => !stopped,
634 ): Promise<boolean> => {
635 const epoch = decoderEpoch;
636 const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
637 const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false }));
638 if (!isCurrent() || epoch !== decoderEpoch) return false;
639 if (!support.supported) {
640 if (platform === "android") fail(`This browser cannot decode ${config.codec}.`);
641 return false;
642 }
643 try {
644 if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
645 videoDecoder.configure(full);
646 return true;
647 } catch (cause) {
648 if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
649 return false;
650 }
651 };
652
653 const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => {
654 if (!videoDecoder || videoDecoder.state !== "configured") return;
655 if (awaitingKeyframe) {
656 if (!isKey) return;
657 awaitingKeyframe = false;
658 }
659 if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
660 if (!avcc) {
661 recoverDecoder();
662 return;
663 }
664 // Native IDRs are spaced by frame count, so an idle simulator may take
665 // minutes to produce one. A new AVCC subscription forces a fresh IDR.
666 controller?.abort();
667 closeDecoder();
668 return;
669 }
670 try {
671 videoDecoder.decode(
672 new EncodedVideoChunk({
673 type: isKey ? "key" : "delta",
674 timestamp: pts ?? timestamp,
675 data,
676 }),
677 );
678 timestamp += FRAME_DURATION_US;
679 } catch {
680 recoverDecoder();
681 }
682 };
683
684 const requestKeyframe = () => {
685 if (platform === "android" && socket?.readyState === WebSocket.OPEN) {
686 socket.send(JSON.stringify({ type: "reset-video", ack: false }));
687 }
688 };
689
690 const scheduleRetry = (channel: "video" | "input", run: () => void) => {
691 if (stopped || retryTimers.has(channel)) return;
692 retryTimers.set(
693 channel,
694 setTimeout(() => {
695 retryTimers.delete(channel);
696 run();
697 }, RETRY_DELAY_MS),
698 );
699 };
700
701 const handleUnauthorized = () => {
702 stop();
703 events.onInputConnected(false);
704 events.onUnauthorized();
705 };
706
707 const readIosVideo = async () => {
708 const lifecycle = generation;
709 const feedGeneration = ++videoGeneration;
710 const demuxer = new AvccDemuxer();
711 const videoController = new AbortController();
712 controller = videoController;
713 const isCurrent = () =>
714 !stopped &&
715 generation === lifecycle &&
716 videoGeneration === feedGeneration &&
717 controller === videoController;
718 let retryDetail: string | undefined;
719 try {
720 const response = await fetch(httpUrl(videoPath), {
721 signal: videoController.signal,
722 credentials: access.credentials ? "include" : "same-origin",
723 });
724 if (!isCurrent()) {
725 await response.body?.cancel();
726 return;
727 }
728 if (response.status === 401 || response.status === 403) return handleUnauthorized();
729 if (response.status === 409 && !target.panelId) {
730 const locked = decodeTransportLocked(await response.json().catch(() => null));
731 if (Option.isSome(locked)) {
732 if (!isCurrent()) return;
733 controller = null;
734 void switchToWebRtc("This browser cannot play this device's WebRTC stream.", true);
735 return;
736 }
737 }
738 if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
739 await response.body?.cancel();
740 setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
741 return;
742 }
743 if (!response.ok || !response.body) throw new Error(`stream ${response.status}`);
744 const reader = response.body.getReader();
745 for (;;) {
746 // An AVCC body can stay open after its helper stops producing frames.
747 const timer = setTimeout(() => {
748 if (!isCurrent()) return;
749 retryDetail = "Device stream stopped receiving video. Reconnecting…";
750 videoController.abort();
751 }, FIRST_FRAME_TIMEOUT_MS);
752 videoReadTimer = timer;
753 let result: ReadableStreamReadResult<Uint8Array>;
754 try {
755 result = await reader.read();
756 } finally {
757 clearTimeout(timer);
758 if (videoReadTimer === timer) videoReadTimer = null;
759 }
760 const { done, value } = result;
761 if (!isCurrent()) return;
762 if (done) break;
763 for (const chunk of demuxer.push(value)) {
764 if (videoController.signal.aborted) break;
765 switch (chunk.type) {
766 case "seed":
767 void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
768 .then((bitmap) => {
769 try {
770 if (isCurrent() && decodedVideoGeneration !== feedGeneration)
771 paint(bitmap, bitmap.width, bitmap.height);
772 } finally {
773 bitmap.close();
774 }
775 })
776 .catch(() => {});
777 break;
778 case "description": {
779 awaitingKeyframe = true;
780 const configured = await configureDecoder(
781 {
782 codec: avcCodecString(chunk.payload),
783 description: chunk.payload,
784 },
785 isCurrent,
786 );
787 if (!isCurrent()) return;
788 if (!configured) {
789 await reader.cancel().catch(() => {});
790 if (!isCurrent()) return;
791 if (serveSimOnly)
792 setStatus(
793 "error",
794 `This browser cannot decode this device's ${avcCodecString(chunk.payload)} stream.`,
795 );
796 else if (target.videoOnly)
797 setStatus(
798 "error",
799 `This browser cannot decode the Duo panel's ${avcCodecString(chunk.payload)} stream.`,
800 );
801 else fallBackToMjpeg();
802 return;
803 }
804 break;
805 }
806 case "keyframe":
807 case "delta":
808 decode(chunk.type === "keyframe", chunk.payload);
809 break;
810 }
811 }
812 }
813 } catch (cause) {
814 if (!isCurrent()) return;
815 retryDetail ??= videoController.signal.aborted
816 ? "Video decoder fell behind. Reconnecting…"
817 : (cause as Error).message;
818 }
819 if (isCurrent()) {
820 controller = null;
821 videoController.abort();
822 closeDecoder();
823 connecting(retryDetail);
824 scheduleRetry("video", () => void readIosVideo());
825 }
826 };
827
828 /**
829 * serve-sim's helper only accepts HID and pushes its screen config once
830 * screen capture is running, and the AVCC stream does not reliably start
831 * it. Touching the MJPEG endpoint does; one aborted request is enough.
832 */
833 const primeIosHelper = async (session: number) => {
834 const controller = new AbortController();
835 primeController = controller;
836 const timeout = setTimeout(() => controller.abort(), 2_000);
837 try {
838 const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), {
839 signal: controller.signal,
840 credentials: access.credentials ? "include" : "same-origin",
841 });
842 if (stopped || generation !== session) return;
843 if (response.status === 401 || response.status === 403) return handleUnauthorized();
844 await response.body?.getReader().read();
845 } catch {
846 } finally {
847 clearTimeout(timeout);
848 controller.abort();
849 if (primeController === controller) primeController = null;
850 }
851 };
852
853 /**
854 * serve-sim can be locked to WebRTC (EAS Simulator always is), and then
855 * refuses AVCC and MJPEG; serve-emu advertises WebRTC the same way. An
856 * unreadable answer keeps the default path.
857 */
858 const probeWebRtc = async (): Promise<WebRtcSettings | null> => {
859 const probe = new AbortController();
860 const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS);
861 try {
862 const response = await fetch(httpUrl(`/api?device=${device}`), {
863 signal: probe.signal,
864 credentials: access.credentials ? "include" : "same-origin",
865 });
866 if (!response.ok) return null;
867 const api = decodeServeSimApi(await response.json());
868 if (Option.isNone(api)) return null;
869 const settings = decodeWebRtcSettings(api.value.streamSettings ?? api.value.stream);
870 return Option.isSome(settings) ? settings.value : null;
871 } catch {
872 return null;
873 } finally {
874 clearTimeout(timeout);
875 }
876 };
877
878 const webRtcUrl = (action: "offer" | "close") =>
879 httpUrl(avcc ? `/helper/${device}/webrtc/${action}` : `/webrtc/${action}?device=${device}`);
880
881 const closePeer = () => {
882 const current = peer;
883 if (!current) return;
884 peer = null;
885 current.connection.close();
886 current.video.srcObject = null;
887 current.video.remove();
888 // Releases the capture session; best effort because the page may be unloading.
889 void fetch(webRtcUrl("close"), {
890 method: "POST",
891 keepalive: true,
892 headers: { "content-type": "application/json" },
893 body: JSON.stringify({ sessionId: current.sessionId }),
894 credentials: access.credentials ? "include" : "same-origin",
895 }).catch(() => {});
896 };
897
898 const readWebRtc = async (settings: WebRtcSettings) => {
899 const lifecycle = generation;
900 const feedGeneration = ++videoGeneration;
901 closePeer();
902 const connection = new RTCPeerConnection({
903 ...(settings.iceServers
904 ? {
905 iceServers: settings.iceServers.map((server) => ({
906 ...server,
907 urls: typeof server.urls === "string" ? server.urls : [...server.urls],
908 })),
909 }
910 : {}),
911 iceTransportPolicy: settings.iceTransportPolicy ?? "all",
912 });
913 // Frames only reach requestVideoFrameCallback from a video that is in the
914 // document; it stays invisible because the sink paints them.
915 const video = document.createElement("video");
916 video.muted = true;
917 video.playsInline = true;
918 video.setAttribute("aria-hidden", "true");
919 video.style.cssText =
920 "position:fixed;width:1px;height:1px;opacity:0;pointer-events:none;left:0;top:0";
921 document.body.append(video);
922 const current = { connection, sessionId: webRtcSessionId(), video };
923 peer = current;
924 const isCurrent = () =>
925 !stopped &&
926 generation === lifecycle &&
927 videoGeneration === feedGeneration &&
928 peer === current;
929 // After the first frame nothing else bounds a feed that stops: the peer
930 // can sit `disconnected` with the last frame on the canvas. A long gap
931 // between frames reconnects through the same path as a failed connection.
932 let checksWithoutFrame = 0;
933 let watchdog: ReturnType<typeof setInterval> | null = null;
934 const clearWatchdog = () => {
935 if (watchdog !== null) clearInterval(watchdog);
936 watchdog = null;
937 };
938 const retry = (detail: string) => {
939 clearWatchdog();
940 if (!isCurrent()) return;
941 closePeer();
942 connecting(detail);
943 scheduleRetry("video", () => void readWebRtc(settings));
944 };
945 const presentFrame = () => {
946 if (!isCurrent()) {
947 clearWatchdog();
948 return;
949 }
950 if (video.videoWidth > 0 && video.videoHeight > 0) {
951 paint(video, video.videoWidth, video.videoHeight);
952 checksWithoutFrame = 0;
953 watchdog ??= setInterval(() => {
954 if (!isCurrent()) return clearWatchdog();
955 checksWithoutFrame++;
956 if (checksWithoutFrame * WEBRTC_STALL_CHECK_MS >= WEBRTC_STALL_TIMEOUT_MS) {
957 retry("Device stream stalled. Reconnecting…");
958 }
959 }, WEBRTC_STALL_CHECK_MS);
960 }
961 video.requestVideoFrameCallback(presentFrame);
962 };
963 const transceiver = connection.addTransceiver("video", { direction: "recvonly" });
964 const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
965 if (codecs.length > 0) {
966 // serve-sim encodes H.264; other codecs stay as fallbacks it may refuse.
967 transceiver.setCodecPreferences([
968 ...codecs.filter((codec) => codec.mimeType.toLowerCase() === "video/h264"),
969 ...codecs.filter((codec) => codec.mimeType.toLowerCase() !== "video/h264"),
970 ]);
971 }
972 connection.ontrack = (event) => {
973 if (!isCurrent()) return;
974 iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
975 video.srcObject = event.streams[0] ?? new MediaStream([event.track]);
976 void video.play().catch(() => {});
977 video.requestVideoFrameCallback(presentFrame);
978 };
979 connection.onconnectionstatechange = () => {
980 if (connection.connectionState === "failed")
981 retry("Device stream connection failed. Reconnecting…");
982 };
983 try {
984 await connection.setLocalDescription(await connection.createOffer());
985 const gathered = await new Promise<boolean>((resolve) => {
986 if (connection.iceGatheringState === "complete") return resolve(true);
987 const timer = setTimeout(() => done(false), iceGatheringDeadlineMs);
988 function done(complete: boolean) {
989 clearTimeout(timer);
990 connection.removeEventListener("icegatheringstatechange", changed);
991 resolve(complete);
992 }
993 function changed() {
994 if (connection.iceGatheringState === "complete") done(true);
995 }
996 connection.addEventListener("icegatheringstatechange", changed);
997 });
998 if (!isCurrent()) return;
999 // serve-sim takes all candidates in the offer; later candidates never trickle.
1000 // Cloud simulators need a reflexive or relay candidate, so retry incomplete offers with
1001 // longer deadlines up to the bound, then send what was gathered.
1002 const sdp = connection.localDescription?.sdp ?? "";
1003 if (
1004 !gathered &&
1005 !/^a=candidate:.* typ (?:srflx|relay)\b/m.test(sdp) &&
1006 iceGatheringDeadlineMs < ICE_GATHERING_MAX_TIMEOUT_MS
1007 ) {
1008 iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS);
1009 return retry("Device stream found no network candidates yet. Retrying…");
1010 }
1011 const response = await fetch(webRtcUrl("offer"), {
1012 method: "POST",
1013 headers: { "content-type": "application/json" },
1014 body: JSON.stringify({
1015 type: "offer",
1016 sdp: connection.localDescription?.sdp,
1017 sessionId: current.sessionId,
1018 codec: "h264",
1019 }),
1020 credentials: access.credentials ? "include" : "same-origin",
1021 });
1022 if (!isCurrent()) {
1023 await response.body?.cancel();
1024 return;
1025 }
1026 if (response.status === 401 || response.status === 403) return handleUnauthorized();
1027 // 409: another viewer is still negotiating with this simulator.
1028 if (!response.ok) return retry(`Device stream offer failed (${response.status}). Retrying…`);
1029 const answer = decodeWebRtcAnswer(await response.json());
1030 if (!isCurrent()) return;
1031 if (Option.isNone(answer))
1032 return retry("Device stream returned an invalid answer. Retrying…");
1033 await connection.setRemoteDescription(answer.value);
1034 } catch (cause) {
1035 retry((cause as Error).message);
1036 }
1037 };
1038
1039 /**
1040 * Called once AVCC or MJPEG was refused. `locked` means serve-sim already
1041 * said it streams WebRTC only, so a missing /api answer still switches.
1042 */
1043 const switchToWebRtc = async (failure: string, locked: boolean) => {
1044 const session = generation;
1045 if (typeof RTCPeerConnection === "undefined") return fail(failure);
1046 const settings = (await probeWebRtc()) ?? (locked ? { transport: "webrtc" as const } : null);
1047 if (stopped || generation !== session) return;
1048 if (!settings) return fail(failure);
1049 if (mjpeg) {
1050 mjpeg = false;
1051 releaseImage?.();
1052 releaseImage = null;
1053 events.onCanvasVideo?.();
1054 }
1055 // MJPEG may have shown frames already: say the stream is connecting again
1056 // rather than leave a blank canvas until the first WebRTC frame.
1057 connecting("Switching to WebRTC.");
1058 void readWebRtc(settings);
1059 };
1060
1061 const startDuoVideo = (panels: DuoPanelSinks) => {
1062 for (const panel of panelClients) panel.stop();
1063 // Physical handoff elects a native surface. Fixed-panel encoders can keep
1064 // an inactive shutdown frame after election, so this build uses one active
1065 // feed instead of decoding a third stream alongside the two fixed feeds.
1066 const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const);
1067 panelClients = ids.map((id) => {
1068 const output = id === 1 ? panels.cover : panels.inner;
1069 return createDeviceStreamClient(
1070 { ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true },
1071 {
1072 present(source, width, height) {
1073 if (id === null) {
1074 return sink.present(source, width, height);
1075 }
1076 // An inactive native LCD can emit its shutdown black frame. Retain its last useful image.
1077 if (screen?.screenId !== id) return true;
1078 const retained = output.present(source, width, height);
1079 const primary = sink.present(source, width, height);
1080 return retained && primary;
1081 },
1082 },
1083 {
1084 onStatus: (status, detail) => {
1085 if (status === "error") events.onDuoUnavailable?.(detail);
1086 },
1087 onScreen: () => {},
1088 onInputConnected: () => {},
1089 onMjpegFallback: () => {},
1090 onUnauthorized: handleUnauthorized,
1091 },
1092 );
1093 });
1094 for (const panel of panelClients) panel.start();
1095 };
1096
1097 const connectIosInput = async () => {
1098 if (stopped) return;
1099 const session = generation;
1100 if (!serveSimOnly) await primeIosHelper(session);
1101 if (stopped || generation !== session) return;
1102 const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
1103 ws.binaryType = "arraybuffer";
1104 socket = ws;
1105 iosInputAdmitted = false;
1106 iosInputUnavailable = false;
1107 ws.onmessage = (event) => {
1108 if (stopped || socket !== ws) return;
1109 if (!(event.data instanceof ArrayBuffer)) return;
1110 const bytes = new Uint8Array(event.data);
1111 if (socket !== ws || stopped || bytes.length < 1) return;
1112 if (bytes[0] === IOS_TAG_INPUT_ADMITTED) {
1113 iosInputAdmitted = true;
1114 events.onInputConnected(!iosInputUnavailable);
1115 return;
1116 }
1117 try {
1118 const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
1119 if (bytes[0] === 0x90) {
1120 const reply = decodeControlReply(payload);
1121 if (Option.isSome(reply)) duoControl.receive(reply.value);
1122 } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
1123 const config = decodeScreenConfig(payload);
1124 if (Option.isSome(config)) {
1125 iosInputUnavailable = config.value.inputUnavailable === true;
1126 events.onInputConnected(
1127 iosInputAdmitted && !iosInputUnavailable,
1128 iosInputUnavailable ? "Simulator input is unavailable" : undefined,
1129 );
1130 const previous = screen;
1131 screen = config.value;
1132 if (screen.hingePose && screen.hingePose !== previous?.hingePose)
1133 rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait";
1134 else if (screen.orientation !== previous?.orientation)
1135 rotationCursor = screen.orientation;
1136 panelSinks?.onScreen?.(screen);
1137 events.onScreen(screen);
1138 // A surface election can leave an existing decoder on the former
1139 // encoder description. Reopen only video to acquire the elected
1140 // surface's seed and codec configuration; HID and the viewer stay.
1141 if (
1142 panelSinks &&
1143 screen.supportsPhysicalOrientation &&
1144 previous &&
1145 screen.screenId !== previous.screenId
1146 )
1147 startDuoVideo(panelSinks);
1148 if (pendingOrientation) {
1149 const receipt = pendingOrientation;
1150 pendingOrientation = null;
1151 duoControl.receive({
1152 requestId: receipt.requestId,
1153 ok: true,
1154 });
1155 }
1156 }
1157 }
1158 } catch {}
1159 };
1160 ws.onclose = (event) => {
1161 if (socket !== ws) return;
1162 socket = null;
1163 iosInputAdmitted = false;
1164 duoControl.clear();
1165 rotationCursor = null;
1166 if (stopped) return;
1167 events.onInputConnected(
1168 false,
1169 event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`),
1170 );
1171 // A rejected HTTP upgrade surfaces as 1006, including an expired stream ticket.
1172 if (
1173 event.code === 1008 ||
1174 event.code === 4401 ||
1175 (event.code === 1006 && access.query.wsTicket)
1176 )
1177 return handleUnauthorized();
1178 scheduleRetry("input", () => void connectIosInput());
1179 };
1180 ws.onerror = () => ws.close();
1181 };
1182
1183 /** `inputOnly` keeps a socket for input beside a WebRTC feed, so its reconnects leave video alone. */
1184 const connectAndroid = (inputOnly = false) => {
1185 if (stopped) return;
1186 const ws = new WebSocket(
1187 wsUrl(`/ws?device=${device}&${inputOnly ? "video=0" : "frame-meta=1"}`),
1188 );
1189 ws.binaryType = "arraybuffer";
1190 socket = ws;
1191 ws.onopen = () => {
1192 if (stopped || socket !== ws) return;
1193 if (!inputOnly) connecting();
1194 events.onInputConnected(true);
1195 };
1196 ws.onmessage = (event) => {
1197 if (stopped || socket !== ws || inputOnly) return;
1198 if (typeof event.data === "string") {
1199 // The encoder restarts at a new size when the device rotates; the
1200 // next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
1201 if (isVideoSessionMessage(event.data)) {
1202 closeDecoder();
1203 configuring = false;
1204 connecting();
1205 requestKeyframe();
1206 }
1207 return;
1208 }
1209 if (!(event.data instanceof ArrayBuffer)) return;
1210 const packet = parseSemuPacket(event.data);
1211 const needsScan =
1212 packet.isKey === null ||
1213 (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured"));
1214 const scanned = needsScan ? scanAccessUnit(packet.data) : null;
1215 const isKey = packet.isKey ?? scanned?.isKey ?? false;
1216 if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) {
1217 if (configuring) return;
1218 configuring = true;
1219 const epoch = decoderEpoch;
1220 const isCurrent = () => !stopped && socket === ws;
1221 void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then(
1222 (configured) => {
1223 if (!isCurrent() || epoch !== decoderEpoch) return;
1224 configuring = false;
1225 awaitingKeyframe = true;
1226 if (configured) requestKeyframe();
1227 },
1228 );
1229 return;
1230 }
1231 if (!videoDecoder || videoDecoder.state !== "configured") {
1232 if (!isKey) requestKeyframe();
1233 return;
1234 }
1235 decode(isKey, packet.data, packet.timestamp);
1236 };
1237 ws.onclose = (event) => {
1238 if (socket !== ws) return;
1239 socket = null;
1240 closeDecoder();
1241 if (stopped) return;
1242 events.onInputConnected(false, event.reason || `closed ${event.code}`);
1243 if (
1244 event.code === 1008 ||
1245 event.code === 4401 ||
1246 (event.code === 1006 && access.query.wsTicket)
1247 )
1248 return handleUnauthorized();
1249 if (!inputOnly) {
1250 configuring = false;
1251 connecting(event.reason || undefined);
1252 }
1253 // A full socket re-probes, so one failed /api read does not pin SEMU on a WebRTC host.
1254 scheduleRetry("input", () => (inputOnly ? connectAndroid(true) : void startAndroid()));
1255 };
1256 ws.onerror = () => ws.close();
1257 };
1258
1259 const startAndroid = async () => {
1260 const session = generation;
1261 const settings = await probeWebRtc();
1262 if (stopped || generation !== session) return;
1263 if (settings) {
1264 if (typeof RTCPeerConnection === "undefined")
1265 return fail("This browser cannot receive the device's WebRTC stream.");
1266 if (!target.videoOnly) connectAndroid(true);
1267 void readWebRtc(settings);
1268 } else if (useWebCodecs) connectAndroid();
1269 else fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
1270 };
1271
1272 const start = () => {
1273 if (!stopped) return;
1274 stopped = false;
1275 generation++;
1276 configuring = false;
1277 iosInputAdmitted = false;
1278 iosInputUnavailable = false;
1279 connecting();
1280 if (serveSimOnly) {
1281 if (!target.videoOnly) void connectIosInput();
1282 if (useWebCodecs) void readIosVideo();
1283 else fail("This browser cannot decode this device's stream (WebCodecs unavailable).");
1284 } else if (platform === "ios") {
1285 if (!target.videoOnly) void connectIosInput();
1286 if (useWebCodecs) void readIosVideo();
1287 else fallBackToMjpeg();
1288 } else void startAndroid();
1289 };
1290
1291 const stop = () => {
1292 if (stopped) return;
1293 stopped = true;
1294 generation++;
1295 videoGeneration++;
1296 duoControl.clear();
1297 rotationCursor = null;
1298 for (const panel of panelClients) panel.stop();
1299 panelClients = [];
1300 panelSinks = null;
1301 mjpeg = false;
1302 clearFrameTimer();
1303 if (videoReadTimer !== null) clearTimeout(videoReadTimer);
1304 videoReadTimer = null;
1305 releaseImage?.();
1306 releaseImage = null;
1307 mjpegImage = null;
1308 for (const timer of retryTimers.values()) clearTimeout(timer);
1309 retryTimers.clear();
1310 primeController?.abort();
1311 primeController = null;
1312 controller?.abort();
1313 controller = null;
1314 closePeer();
1315 const discarded = socket;
1316 socket = null;
1317 discarded?.close();
1318 closeDecoder();
1319 };
1320
1321 const send = (payload: Uint8Array<ArrayBuffer> | string) => {
1322 if (
1323 !stopped &&
1324 (!avcc || (iosInputAdmitted && !iosInputUnavailable)) &&
1325 socket?.readyState === WebSocket.OPEN
1326 )
1327 socket.send(payload);
1328 };
1329
1330 const rawPoint = (x: number, y: number) => {
1331 // serve-sim streams the raw framebuffer; rotated devices need input
1332 // remapped into that raw space.
1333 if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y };
1334 switch (screen.orientation) {
1335 case "landscape_left":
1336 return { x: y, y: 1 - x };
1337 case "landscape_right":
1338 return { x: 1 - y, y: x };
1339 case "portrait_upside_down":
1340 return { x: 1 - x, y: 1 - y };
1341 default:
1342 return { x, y };
1343 }
1344 };
1345
1346 return {
1347 start,
1348 stop,
1349 setMjpegImage,
1350 controlDuo: duoControl.enqueue,
1351 sendRawTouch: (phase, x, y) => {
1352 if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y }));
1353 },
1354 setDuoPanels(panels) {
1355 if (platform !== "ios" || target.videoOnly || stopped || panelSinks === panels) return;
1356 if (panels && !screen?.supportsHingeAngle) return;
1357 panelSinks = panels;
1358 videoGeneration++;
1359 controller?.abort();
1360 controller = null;
1361 closePeer();
1362 closeDecoder();
1363 const retry = retryTimers.get("video");
1364 if (retry) clearTimeout(retry);
1365 retryTimers.delete("video");
1366 for (const panel of panelClients) panel.stop();
1367 panelClients = [];
1368 if (!panels) {
1369 if (useWebCodecs) void readIosVideo();
1370 return;
1371 }
1372 startDuoVideo(panels);
1373 },
1374 sendTouch: (phase, x, y) => {
1375 if (avcc) {
1376 send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));
1377 return;
1378 }
1379 const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up";
1380 send(JSON.stringify({ type: "touch", action, x, y }));
1381 },
1382 sendKey: (event, phase) => {
1383 if (avcc) {
1384 const usage = hidUsageForCode(event.code);
1385 if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage }));
1386 return;
1387 }
1388 if (phase !== "down") return;
1389 if (event.key === "Escape") return send(JSON.stringify({ type: "back" }));
1390 const keycode = ANDROID_KEYCODE_BY_KEY[event.key];
1391 if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode }));
1392 if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) {
1393 send(JSON.stringify({ type: "text", text: event.key }));
1394 }
1395 },
1396 sendText: (text) => {
1397 if (avcc) return;
1398 let chunk = "";
1399 let chunkBytes = 0;
1400 let sentChunks = 0;
1401 for (const character of text) {
1402 const characterBytes = utf8ByteLength(character.codePointAt(0)!);
1403 if (chunkBytes + characterBytes > ANDROID_MAX_TEXT_BYTES) {
1404 send(JSON.stringify({ type: "text", text: chunk }));
1405 if (++sentChunks >= ANDROID_MAX_TEXT_CHUNKS) return;
1406 chunk = "";
1407 chunkBytes = 0;
1408 }
1409 chunk += character;
1410 chunkBytes += characterBytes;
1411 }
1412 if (chunk.length > 0) send(JSON.stringify({ type: "text", text: chunk }));
1413 },
1414 pressButton: (button) => {
1415 if (avcc) {
1416 const name =
1417 button === "home"
1418 ? "home"
1419 : button === "appSwitcher" || button === "recents"
1420 ? "app_switcher"
1421 : button === "power"
1422 ? "lock"
1423 : button === "back" && platform === "android"
1424 ? "back"
1425 : null;
1426 if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name }));
1427 return;
1428 }
1429 const type = button === "appSwitcher" ? "recents" : button;
1430 if (type === "home" || type === "back" || type === "recents" || type === "power") {
1431 send(JSON.stringify({ type }));
1432 }
1433 },
1434 rotate: () => {
1435 if (platform !== "ios") return;
1436 const current = screen?.supportsHingeAngle
1437 ? (rotationCursor ?? screen.orientation)
1438 : (screen?.orientation ?? "portrait");
1439 const next =
1440 IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
1441 if (screen?.supportsHingeAngle) {
1442 rotationCursor = next;
1443 duoControl.enqueue({ control: "orientation", value: next });
1444 } else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
1445 },
1446 setOrientation: (orientation) => {
1447 if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));
1448 },
1449 };
1450}