1// @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime.
2
3import {
4 createDuoControl,
5 type DuoCommand,
6 type DuoControlState,
7 type DuoPose,
8} from "./duoControl.ts";
9import * as Schema from "effect/Schema";
10import * as Option from "effect/Option";
11import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
12import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
13import type { DevicePlatform } from "@t3tools/contracts";
14
15export type DeviceStreamStatus = "connecting" | "streaming" | "error";
16
17export interface DeviceScreenSize {
18 readonly width: number;
19 readonly height: number;
20 readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right";
21 readonly screenId?: number;
22 readonly supportsHingeAngle?: boolean;
23 readonly supportsPhysicalOrientation?: boolean;
24 readonly hingeAngle?: number;
25 readonly hingePose?: DuoPose | null;
26 readonly tableMode?: boolean;
27 readonly tableModeAvailable?: boolean;
28}
29
30const screenConfigSchema = Schema.Struct({
31 width: Schema.Finite.check(Schema.isGreaterThan(0)),
32 height: Schema.Finite.check(Schema.isGreaterThan(0)),
33 orientation: Schema.Literals([
34 "portrait",
35 "portrait_upside_down",
36 "landscape_left",
37 "landscape_right",
38 ]),
39 inputUnavailable: Schema.optionalKey(Schema.Boolean),
40 screenId: Schema.optionalKey(Schema.Number),
41 supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
42 supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
43 hingeAngle: Schema.optionalKey(
44 Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })),
45 ),
46 hingePose: Schema.optionalKey(
47 Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])),
48 ),
49 tableMode: Schema.optionalKey(Schema.Boolean),
50 tableModeAvailable: Schema.optionalKey(Schema.Boolean),
51});
52const controlReplySchema = Schema.Struct({
53 requestId: Schema.Int,
54 ok: Schema.Boolean,
55 error: Schema.optionalKey(Schema.String),
56});
57const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
58const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
59const iceServerSchema = Schema.Struct({
60 urls: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
61 username: Schema.optionalKey(Schema.String),
62 credential: Schema.optionalKey(Schema.String),
63});
64const webRtcSettingsSchema = Schema.Struct({
65 transport: Schema.Literal("webrtc"),
66 iceServers: Schema.optionalKey(Schema.Array(iceServerSchema)),
67 iceTransportPolicy: Schema.optionalKey(Schema.Literals(["all", "relay"])),
68});
69// serve-sim reports its stream settings as `streamSettings` (EAS) or `stream`.
70const serveSimApiSchema = Schema.Struct({
71 streamSettings: Schema.optionalKey(Schema.Unknown),
72 stream: Schema.optionalKey(Schema.Unknown),
73});
74const decodeServeSimApi = Schema.decodeUnknownOption(serveSimApiSchema);
75const decodeWebRtcSettings = Schema.decodeUnknownOption(webRtcSettingsSchema);
76type WebRtcSettings = typeof webRtcSettingsSchema.Type;
77const decodeLeaseEnd = Schema.decodeUnknownOption(
78 Schema.fromJsonString(Schema.Struct({ reason: Schema.String })),
79);
80const leaseEnded = (reason: string) => `Device lease ended: ${reason}`;
81const decodeTransportLocked = Schema.decodeUnknownOption(
82 Schema.Struct({
83 error: Schema.Literal("stream_transport_locked"),
84 transport: Schema.Literal("webrtc"),
85 }),
86);
87const decodeWebRtcAnswer = Schema.decodeUnknownOption(
88 Schema.Struct({ type: Schema.Literal("answer"), sdp: Schema.String }),
89);
90
91export interface DuoPanelSinks {
92 readonly cover: DeviceFrameSink;
93 readonly inner: DeviceFrameSink;
94 /** Invalidate captured input synchronously, before React can commit the new layout. */
95 readonly onScreen?: (screen: DeviceScreenSize) => void;
96}
97
98export interface DeviceStreamEvents {
99 readonly onDuoControl?: (state: DuoControlState) => void;
100 readonly onDuoUnavailable?: (detail?: string) => void;
101 readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;
102 readonly onScreen: (screen: DeviceScreenSize) => void;
103 readonly onUnauthorized: () => void;
104 /**
105 * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is
106 * unsupported); the owner should show an `<img>` instead of the canvas
107 * and attach it with `setMjpegImage` so the client can observe real frames.
108 */
109 readonly onMjpegFallback: (url: string) => void;
110 /**
111 * The stream moved from MJPEG back to painting the canvas, because the
112 * device serves WebRTC only; the owner should show the canvas again.
113 */
114 readonly onCanvasVideo?: () => void;
115 readonly onInputConnected: (connected: boolean, detail?: string) => void;
116}
117
118export interface DeviceStreamTarget {
119 readonly platform: DevicePlatform;
120 readonly deviceId: string;
121 readonly access: DeviceHubAccess;
122 /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */
123 readonly preferMjpeg?: boolean;
124 readonly panelId?: 1 | 3;
125 readonly videoOnly?: boolean;
126 /** The host speaks only serve-sim's AVCC stream and input socket, on both platforms (Simlock). */
127 readonly serveSimOnly?: boolean;
128}
129
130export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher";
131
132const RETRY_DELAY_MS = 1_000;
133const FIRST_FRAME_TIMEOUT_MS = 15_000;
134// A WebRTC feed that shows no frame for this long is reconnected; checked
135// every WEBRTC_STALL_CHECK_MS, so a transient `disconnected` peer that
136// recovers is left alone.
137const WEBRTC_STALL_TIMEOUT_MS = FIRST_FRAME_TIMEOUT_MS;
138const WEBRTC_STALL_CHECK_MS = 5_000;
139const TRANSPORT_PROBE_TIMEOUT_MS = 5_000;
140// Offers carry every candidate: serve-sim does not accept trickled ICE.
141const ICE_GATHERING_TIMEOUT_MS = 2_000;
142// A gathering cut short without a reflexive or relay candidate is retried
143// with twice the deadline, up to this; then the offer goes as gathered. The
144// whole sequence stays inside the first-frame deadline, which retries do
145// not extend.
146const ICE_GATHERING_MAX_TIMEOUT_MS = 4_000;
147const MJPEG_FRAME_CHECK_MS = 250;
148const FRAME_DURATION_US = 16_667;
149const SEMU_MAGIC = 0x53454d55;
150const SEMU_HEADER_BYTES = 16;
151const SEMU_FLAG_KEY = 1;
152const SOFT_DECODE_QUEUE = 8;
153// expo-device-hub 0.15.3 serve-emu input.js MAX_TEXT_BYTES limits packets to 300 UTF-8 bytes.
154const ANDROID_MAX_TEXT_BYTES = 300;
155// The pinned hub queue depth is 128; the client does not read queue rejections.
156const ANDROID_MAX_TEXT_CHUNKS = 64;
157
158const IOS_MSG_TOUCH = 0x03;
159const IOS_MSG_BUTTON = 0x04;
160const IOS_MSG_KEY = 0x06;
161const IOS_MSG_ORIENTATION = 0x07;
162const IOS_TAG_SCREEN_CONFIG = 0x82;
163const IOS_TAG_INPUT_ADMITTED = 0x83;
164
165const encoder = new TextEncoder();
166const decoder = new TextDecoder();
167
168/** serve-sim requires a v4 UUID; `crypto.randomUUID` is missing over plain-HTTP LAN origins. */
169function webRtcSessionId(): string {
170 const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
171 bytes[6] = (bytes[6]! & 0x0f) | 0x40;
172 bytes[8] = (bytes[8]! & 0x3f) | 0x80;
173 const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
174 return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
175}
176
177const isWebCodecsSupported = (): boolean =>
178 typeof globalThis !== "undefined" &&
179 "VideoDecoder" in globalThis &&
180 "EncodedVideoChunk" in globalThis;
181
182function utf8ByteLength(codePoint: number): number {
183 if (codePoint <= 0x7f) return 1;
184 if (codePoint <= 0x7ff) return 2;
185 if (codePoint <= 0xffff) return 3;
186 return 4;
187}
188
189function taggedJson(tag: number, payload: unknown): Uint8Array<ArrayBuffer> {
190 const json = encoder.encode(JSON.stringify(payload));
191 const out = new Uint8Array(1 + json.length);
192 out[0] = tag;
193 out.set(json, 1);
194 return out;
195}
196
197export function avcCodecString(bytes: Uint8Array): string {
198 if (bytes.length < 4) return "avc1.42E01E";
199 const hex = (byte: number) => byte.toString(16).padStart(2, "0");
200 return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`;
201}
202
203export function parseSemuPacket(raw: ArrayBuffer): {
204 readonly data: Uint8Array;
205 readonly isKey: boolean | null;
206 readonly timestamp: number | null;
207} {
208 const bytes = new Uint8Array(raw);
209 if (bytes.byteLength > SEMU_HEADER_BYTES) {
210 const view = new DataView(raw, 0, SEMU_HEADER_BYTES);
211 if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) {
212 const pts = view.getBigUint64(8, false);
213 return {
214 data: bytes.subarray(SEMU_HEADER_BYTES),
215 isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0,
216 timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null,
217 };
218 }
219 }
220 return { data: bytes, isKey: null, timestamp: null };
221}
222
223const isVideoSessionMessage = (text: string) => {
224 try {
225 const message = JSON.parse(text) as { type?: unknown };
226 return message.type === "video-session";
227 } catch {
228 return false;
229 }
230};
231
232export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } {
233 let isKey = false;
234 let sps: Uint8Array | null = null;
235 const len = buf.length;
236 let i = 0;
237 while (i + 2 < len) {
238 if (buf[i] === 0 && buf[i + 1] === 0) {
239 let codeLen = 0;
240 if (buf[i + 2] === 1) codeLen = 3;
241 else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4;
242 if (codeLen) {
243 const nalType = buf[i + codeLen]! & 0x1f;
244 if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen);
245 if (nalType === 5) isKey = true;
246 i += codeLen + 1;
247 continue;
248 }
249 }
250 i++;
251 }
252 return { isKey, sps };
253}
254
255export type AvccChunk = {
256 readonly type: "description" | "keyframe" | "delta" | "seed" | "end";
257 readonly payload: Uint8Array;
258};
259
260const AVCC_TAGS: Record<number, AvccChunk["type"] | undefined> = {
261 1: "description",
262 2: "keyframe",
263 3: "delta",
264 4: "seed",
265 // Simlock's final frame on a lease that ended, carrying `{ reason }`.
266 7: "end",
267};
268
269export class AvccDemuxer {
270 private buffer = new Uint8Array(64 * 1024);
271 private length = 0;
272
273 push(bytes: Uint8Array): AvccChunk[] {
274 if (this.length + bytes.length > this.buffer.length) {
275 let capacity = this.buffer.length;
276 while (capacity < this.length + bytes.length) capacity *= 2;
277 const grown = new Uint8Array(capacity);
278 grown.set(this.buffer.subarray(0, this.length));
279 this.buffer = grown;
280 }
281 this.buffer.set(bytes, this.length);
282 this.length += bytes.length;
283
284 const chunks: AvccChunk[] = [];
285 let offset = 0;
286 while (this.length - offset >= 4) {
287 const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4);
288 const frameLength = view.getUint32(0, false);
289 if (this.length - offset - 4 < frameLength) break;
290 if (frameLength >= 1) {
291 const type = AVCC_TAGS[this.buffer[offset + 4]!];
292 if (type) {
293 chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) });
294 }
295 }
296 offset += 4 + frameLength;
297 }
298 if (offset > 0) {
299 this.buffer.copyWithin(0, offset, this.length);
300 this.length -= offset;
301 }
302 return chunks;
303 }
304
305 reset(): void {
306 this.length = 0;
307 }
308}
309
310export interface DeviceStreamClient {
311 readonly start: () => void;
312 readonly stop: () => void;
313 /**
314 * `stop()` detaches it; attach a fresh image for each restart.
315 */
316 readonly setMjpegImage: (image: HTMLImageElement | null) => void;
317 /** Normalized 0..1 coordinates in the displayed frame. */
318 readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
319 readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void;
320 /** Android only: iOS input is HID key codes, which cannot carry arbitrary text. */
321 readonly sendText: (text: string) => void;
322 readonly pressButton: (button: DeviceHardwareButton) => void;
323 readonly rotate: () => void;
324 readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
325 readonly controlDuo: (command: DuoCommand) => void;
326 readonly setDuoPanels: (panels: DuoPanelSinks | null) => void;
327 readonly sendRawTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
328}
329
330const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
331 Enter: 0x28,
332 Escape: 0x29,
333 Backspace: 0x2a,
334 Tab: 0x2b,
335 Space: 0x2c,
336 Minus: 0x2d,
337 Equal: 0x2e,
338 BracketLeft: 0x2f,
339 BracketRight: 0x30,
340 Backslash: 0x31,
341 Semicolon: 0x33,
342 Quote: 0x34,
343 Backquote: 0x35,
344 Comma: 0x36,
345 Period: 0x37,
346 Slash: 0x38,
347 Delete: 0x4c,
348 ArrowRight: 0x4f,
349 ArrowLeft: 0x50,
350 ArrowDown: 0x51,
351 ArrowUp: 0x52,
352 NumpadDivide: 0x54,
353 NumpadMultiply: 0x55,
354 NumpadSubtract: 0x56,
355 NumpadAdd: 0x57,
356 NumpadEnter: 0x58,
357 Numpad1: 0x59,
358 Numpad2: 0x5a,
359 Numpad3: 0x5b,
360 Numpad4: 0x5c,
361 Numpad5: 0x5d,
362 Numpad6: 0x5e,
363 Numpad7: 0x5f,
364 Numpad8: 0x60,
365 Numpad9: 0x61,
366 Numpad0: 0x62,
367 NumpadDecimal: 0x63,
368 NumpadEqual: 0x67,
369 ControlLeft: 0xe0,
370 ShiftLeft: 0xe1,
371 AltLeft: 0xe2,
372 MetaLeft: 0xe3,
373 ControlRight: 0xe4,
374 ShiftRight: 0xe5,
375 AltRight: 0xe6,
376 MetaRight: 0xe7,
377};
378
379function hidUsageForCode(code: string): number | null {
380 if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65);
381 if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49);
382 if (code === "Digit0") return 0x27;
383 return HID_USAGE_BY_CODE[code] ?? null;
384}
385
386const ANDROID_KEYCODE_BY_KEY: Readonly<Record<string, number>> = {
387 ArrowUp: 19,
388 ArrowDown: 20,
389 ArrowLeft: 21,
390 ArrowRight: 22,
391 Tab: 61,
392 Enter: 66,
393 Backspace: 67,
394 Delete: 112,
395 Home: 122,
396 End: 123,
397 PageUp: 92,
398 PageDown: 93,
399};
400
401const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [
402 "portrait",
403 "landscape_left",
404 "portrait_upside_down",
405 "landscape_right",
406];
407
408export function createDeviceStreamClient(
409 target: DeviceStreamTarget,
410 output: HTMLCanvasElement | DeviceFrameSink,
411 events: DeviceStreamEvents,
412): DeviceStreamClient {
413 const { access, platform, deviceId } = target;
414 const sink = "present" in output ? output : createCanvasFrameSink(output);
415 // Android normally streams over serve-emu's socket; a serve-sim-only host serves it like iOS.
416 const serveSimOnly = target.serveSimOnly === true;
417 const avcc = platform === "ios" || serveSimOnly;
418 const vendor = avcc ? "/vendor/serve-sim" : "/vendor/serve-emu";
419 const device = encodeURIComponent(deviceId);
420 const httpUrl = (path: string) =>
421 withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access);
422 const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access);
423 const useWebCodecs =
424 isWebCodecsSupported() && !(platform === "ios" && target.preferMjpeg && !serveSimOnly);
425
426 let stopped = true;
427 let socket: WebSocket | null = null;
428 let controller: AbortController | null = null;
429 const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();
430 let primeController: AbortController | null = null;
431 let videoDecoder: VideoDecoder | null = null;
432 let decodedVideoGeneration: number | null = null;
433 let iosInputAdmitted = false;
434 let iosInputUnavailable = false;
435 let timestamp = 0;
436 let awaitingKeyframe = true;
437 let screen: DeviceScreenSize | null = null;
438 let firstFrame = false;
439 let configuring = false;
440 let mjpeg = false;
441 let generation = 0;
442 let decoderEpoch = 0;
443 let frameTimer: ReturnType<typeof setTimeout> | null = null;
444 let videoReadTimer: ReturnType<typeof setTimeout> | null = null;
445 let mjpegImage: HTMLImageElement | null = null;
446 let releaseImage: (() => void) | null = null;
447 let videoGeneration = 0;
448 let panelClients: DeviceStreamClient[] = [];
449 let panelSinks: DuoPanelSinks | null = null;
450 let rotationCursor: DeviceScreenSize["orientation"] | null = null;
451 let pendingOrientation: { requestId: number } | null = null;
452 let iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
453 let peer: { connection: RTCPeerConnection; sessionId: string; video: HTMLVideoElement } | null =
454 null;
455 const duoControl = createDuoControl({
456 send(request) {
457 if (
458 !iosInputAdmitted ||
459 iosInputUnavailable ||
460 socket?.readyState !== WebSocket.OPEN ||
461 !screen?.supportsHingeAngle
462 )
463 return false;
464 if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
465 return false;
466 try {
467 pendingOrientation = null;
468 if (request.command.control === "orientation") {
469 const value = request.command.value;
470 pendingOrientation = { requestId: request.requestId };
471 rotationCursor = value;
472 // Upstream serializes orientation with hinge commands, then broadcasts config.
473 // An orientation-locked app can keep its framebuffer orientation after the sensor rotates.
474 socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value }));
475 } else socket.send(taggedJson(0x10, request));
476 return true;
477 } catch {
478 return false;
479 }
480 },
481 onChange(state) {
482 if (!state.pending) pendingOrientation = null;
483 events.onDuoControl?.(state);
484 },
485 });
486 const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`;
487
488 const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
489
490 const setStatus = (status: DeviceStreamStatus, detail?: string) => {
491 if (!stopped) events.onStatus(status, detail);
492 };
493
494 const clearFrameTimer = () => {
495 if (frameTimer !== null) clearTimeout(frameTimer);
496 frameTimer = null;
497 };
498
499 const fail = (detail: string) => {
500 if (stopped) return;
501 stop();
502 events.onInputConnected(false, detail);
503 events.onStatus("error", detail);
504 };
505
506 const connecting = (detail?: string) => {
507 firstFrame = false;
508 if (frameTimer === null) {
509 frameTimer = setTimeout(
510 () => fail("No video received from the device. Reconnect to try again."),
511 FIRST_FRAME_TIMEOUT_MS,
512 );
513 }
514 setStatus("connecting", detail);
515 };
516
517 const frameReceived = () => {
518 if (firstFrame) return;
519 firstFrame = true;
520 clearFrameTimer();
521 setStatus("streaming");
522 };
523
524 const observeMjpegImage = () => {
525 releaseImage?.();
526 releaseImage = null;
527 const image = mjpegImage;
528 if (!image || stopped || !mjpeg) return;
529 let timer: ReturnType<typeof setTimeout> | null = null;
530 let released = false;
531 const check = () => {
532 if (released || stopped) return;
533 if (timer !== null) clearTimeout(timer);
534 timer = null;
535 if (image.naturalWidth > 0 && image.naturalHeight > 0) frameReceived();
536 else {
537 // Multipart images may not emit load until the response ends.
538 timer = setTimeout(check, MJPEG_FRAME_CHECK_MS);
539 }
540 };
541 const error = () => {
542 if (released) return;
543 const failure = "Could not receive the device stream. Reconnect to try again.";
544 // An <img> cannot read the 409 that a WebRTC-only serve-sim answers with.
545 if (typeof RTCPeerConnection === "undefined") fail(failure);
546 else void switchToWebRtc(failure, false);
547 };
548 image.addEventListener("load", check);
549 image.addEventListener("error", error);
550 releaseImage = () => {
551 released = true;
552 if (timer !== null) clearTimeout(timer);
553 image.removeEventListener("load", check);
554 image.removeEventListener("error", error);
555 image.removeAttribute("src");
556 };
557 image.src = mjpegUrl();
558 check();
559 };
560
561 const setMjpegImage = (image: HTMLImageElement | null) => {
562 if (mjpegImage === image) return;
563 releaseImage?.();
564 releaseImage = null;
565 mjpegImage = image;
566 observeMjpegImage();
567 };
568
569 const fallBackToMjpeg = () => {
570 if (stopped || mjpeg) return;
571 mjpeg = true;
572 controller?.abort();
573 controller = null;
574 closeDecoder();
575 connecting();
576 events.onMjpegFallback(mjpegUrl());
577 if (!releaseImage) observeMjpegImage();
578 };
579
580 const paint = (source: CanvasImageSource, width: number, height: number) => {
581 if (stopped) return;
582 if (!avcc && (screen?.width !== width || screen.height !== height)) {
583 screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" };
584 events.onScreen(screen);
585 }
586 if (!sink.present(source, width, height)) {
587 fail("Could not display the device stream. Reconnect to try again.");
588 return;
589 }
590 frameReceived();
591 };
592
593 const closeDecoder = () => {
594 decoderEpoch++;
595 try {
596 videoDecoder?.close();
597 } catch {}
598 videoDecoder = null;
599 awaitingKeyframe = true;
600 };
601
602 const recoverDecoder = () => {
603 if (serveSimOnly) {
604 // A new AVCC request starts a fresh capture with a keyframe; there is no MJPEG.
605 closeDecoder();
606 controller?.abort();
607 } else if (platform === "ios") fallBackToMjpeg();
608 else {
609 closeDecoder();
610 connecting("Video decoder restarted.");
611 requestKeyframe();
612 }
613 };
614
615 const makeDecoder = () => {
616 const feedGeneration = videoGeneration;
617 const decoder = new VideoDecoder({
618 output: (frame) => {
619 try {
620 if (videoDecoder === decoder && (!avcc || feedGeneration === videoGeneration)) {
621 decodedVideoGeneration = feedGeneration;
622 paint(frame, frame.displayWidth, frame.displayHeight);
623 }
624 } finally {
625 frame.close();
626 }
627 },
628 error: () => {
629 if (stopped || videoDecoder !== decoder || (avcc && feedGeneration !== videoGeneration))
630 return;
631 recoverDecoder();
632 },
633 });
634 return decoder;
635 };
636
637 const configureDecoder = async (
638 config: VideoDecoderConfig,
639 isCurrent = () => !stopped,
640 ): Promise<boolean> => {
641 const epoch = decoderEpoch;
642 const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
643 const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false }));
644 if (!isCurrent() || epoch !== decoderEpoch) return false;
645 if (!support.supported) {
646 if (platform === "android") fail(`This browser cannot decode ${config.codec}.`);
647 return false;
648 }
649 try {
650 if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
651 videoDecoder.configure(full);
652 return true;
653 } catch (cause) {
654 if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
655 return false;
656 }
657 };
658
659 const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => {
660 if (!videoDecoder || videoDecoder.state !== "configured") return;
661 if (awaitingKeyframe) {
662 if (!isKey) return;
663 awaitingKeyframe = false;
664 }
665 if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
666 if (!avcc) {
667 recoverDecoder();
668 return;
669 }
670 // Native IDRs are spaced by frame count, so an idle simulator may take
671 // minutes to produce one. A new AVCC subscription forces a fresh IDR.
672 controller?.abort();
673 closeDecoder();
674 return;
675 }
676 try {
677 videoDecoder.decode(
678 new EncodedVideoChunk({
679 type: isKey ? "key" : "delta",
680 timestamp: pts ?? timestamp,
681 data,
682 }),
683 );
684 timestamp += FRAME_DURATION_US;
685 } catch {
686 recoverDecoder();
687 }
688 };
689
690 const requestKeyframe = () => {
691 if (platform === "android" && socket?.readyState === WebSocket.OPEN) {
692 socket.send(JSON.stringify({ type: "reset-video", ack: false }));
693 }
694 };
695
696 const scheduleRetry = (channel: "video" | "input", run: () => void) => {
697 if (stopped || retryTimers.has(channel)) return;
698 retryTimers.set(
699 channel,
700 setTimeout(() => {
701 retryTimers.delete(channel);
702 run();
703 }, RETRY_DELAY_MS),
704 );
705 };
706
707 const handleUnauthorized = () => {
708 stop();
709 events.onInputConnected(false);
710 events.onUnauthorized();
711 };
712
713 const readIosVideo = async () => {
714 const lifecycle = generation;
715 const feedGeneration = ++videoGeneration;
716 const demuxer = new AvccDemuxer();
717 const videoController = new AbortController();
718 controller = videoController;
719 const isCurrent = () =>
720 !stopped &&
721 generation === lifecycle &&
722 videoGeneration === feedGeneration &&
723 controller === videoController;
724 let retryDetail: string | undefined;
725 let leaseEnd: string | undefined;
726 try {
727 const response = await fetch(httpUrl(videoPath), {
728 signal: videoController.signal,
729 credentials: access.credentials ? "include" : "same-origin",
730 });
731 if (!isCurrent()) {
732 await response.body?.cancel();
733 return;
734 }
735 if (response.status === 401 || response.status === 403) return handleUnauthorized();
736 if (response.status === 409 && !target.panelId) {
737 const locked = decodeTransportLocked(await response.json().catch(() => null));
738 if (Option.isSome(locked)) {
739 if (!isCurrent()) return;
740 controller = null;
741 void switchToWebRtc("This browser cannot play this device's WebRTC stream.", true);
742 return;
743 }
744 }
745 if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
746 await response.body?.cancel();
747 setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
748 return;
749 }
750 if (!response.ok || !response.body) throw new Error(`stream ${response.status}`);
751 const reader = response.body.getReader();
752 for (;;) {
753 // An AVCC body can stay open after its helper stops producing frames.
754 const timer = setTimeout(() => {
755 if (!isCurrent()) return;
756 retryDetail = "Device stream stopped receiving video. Reconnecting…";
757 videoController.abort();
758 }, FIRST_FRAME_TIMEOUT_MS);
759 videoReadTimer = timer;
760 let result: ReadableStreamReadResult<Uint8Array>;
761 try {
762 result = await reader.read();
763 } finally {
764 clearTimeout(timer);
765 if (videoReadTimer === timer) videoReadTimer = null;
766 }
767 const { done, value } = result;
768 if (!isCurrent()) return;
769 if (done) break;
770 for (const chunk of demuxer.push(value)) {
771 if (videoController.signal.aborted) break;
772 switch (chunk.type) {
773 case "seed":
774 void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
775 .then((bitmap) => {
776 try {
777 if (isCurrent() && decodedVideoGeneration !== feedGeneration)
778 paint(bitmap, bitmap.width, bitmap.height);
779 } finally {
780 bitmap.close();
781 }
782 })
783 .catch(() => {});
784 break;
785 case "description": {
786 awaitingKeyframe = true;
787 const configured = await configureDecoder(
788 {
789 codec: avcCodecString(chunk.payload),
790 description: chunk.payload,
791 },
792 isCurrent,
793 );
794 if (!isCurrent()) return;
795 if (!configured) {
796 await reader.cancel().catch(() => {});
797 if (!isCurrent()) return;
798 if (serveSimOnly)
799 setStatus(
800 "error",
801 `This browser cannot decode this device's ${avcCodecString(chunk.payload)} stream.`,
802 );
803 else if (target.videoOnly)
804 setStatus(
805 "error",
806 `This browser cannot decode the Duo panel's ${avcCodecString(chunk.payload)} stream.`,
807 );
808 else fallBackToMjpeg();
809 return;
810 }
811 break;
812 }
813 case "keyframe":
814 case "delta":
815 decode(chunk.type === "keyframe", chunk.payload);
816 break;
817 case "end": {
818 const end = decodeLeaseEnd(decoder.decode(chunk.payload));
819 leaseEnd = Option.isSome(end) ? end.value.reason : "ended";
820 break;
821 }
822 }
823 }
824 }
825 } catch (cause) {
826 if (!isCurrent()) return;
827 retryDetail ??= videoController.signal.aborted
828 ? "Video decoder fell behind. Reconnecting…"
829 : (cause as Error).message;
830 }
831 if (isCurrent()) {
832 if (leaseEnd !== undefined) return fail(leaseEnded(leaseEnd));
833 controller = null;
834 videoController.abort();
835 closeDecoder();
836 connecting(retryDetail);
837 scheduleRetry("video", () => void readIosVideo());
838 }
839 };
840
841 /**
842 * serve-sim's helper only accepts HID and pushes its screen config once
843 * screen capture is running, and the AVCC stream does not reliably start
844 * it. Touching the MJPEG endpoint does; one aborted request is enough.
845 */
846 const primeIosHelper = async (session: number) => {
847 const controller = new AbortController();
848 primeController = controller;
849 const timeout = setTimeout(() => controller.abort(), 2_000);
850 try {
851 const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), {
852 signal: controller.signal,
853 credentials: access.credentials ? "include" : "same-origin",
854 });
855 if (stopped || generation !== session) return;
856 if (response.status === 401 || response.status === 403) return handleUnauthorized();
857 await response.body?.getReader().read();
858 } catch {
859 } finally {
860 clearTimeout(timeout);
861 controller.abort();
862 if (primeController === controller) primeController = null;
863 }
864 };
865
866 /**
867 * serve-sim can be locked to WebRTC (EAS Simulator always is), and then
868 * refuses AVCC and MJPEG; serve-emu advertises WebRTC the same way. An
869 * unreadable answer keeps the default path.
870 */
871 const probeWebRtc = async (): Promise<WebRtcSettings | null> => {
872 const probe = new AbortController();
873 const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS);
874 try {
875 const response = await fetch(httpUrl(`/api?device=${device}`), {
876 signal: probe.signal,
877 credentials: access.credentials ? "include" : "same-origin",
878 });
879 if (!response.ok) return null;
880 const api = decodeServeSimApi(await response.json());
881 if (Option.isNone(api)) return null;
882 const settings = decodeWebRtcSettings(api.value.streamSettings ?? api.value.stream);
883 return Option.isSome(settings) ? settings.value : null;
884 } catch {
885 return null;
886 } finally {
887 clearTimeout(timeout);
888 }
889 };
890
891 const webRtcUrl = (action: "offer" | "close") =>
892 httpUrl(avcc ? `/helper/${device}/webrtc/${action}` : `/webrtc/${action}?device=${device}`);
893
894 const closePeer = () => {
895 const current = peer;
896 if (!current) return;
897 peer = null;
898 current.connection.close();
899 current.video.srcObject = null;
900 current.video.remove();
901 // Releases the capture session; best effort because the page may be unloading.
902 void fetch(webRtcUrl("close"), {
903 method: "POST",
904 keepalive: true,
905 headers: { "content-type": "application/json" },
906 body: JSON.stringify({ sessionId: current.sessionId }),
907 credentials: access.credentials ? "include" : "same-origin",
908 }).catch(() => {});
909 };
910
911 const readWebRtc = async (settings: WebRtcSettings) => {
912 const lifecycle = generation;
913 const feedGeneration = ++videoGeneration;
914 closePeer();
915 const connection = new RTCPeerConnection({
916 ...(settings.iceServers
917 ? {
918 iceServers: settings.iceServers.map((server) => ({
919 ...server,
920 urls: typeof server.urls === "string" ? server.urls : [...server.urls],
921 })),
922 }
923 : {}),
924 iceTransportPolicy: settings.iceTransportPolicy ?? "all",
925 });
926 // Frames only reach requestVideoFrameCallback from a video that is in the
927 // document; it stays invisible because the sink paints them.
928 const video = document.createElement("video");
929 video.muted = true;
930 video.playsInline = true;
931 video.setAttribute("aria-hidden", "true");
932 video.style.cssText =
933 "position:fixed;width:1px;height:1px;opacity:0;pointer-events:none;left:0;top:0";
934 document.body.append(video);
935 const current = { connection, sessionId: webRtcSessionId(), video };
936 peer = current;
937 const isCurrent = () =>
938 !stopped &&
939 generation === lifecycle &&
940 videoGeneration === feedGeneration &&
941 peer === current;
942 // After the first frame nothing else bounds a feed that stops: the peer
943 // can sit `disconnected` with the last frame on the canvas. A long gap
944 // between frames reconnects through the same path as a failed connection.
945 let checksWithoutFrame = 0;
946 let watchdog: ReturnType<typeof setInterval> | null = null;
947 const clearWatchdog = () => {
948 if (watchdog !== null) clearInterval(watchdog);
949 watchdog = null;
950 };
951 const retry = (detail: string) => {
952 clearWatchdog();
953 if (!isCurrent()) return;
954 closePeer();
955 connecting(detail);
956 scheduleRetry("video", () => void readWebRtc(settings));
957 };
958 const presentFrame = () => {
959 if (!isCurrent()) {
960 clearWatchdog();
961 return;
962 }
963 if (video.videoWidth > 0 && video.videoHeight > 0) {
964 paint(video, video.videoWidth, video.videoHeight);
965 checksWithoutFrame = 0;
966 watchdog ??= setInterval(() => {
967 if (!isCurrent()) return clearWatchdog();
968 checksWithoutFrame++;
969 if (checksWithoutFrame * WEBRTC_STALL_CHECK_MS >= WEBRTC_STALL_TIMEOUT_MS) {
970 retry("Device stream stalled. Reconnecting…");
971 }
972 }, WEBRTC_STALL_CHECK_MS);
973 }
974 video.requestVideoFrameCallback(presentFrame);
975 };
976 const transceiver = connection.addTransceiver("video", { direction: "recvonly" });
977 const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
978 if (codecs.length > 0) {
979 // serve-sim encodes H.264; other codecs stay as fallbacks it may refuse.
980 transceiver.setCodecPreferences([
981 ...codecs.filter((codec) => codec.mimeType.toLowerCase() === "video/h264"),
982 ...codecs.filter((codec) => codec.mimeType.toLowerCase() !== "video/h264"),
983 ]);
984 }
985 connection.ontrack = (event) => {
986 if (!isCurrent()) return;
987 iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
988 video.srcObject = event.streams[0] ?? new MediaStream([event.track]);
989 void video.play().catch(() => {});
990 video.requestVideoFrameCallback(presentFrame);
991 };
992 connection.onconnectionstatechange = () => {
993 if (connection.connectionState === "failed")
994 retry("Device stream connection failed. Reconnecting…");
995 };
996 try {
997 await connection.setLocalDescription(await connection.createOffer());
998 const gathered = await new Promise<boolean>((resolve) => {
999 if (connection.iceGatheringState === "complete") return resolve(true);
1000 const timer = setTimeout(() => done(false), iceGatheringDeadlineMs);
1001 function done(complete: boolean) {
1002 clearTimeout(timer);
1003 connection.removeEventListener("icegatheringstatechange", changed);
1004 resolve(complete);
1005 }
1006 function changed() {
1007 if (connection.iceGatheringState === "complete") done(true);
1008 }
1009 connection.addEventListener("icegatheringstatechange", changed);
1010 });
1011 if (!isCurrent()) return;
1012 // serve-sim takes all candidates in the offer; later candidates never trickle.
1013 // Cloud simulators need a reflexive or relay candidate, so retry incomplete offers with
1014 // longer deadlines up to the bound, then send what was gathered.
1015 const sdp = connection.localDescription?.sdp ?? "";
1016 if (
1017 !gathered &&
1018 !/^a=candidate:.* typ (?:srflx|relay)\b/m.test(sdp) &&
1019 iceGatheringDeadlineMs < ICE_GATHERING_MAX_TIMEOUT_MS
1020 ) {
1021 iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS);
1022 return retry("Device stream found no network candidates yet. Retrying…");
1023 }
1024 const response = await fetch(webRtcUrl("offer"), {
1025 method: "POST",
1026 headers: { "content-type": "application/json" },
1027 body: JSON.stringify({
1028 type: "offer",
1029 sdp: connection.localDescription?.sdp,
1030 sessionId: current.sessionId,
1031 codec: "h264",
1032 }),
1033 credentials: access.credentials ? "include" : "same-origin",
1034 });
1035 if (!isCurrent()) {
1036 await response.body?.cancel();
1037 return;
1038 }
1039 if (response.status === 401 || response.status === 403) return handleUnauthorized();
1040 // 409: another viewer is still negotiating with this simulator.
1041 if (!response.ok) return retry(`Device stream offer failed (${response.status}). Retrying…`);
1042 const answer = decodeWebRtcAnswer(await response.json());
1043 if (!isCurrent()) return;
1044 if (Option.isNone(answer))
1045 return retry("Device stream returned an invalid answer. Retrying…");
1046 await connection.setRemoteDescription(answer.value);
1047 } catch (cause) {
1048 retry((cause as Error).message);
1049 }
1050 };
1051
1052 /**
1053 * Called once AVCC or MJPEG was refused. `locked` means serve-sim already
1054 * said it streams WebRTC only, so a missing /api answer still switches.
1055 */
1056 const switchToWebRtc = async (failure: string, locked: boolean) => {
1057 const session = generation;
1058 if (typeof RTCPeerConnection === "undefined") return fail(failure);
1059 const settings = (await probeWebRtc()) ?? (locked ? { transport: "webrtc" as const } : null);
1060 if (stopped || generation !== session) return;
1061 if (!settings) return fail(failure);
1062 if (mjpeg) {
1063 mjpeg = false;
1064 releaseImage?.();
1065 releaseImage = null;
1066 events.onCanvasVideo?.();
1067 }
1068 // MJPEG may have shown frames already: say the stream is connecting again
1069 // rather than leave a blank canvas until the first WebRTC frame.
1070 connecting("Switching to WebRTC.");
1071 void readWebRtc(settings);
1072 };
1073
1074 const startDuoVideo = (panels: DuoPanelSinks) => {
1075 for (const panel of panelClients) panel.stop();
1076 // Physical handoff elects a native surface. Fixed-panel encoders can keep
1077 // an inactive shutdown frame after election, so this build uses one active
1078 // feed instead of decoding a third stream alongside the two fixed feeds.
1079 const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const);
1080 panelClients = ids.map((id) => {
1081 const output = id === 1 ? panels.cover : panels.inner;
1082 return createDeviceStreamClient(
1083 { ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true },
1084 {
1085 present(source, width, height) {
1086 if (id === null) {
1087 return sink.present(source, width, height);
1088 }
1089 // An inactive native LCD can emit its shutdown black frame. Retain its last useful image.
1090 if (screen?.screenId !== id) return true;
1091 const retained = output.present(source, width, height);
1092 const primary = sink.present(source, width, height);
1093 return retained && primary;
1094 },
1095 },
1096 {
1097 onStatus: (status, detail) => {
1098 if (status === "error") events.onDuoUnavailable?.(detail);
1099 },
1100 onScreen: () => {},
1101 onInputConnected: () => {},
1102 onMjpegFallback: () => {},
1103 onUnauthorized: handleUnauthorized,
1104 },
1105 );
1106 });
1107 for (const panel of panelClients) panel.start();
1108 };
1109
1110 const connectIosInput = async () => {
1111 if (stopped) return;
1112 const session = generation;
1113 if (!serveSimOnly) await primeIosHelper(session);
1114 if (stopped || generation !== session) return;
1115 const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
1116 ws.binaryType = "arraybuffer";
1117 socket = ws;
1118 iosInputAdmitted = false;
1119 iosInputUnavailable = false;
1120 ws.onmessage = (event) => {
1121 if (stopped || socket !== ws) return;
1122 if (!(event.data instanceof ArrayBuffer)) return;
1123 const bytes = new Uint8Array(event.data);
1124 if (socket !== ws || stopped || bytes.length < 1) return;
1125 if (bytes[0] === IOS_TAG_INPUT_ADMITTED) {
1126 iosInputAdmitted = true;
1127 events.onInputConnected(!iosInputUnavailable);
1128 return;
1129 }
1130 try {
1131 const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
1132 if (bytes[0] === 0x90) {
1133 const reply = decodeControlReply(payload);
1134 if (Option.isSome(reply)) duoControl.receive(reply.value);
1135 } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
1136 const config = decodeScreenConfig(payload);
1137 if (Option.isSome(config)) {
1138 iosInputUnavailable = config.value.inputUnavailable === true;
1139 events.onInputConnected(
1140 iosInputAdmitted && !iosInputUnavailable,
1141 iosInputUnavailable ? "Simulator input is unavailable" : undefined,
1142 );
1143 const previous = screen;
1144 screen = config.value;
1145 if (screen.hingePose && screen.hingePose !== previous?.hingePose)
1146 rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait";
1147 else if (screen.orientation !== previous?.orientation)
1148 rotationCursor = screen.orientation;
1149 panelSinks?.onScreen?.(screen);
1150 events.onScreen(screen);
1151 // A surface election can leave an existing decoder on the former
1152 // encoder description. Reopen only video to acquire the elected
1153 // surface's seed and codec configuration; HID and the viewer stay.
1154 if (
1155 panelSinks &&
1156 screen.supportsPhysicalOrientation &&
1157 previous &&
1158 screen.screenId !== previous.screenId
1159 )
1160 startDuoVideo(panelSinks);
1161 if (pendingOrientation) {
1162 const receipt = pendingOrientation;
1163 pendingOrientation = null;
1164 duoControl.receive({
1165 requestId: receipt.requestId,
1166 ok: true,
1167 });
1168 }
1169 }
1170 }
1171 } catch {}
1172 };
1173 ws.onclose = (event) => {
1174 if (socket !== ws) return;
1175 socket = null;
1176 iosInputAdmitted = false;
1177 duoControl.clear();
1178 rotationCursor = null;
1179 if (stopped) return;
1180 if (event.code === 4001) return fail(leaseEnded(event.reason || "ended"));
1181 events.onInputConnected(
1182 false,
1183 event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`),
1184 );
1185 // A rejected HTTP upgrade surfaces as 1006, including an expired stream ticket.
1186 if (
1187 event.code === 1008 ||
1188 event.code === 4401 ||
1189 (event.code === 1006 && access.query.wsTicket)
1190 )
1191 return handleUnauthorized();
1192 scheduleRetry("input", () => void connectIosInput());
1193 };
1194 ws.onerror = () => ws.close();
1195 };
1196
1197 /** `inputOnly` keeps a socket for input beside a WebRTC feed, so its reconnects leave video alone. */
1198 const connectAndroid = (inputOnly = false) => {
1199 if (stopped) return;
1200 const ws = new WebSocket(
1201 wsUrl(`/ws?device=${device}&${inputOnly ? "video=0" : "frame-meta=1"}`),
1202 );
1203 ws.binaryType = "arraybuffer";
1204 socket = ws;
1205 ws.onopen = () => {
1206 if (stopped || socket !== ws) return;
1207 if (!inputOnly) connecting();
1208 events.onInputConnected(true);
1209 };
1210 ws.onmessage = (event) => {
1211 if (stopped || socket !== ws || inputOnly) return;
1212 if (typeof event.data === "string") {
1213 // The encoder restarts at a new size when the device rotates; the
1214 // next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
1215 if (isVideoSessionMessage(event.data)) {
1216 closeDecoder();
1217 configuring = false;
1218 connecting();
1219 requestKeyframe();
1220 }
1221 return;
1222 }
1223 if (!(event.data instanceof ArrayBuffer)) return;
1224 const packet = parseSemuPacket(event.data);
1225 const needsScan =
1226 packet.isKey === null ||
1227 (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured"));
1228 const scanned = needsScan ? scanAccessUnit(packet.data) : null;
1229 const isKey = packet.isKey ?? scanned?.isKey ?? false;
1230 if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) {
1231 if (configuring) return;
1232 configuring = true;
1233 const epoch = decoderEpoch;
1234 const isCurrent = () => !stopped && socket === ws;
1235 void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then(
1236 (configured) => {
1237 if (!isCurrent() || epoch !== decoderEpoch) return;
1238 configuring = false;
1239 awaitingKeyframe = true;
1240 if (configured) requestKeyframe();
1241 },
1242 );
1243 return;
1244 }
1245 if (!videoDecoder || videoDecoder.state !== "configured") {
1246 if (!isKey) requestKeyframe();
1247 return;
1248 }
1249 decode(isKey, packet.data, packet.timestamp);
1250 };
1251 ws.onclose = (event) => {
1252 if (socket !== ws) return;
1253 socket = null;
1254 closeDecoder();
1255 if (stopped) return;
1256 events.onInputConnected(false, event.reason || `closed ${event.code}`);
1257 if (
1258 event.code === 1008 ||
1259 event.code === 4401 ||
1260 (event.code === 1006 && access.query.wsTicket)
1261 )
1262 return handleUnauthorized();
1263 if (!inputOnly) {
1264 configuring = false;
1265 connecting(event.reason || undefined);
1266 }
1267 // A full socket re-probes, so one failed /api read does not pin SEMU on a WebRTC host.
1268 scheduleRetry("input", () => (inputOnly ? connectAndroid(true) : void startAndroid()));
1269 };
1270 ws.onerror = () => ws.close();
1271 };
1272
1273 const startAndroid = async () => {
1274 const session = generation;
1275 const settings = await probeWebRtc();
1276 if (stopped || generation !== session) return;
1277 if (settings) {
1278 if (typeof RTCPeerConnection === "undefined")
1279 return fail("This browser cannot receive the device's WebRTC stream.");
1280 if (!target.videoOnly) connectAndroid(true);
1281 void readWebRtc(settings);
1282 } else if (useWebCodecs) connectAndroid();
1283 else fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
1284 };
1285
1286 const start = () => {
1287 if (!stopped) return;
1288 stopped = false;
1289 generation++;
1290 configuring = false;
1291 iosInputAdmitted = false;
1292 iosInputUnavailable = false;
1293 connecting();
1294 if (serveSimOnly) {
1295 if (!target.videoOnly) void connectIosInput();
1296 if (useWebCodecs) void readIosVideo();
1297 else fail("This browser cannot decode this device's stream (WebCodecs unavailable).");
1298 } else if (platform === "ios") {
1299 if (!target.videoOnly) void connectIosInput();
1300 if (useWebCodecs) void readIosVideo();
1301 else fallBackToMjpeg();
1302 } else void startAndroid();
1303 };
1304
1305 const stop = () => {
1306 if (stopped) return;
1307 stopped = true;
1308 generation++;
1309 videoGeneration++;
1310 duoControl.clear();
1311 rotationCursor = null;
1312 for (const panel of panelClients) panel.stop();
1313 panelClients = [];
1314 panelSinks = null;
1315 mjpeg = false;
1316 clearFrameTimer();
1317 if (videoReadTimer !== null) clearTimeout(videoReadTimer);
1318 videoReadTimer = null;
1319 releaseImage?.();
1320 releaseImage = null;
1321 mjpegImage = null;
1322 for (const timer of retryTimers.values()) clearTimeout(timer);
1323 retryTimers.clear();
1324 primeController?.abort();
1325 primeController = null;
1326 controller?.abort();
1327 controller = null;
1328 closePeer();
1329 const discarded = socket;
1330 socket = null;
1331 discarded?.close();
1332 closeDecoder();
1333 };
1334
1335 const send = (payload: Uint8Array<ArrayBuffer> | string) => {
1336 if (
1337 !stopped &&
1338 (!avcc || (iosInputAdmitted && !iosInputUnavailable)) &&
1339 socket?.readyState === WebSocket.OPEN
1340 )
1341 socket.send(payload);
1342 };
1343
1344 const rawPoint = (x: number, y: number) => {
1345 // serve-sim streams the raw framebuffer; rotated devices need input
1346 // remapped into that raw space.
1347 if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y };
1348 switch (screen.orientation) {
1349 case "landscape_left":
1350 return { x: y, y: 1 - x };
1351 case "landscape_right":
1352 return { x: 1 - y, y: x };
1353 case "portrait_upside_down":
1354 return { x: 1 - x, y: 1 - y };
1355 default:
1356 return { x, y };
1357 }
1358 };
1359
1360 return {
1361 start,
1362 stop,
1363 setMjpegImage,
1364 controlDuo: duoControl.enqueue,
1365 sendRawTouch: (phase, x, y) => {
1366 if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y }));
1367 },
1368 setDuoPanels(panels) {
1369 if (platform !== "ios" || target.videoOnly || stopped || panelSinks === panels) return;
1370 if (panels && !screen?.supportsHingeAngle) return;
1371 panelSinks = panels;
1372 videoGeneration++;
1373 controller?.abort();
1374 controller = null;
1375 closePeer();
1376 closeDecoder();
1377 const retry = retryTimers.get("video");
1378 if (retry) clearTimeout(retry);
1379 retryTimers.delete("video");
1380 for (const panel of panelClients) panel.stop();
1381 panelClients = [];
1382 if (!panels) {
1383 if (useWebCodecs) void readIosVideo();
1384 return;
1385 }
1386 startDuoVideo(panels);
1387 },
1388 sendTouch: (phase, x, y) => {
1389 if (avcc) {
1390 send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));
1391 return;
1392 }
1393 const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up";
1394 send(JSON.stringify({ type: "touch", action, x, y }));
1395 },
1396 sendKey: (event, phase) => {
1397 if (avcc) {
1398 const usage = hidUsageForCode(event.code);
1399 if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage }));
1400 return;
1401 }
1402 if (phase !== "down") return;
1403 if (event.key === "Escape") return send(JSON.stringify({ type: "back" }));
1404 const keycode = ANDROID_KEYCODE_BY_KEY[event.key];
1405 if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode }));
1406 if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) {
1407 send(JSON.stringify({ type: "text", text: event.key }));
1408 }
1409 },
1410 sendText: (text) => {
1411 if (avcc) return;
1412 let chunk = "";
1413 let chunkBytes = 0;
1414 let sentChunks = 0;
1415 for (const character of text) {
1416 const characterBytes = utf8ByteLength(character.codePointAt(0)!);
1417 if (chunkBytes + characterBytes > ANDROID_MAX_TEXT_BYTES) {
1418 send(JSON.stringify({ type: "text", text: chunk }));
1419 if (++sentChunks >= ANDROID_MAX_TEXT_CHUNKS) return;
1420 chunk = "";
1421 chunkBytes = 0;
1422 }
1423 chunk += character;
1424 chunkBytes += characterBytes;
1425 }
1426 if (chunk.length > 0) send(JSON.stringify({ type: "text", text: chunk }));
1427 },
1428 pressButton: (button) => {
1429 if (avcc) {
1430 const name =
1431 button === "home"
1432 ? "home"
1433 : button === "appSwitcher" || button === "recents"
1434 ? "app_switcher"
1435 : button === "power"
1436 ? "lock"
1437 : button === "back" && platform === "android"
1438 ? "back"
1439 : null;
1440 if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name }));
1441 return;
1442 }
1443 const type = button === "appSwitcher" ? "recents" : button;
1444 if (type === "home" || type === "back" || type === "recents" || type === "power") {
1445 send(JSON.stringify({ type }));
1446 }
1447 },
1448 rotate: () => {
1449 if (platform !== "ios") return;
1450 const current = screen?.supportsHingeAngle
1451 ? (rotationCursor ?? screen.orientation)
1452 : (screen?.orientation ?? "portrait");
1453 const next =
1454 IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
1455 if (screen?.supportsHingeAngle) {
1456 rotationCursor = next;
1457 duoControl.enqueue({ control: "orientation", value: next });
1458 } else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
1459 },
1460 setOrientation: (orientation) => {
1461 if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));
1462 },
1463 };
1464}