1// @effect-diagnostics globalTimers:off - Native display handoff has a bounded acknowledgement window.
2import {
3 AmbientLight,
4 Box3,
5 CanvasTexture,
6 DirectionalLight,
7 Euler,
8 LinearFilter,
9 PerspectiveCamera,
10 PMREMGenerator,
11 Quaternion,
12 Scene,
13 SRGBColorSpace,
14 Vector3,
15 WebGLRenderer,
16} from "three";
17import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js";
18import {
19 createDuoScene,
20 duoDisplayKey,
21 duoFrameMatches,
22 type DuoPanelId,
23 type DuoHingeLeaf,
24} from "./duoScene.ts";
25import { loadDeviceModel } from "./modelScene.ts";
26import { createDeviceModelSlot, type DeviceModelSource } from "./model.ts";
27import { createDeviceFraming } from "./deviceFraming.ts";
28import { createDeviceMotion } from "./deviceMotion.ts";
29import { duoViewSnaps, nearestDuoView, type DuoRestFace } from "./duoSnap.ts";
30import { createRenderScheduler } from "./renderScheduler.ts";
31import type { DeviceScreenSize } from "./stream.ts";
32import { duoScreenSettled, type DuoPose } from "./duoControl.ts";
33
34export interface DuoViewer {
35 readonly setScreen: (screen: DeviceScreenSize | null) => void;
36 readonly setHingePreview: (angle: number | null, pinch?: boolean) => void;
37 readonly setInteractionActive: (active: boolean, mode?: "touch" | "orbit") => void;
38 readonly rejectOrientation: () => void;
39 readonly frameUpdated: (panel: DuoPanelId, primary?: HTMLCanvasElement) => void;
40 readonly resize: (width: number, height: number, pixelRatio: number) => void;
41 readonly screenPoint: (
42 x: number,
43 y: number,
44 captured?: boolean,
45 ) => { x: number; y: number } | null;
46 readonly orbit: (x: number, y: number) => void;
47 readonly beginHinge: (x: number, y: number) => boolean;
48 readonly resetPose: () => void;
49 readonly cancelInput: () => void;
50 readonly dispose: () => void;
51}
52
53export function createDuoViewer(options: {
54 canvas: HTMLCanvasElement;
55 sources: Record<DuoPanelId, HTMLCanvasElement>;
56 model: DeviceModelSource;
57 onUnavailable: () => void;
58 onModelError?: (cause: unknown) => void;
59 onPanelRequested?: (panel: DuoPanelId) => void;
60 onOrientationRequested?: (orientation: DeviceScreenSize["orientation"]) => void;
61}): DuoViewer {
62 const surfaces = ([1, 3] as const)
63 .map((id) => {
64 const canvas = document.createElement("canvas");
65 canvas.width = id === 1 ? 784 : 1600;
66 canvas.height = id === 1 ? 1140 : 1125;
67 const context = canvas.getContext("2d", { alpha: false });
68 if (!context) throw new Error("Duo display canvas is unavailable");
69 context.fillStyle = "#080a10";
70 context.fillRect(0, 0, canvas.width, canvas.height);
71 return { id, canvas, context };
72 })
73 .map(({ id, canvas, context }) => {
74 const texture = new CanvasTexture(canvas);
75 texture.colorSpace = SRGBColorSpace;
76 texture.minFilter = texture.magFilter = LinearFilter;
77 texture.generateMipmaps = false;
78 return { id, canvas, context, texture };
79 });
80 const renderer = new WebGLRenderer({
81 canvas: options.canvas,
82 alpha: true,
83 antialias: true,
84 powerPreference: "low-power",
85 });
86 renderer.outputColorSpace = SRGBColorSpace;
87 const scene = new Scene();
88 const environment = (() => {
89 const generator = new PMREMGenerator(renderer);
90 const room = new RoomEnvironment();
91 try {
92 return generator.fromScene(room, 0.04);
93 } catch (cause) {
94 for (const surface of surfaces) surface.texture.dispose();
95 renderer.dispose();
96 renderer.forceContextLoss();
97 throw cause;
98 } finally {
99 room.dispose();
100 generator.dispose();
101 }
102 })();
103 scene.environment = environment.texture;
104 const camera = new PerspectiveCamera(36, 1, 0.1, 150);
105 const key = new DirectionalLight(0xffffff, 4);
106 key.position.set(-6, 10, 14);
107 const fill = new DirectionalLight(0xc7dcff, 2);
108 fill.position.set(8, -3, -5);
109 scene.add(new AmbientLight(0xffffff, 2.4), key, fill);
110 let model: ReturnType<typeof createDuoScene> | null = null;
111 let screen: DeviceScreenSize | null = null;
112 let readyKey = "";
113 let primaryKey = "";
114 let activationAt = 0;
115 let disposed = false;
116 let viewport = { width: 0, height: 0, ratio: 1 };
117 let buffer = { width: 0, height: 0, ratio: 0 };
118 let restFace: DuoRestFace = "inside";
119 let requestedOrientation: DeviceScreenSize["orientation"] | null = null;
120 let viewOrientation: DeviceScreenSize["orientation"] | null = null;
121 const pivot = new Vector3();
122 const targetPivot = new Vector3();
123 let requestedPanel: DuoPanelId | null = null;
124 let handoffTimer: ReturnType<typeof setTimeout> | null = null;
125 const clearHandoff = () => {
126 if (handoffTimer) clearTimeout(handoffTimer);
127 handoffTimer = null;
128 requestedPanel = null;
129 };
130 const activeSnaps = () =>
131 model
132 ? duoViewSnaps(
133 model.restFrames(screen?.screenId === 1 ? 1 : 3),
134 screen?.screenId === 1 ? 1 : 3,
135 )
136 : [];
137 const snaps = () => {
138 if (!model || !screen?.supportsPhysicalOrientation || !options.onPanelRequested)
139 return activeSnaps();
140 const cover = duoViewSnaps(model.restFrames(1), 1);
141 return angle > 20 && angle < 180
142 ? [...duoViewSnaps(model.restFrames(3), 3), ...cover]
143 : activeSnaps();
144 };
145 const orbit = createDeviceMotion({
146 choose(rotation) {
147 const snap = nearestDuoView(rotation, snaps());
148 if (!snap) return rotation;
149 restFace = snap.face;
150 const panel = snap.face === "cover" ? 1 : 3;
151 if (screen && panel !== (requestedPanel ?? screen.screenId) && options.onPanelRequested) {
152 clearHandoff();
153 requestedPanel = panel;
154 model?.cancelInput();
155 options.onPanelRequested(panel);
156 handoffTimer = setTimeout(() => {
157 clearHandoff();
158 const confirmed = nearestDuoView(orbit.rotation, activeSnaps());
159 if (confirmed) {
160 restFace = confirmed.face;
161 orbit.setPose(confirmed.rotation, performance.now());
162 }
163 scheduler.invalidate();
164 }, 5000);
165 } else if (
166 screen &&
167 requestedPanel === null &&
168 snap.orientation !== viewOrientation &&
169 options.onOrientationRequested
170 ) {
171 viewOrientation = snap.orientation;
172 requestedOrientation = snap.orientation;
173 options.onOrientationRequested(snap.orientation);
174 }
175 return snap.rotation;
176 },
177 });
178 let angle = 180;
179 let targetAngle = 180;
180 let previewAngle: number | null = null;
181 let hingeLeaf: DuoHingeLeaf | null = null;
182 let appliedAngle = Number.NaN;
183 let interactionActive = false;
184 const framing = createDeviceFraming();
185 const framingBounds = new Box3();
186 let firstPose = true;
187 let foldPending = false;
188 let physicalPose: DuoPose = "open";
189 let presentationAngle = 180;
190 let targetPresentation = new Quaternion();
191 let lastTime = 0;
192 const reduced =
193 typeof matchMedia === "function" && matchMedia("(prefers-reduced-motion: reduce)").matches;
194 const applyPose = () => {
195 if (!model) return;
196 const before = hingeLeaf && appliedAngle !== angle ? model.leafRotation(hingeLeaf) : null;
197 model.setAngle(angle);
198 appliedAngle = angle;
199 if (before && hingeLeaf) {
200 // Rebase both the displayed rotation and its rest target so releasing a
201 // pinch cannot resume the pre-fold body rotation.
202 const correction = before.multiply(model.leafRotation(hingeLeaf).invert());
203 if (correction.angleTo(new Quaternion()) > 1e-8)
204 orbit.setPose(orbit.rotation.clone().multiply(correction), performance.now(), true);
205 }
206 model.root.quaternion.copy(orbit.rotation);
207 model.root.position.set(0, 0, 0);
208 const frames = model.restFrames(restFace === "cover" ? 1 : 3);
209 const frame = frames.find((frame) => frame.face === restFace) ?? frames[0];
210 targetPivot.copy(frame?.center ?? new Vector3());
211 model.root.position.copy(pivot).applyQuaternion(model.root.quaternion).negate();
212 };
213 const updateCamera = () => {
214 camera.position.set(framing.center.x, framing.center.y, framing.distance());
215 camera.lookAt(framing.center.x, framing.center.y, 0);
216 camera.updateProjectionMatrix();
217 };
218 const fit = (immediate = false) => {
219 if (!model || !viewport.width || !viewport.height) return;
220 camera.aspect = viewport.width / viewport.height;
221 framing.setBounds(
222 framingBounds.setFromObject(model.root),
223 (camera.fov * Math.PI) / 360,
224 camera.aspect,
225 performance.now(),
226 immediate,
227 );
228 updateCamera();
229 };
230 const moving = () =>
231 Math.abs(angle - targetAngle) > 0.01 || pivot.distanceTo(targetPivot) > 0.001;
232 const scheduler = createRenderScheduler(() => {
233 if (disposed || !model || !viewport.width || !viewport.height) return;
234 try {
235 const now = performance.now();
236 const elapsed = Math.max(0, (now - lastTime) / 1000);
237 const amount = reduced
238 ? 1
239 : 1 - Math.exp(-14 * Math.max(0, (now - lastTime) / 1000 || 0.016));
240 lastTime = now;
241 const inMotion = moving();
242 const orbitChanged = orbit.advance(now, reduced);
243 if (inMotion) {
244 angle += (targetAngle - angle) * amount;
245
246 if (Math.abs(angle - targetAngle) <= 0.01) angle = targetAngle;
247 applyPose();
248 pivot.lerp(targetPivot, amount);
249 if (pivot.distanceTo(targetPivot) <= 0.001) pivot.copy(targetPivot);
250 applyPose();
251 fit(reduced || elapsed > 0.5);
252 }
253 if (orbitChanged) {
254 applyPose();
255 fit(reduced || elapsed > 0.5);
256 }
257 if (framing.advance(now, reduced)) updateCamera();
258 if (
259 buffer.width !== viewport.width ||
260 buffer.height !== viewport.height ||
261 buffer.ratio !== viewport.ratio
262 ) {
263 renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.ratio);
264 buffer = viewport;
265 }
266 renderer.render(scene, camera);
267 if (inMotion || moving() || orbit.needsFrame() || framing.needsFrame())
268 scheduler.invalidate();
269 } catch {
270 options.onUnavailable();
271 }
272 });
273 const slot = createDeviceModelSlot({
274 load: loadDeviceModel,
275 onError(cause) {
276 options.onModelError?.(cause);
277 options.onUnavailable();
278 },
279 install(loaded) {
280 const next = loaded
281 ? createDuoScene(loaded.asset, { 1: surfaces[0]!.texture, 3: surfaces[1]!.texture })
282 : null;
283 if (model) {
284 scene.remove(model.root);
285 model.dispose();
286 }
287 model = next;
288 appliedAngle = Number.NaN;
289 if (!model || disposed) return;
290 scene.add(model.root);
291 applyPose();
292 if (screen?.screenId === 1 && physicalPose !== "tent") {
293 const snap = nearestDuoView(orbit.rotation, activeSnaps());
294 if (snap) {
295 restFace = snap.face;
296 orbit.setPose(snap.rotation, performance.now(), true);
297 applyPose();
298 }
299 }
300 pivot.copy(targetPivot);
301 applyPose();
302 fit();
303 scheduler.invalidate();
304 },
305 });
306 const lost = (event: Event) => {
307 event.preventDefault();
308 options.onUnavailable();
309 };
310 options.canvas.addEventListener("webglcontextlost", lost);
311 slot.set(options.model);
312 return {
313 setScreen(next) {
314 if (disposed) return;
315 const wasMoving = moving() || orbit.needsFrame();
316 if (duoDisplayKey(screen) !== duoDisplayKey(next)) {
317 readyKey = "";
318 activationAt = performance.now();
319 model?.cancelInput();
320 }
321 const previous = screen;
322 const ownedHandoff = requestedPanel !== null;
323 if (!next || next.screenId === requestedPanel) clearHandoff();
324 const requestedRotation = requestedOrientation !== null;
325 const ownedRotation = requestedRotation && next?.screenId === previous?.screenId;
326 const changedDisplay = next?.screenId !== previous?.screenId;
327 requestedOrientation = null;
328 screen = next;
329 const changedPose = next?.hingePose && next.hingePose !== previous?.hingePose;
330 const folding = hingeLeaf !== null && !changedPose;
331 const angleBefore = previous?.hingeAngle ?? (previous?.screenId === 1 ? 0 : 180);
332 const angleAfter = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
333 if (!next || folding || changedPose) foldPending = false;
334 else if (previous && angleBefore !== angleAfter) foldPending = true;
335 const landedFold = foldPending && !!next && duoScreenSettled(next) && !ownedHandoff;
336 if (next && duoScreenSettled(next)) foldPending = false;
337 const rotated =
338 !foldPending &&
339 next?.screenId === previous?.screenId &&
340 next?.hingeAngle === previous?.hingeAngle &&
341 next?.orientation !== previous?.orientation;
342 const leftPhysicalPose =
343 rotated &&
344 !ownedRotation &&
345 !ownedHandoff &&
346 !folding &&
347 !next?.hingePose &&
348 previewAngle === null;
349 if (firstPose || changedPose || leftPhysicalPose || landedFold) {
350 physicalPose = next?.hingePose ?? (next?.screenId === 1 ? "closed" : "open");
351 presentationAngle = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
352 }
353 // A command reply/config confirms hinge state. Display identity, never an angle heuristic, owns input.
354 targetAngle = previewAngle ?? next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
355 if (
356 firstPose ||
357 changedPose ||
358 landedFold ||
359 (rotated && !ownedRotation && !ownedHandoff && !folding)
360 ) {
361 hingeLeaf = null;
362 viewOrientation = next?.orientation ?? null;
363 const poseAngle = presentationAngle;
364 const fold = ((180 - poseAngle) * Math.PI) / 360;
365 let roll = physicalPose === "closed" ? 0 : Math.PI / 2;
366 if (next && next.width < next.height) {
367 if (next.orientation === "landscape_left") roll -= Math.PI / 2;
368 if (next.orientation === "landscape_right") roll += Math.PI / 2;
369 }
370 if (next?.orientation === "portrait_upside_down") roll -= Math.PI;
371 const physical =
372 physicalPose === "laptop"
373 ? new Euler(-fold + Math.PI / 9, -Math.PI / 9, Math.PI / 2, "YXZ")
374 : physicalPose === "tent"
375 ? new Euler(Math.PI / 2 + Math.PI / 18, -Math.PI / 9, -Math.PI / 2, "YXZ")
376 : new Euler(0, (Math.PI / 2) * Math.pow(1 - poseAngle / 180, 3), 0, "YXZ");
377 targetPresentation = new Quaternion().setFromEuler(physical);
378 if (physicalPose !== "laptop" && physicalPose !== "tent")
379 targetPresentation.premultiply(
380 new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), roll),
381 );
382 if (next?.screenId === 1 && !landedFold && physicalPose !== "tent") {
383 // Open presets can retain native cover readback; face the display's actual hinged normal.
384 // Landed folds already face the cover while the hinge normal is stale.
385 // Tent must stand on both edges with the cover outward; facing it flat would tip it over.
386 const snap = nearestDuoView(targetPresentation, activeSnaps());
387 if (snap) targetPresentation.copy(snap.rotation);
388 }
389 if (firstPose) angle = targetAngle;
390 restFace = next?.screenId === 1 ? "cover" : physicalPose === "laptop" ? "right" : "inside";
391 orbit.setPose(targetPresentation, performance.now(), firstPose);
392 firstPose = false;
393 } else if (
394 next &&
395 changedDisplay &&
396 !ownedHandoff &&
397 !folding &&
398 (restFace === "cover") !== (next.screenId === 1) &&
399 (requestedRotation || duoScreenSettled(next))
400 ) {
401 // A sensor rotation can hand ownership to the other native display.
402 // Face that display without sending another rotation and creating a feedback loop.
403 // Native election can briefly flap to the display a fold does not show and
404 // back; neither is a handoff, and the view may still be easing into place.
405 const snap = nearestDuoView(orbit.rotation, activeSnaps());
406 if (snap) {
407 restFace = snap.face;
408 orbit.setPose(snap.rotation, performance.now());
409 }
410 }
411 if (!wasMoving) lastTime = performance.now();
412 applyPose();
413 fit();
414 scheduler.invalidate();
415 },
416 setHingePreview(next, pinch = false) {
417 if (disposed || (next !== null && (!Number.isFinite(next) || next < 0 || next > 180))) return;
418 if (!moving()) lastTime = performance.now();
419 previewAngle = next;
420 if (next !== null && !pinch) {
421 hingeLeaf = null;
422 } else if (next !== null && !hingeLeaf && model) {
423 clearHandoff();
424 requestedOrientation = null;
425 hingeLeaf = screen?.screenId === 1 && angle > 20 ? "left" : "right";
426 orbit.setPose(orbit.rotation.clone(), performance.now(), true);
427 }
428 orbit.hold(interactionActive || (pinch && next !== null), performance.now());
429 targetAngle = next ?? screen?.hingeAngle ?? (screen?.screenId === 1 ? 0 : 180);
430 if (next !== null && pinch) {
431 angle = next;
432 applyPose();
433 fit();
434 }
435 model?.cancelInput();
436 scheduler.invalidate();
437 },
438 setInteractionActive(active, mode = "touch") {
439 if (disposed) return;
440 if (mode === "orbit") {
441 if (active) hingeLeaf = null;
442 orbit.dragActive(active, performance.now());
443 } else {
444 interactionActive = active;
445 orbit.hold(active || previewAngle !== null, performance.now());
446 framing.hold(active, performance.now());
447 }
448 scheduler.invalidate();
449 },
450 rejectOrientation() {
451 if (disposed || (requestedOrientation === null && requestedPanel === null)) return;
452 clearHandoff();
453 requestedOrientation = null;
454 viewOrientation = screen?.orientation ?? null;
455 const confirmed = activeSnaps().filter((snap) => snap.orientation === screen?.orientation);
456 const snap = nearestDuoView(orbit.rotation, confirmed);
457 if (snap) {
458 restFace = snap.face;
459 orbit.setPose(snap.rotation, performance.now());
460 }
461 scheduler.invalidate();
462 },
463 frameUpdated(id, primary) {
464 if (disposed || screen?.screenId !== id) return;
465 const key = duoDisplayKey(screen);
466 // The elected native feed is authoritative during handoff. Fixed-panel
467 // encoders can retain an inactive blank until that surface changes again.
468 if (!primary && primaryKey === key) return;
469 const source = primary ?? options.sources[id];
470 if (!duoFrameMatches(source, screen)) return;
471 const surface = surfaces[id === 1 ? 0 : 1]!;
472 const { context, canvas } = surface;
473 // Ignore native shutdown blanks only while waiting for an activation. Steady black application content remains valid.
474 if (!primary && !readyKey && performance.now() - activationAt < 1500) {
475 const probe = document.createElement("canvas");
476 probe.width = probe.height = 8;
477 const probeContext = probe.getContext("2d", { willReadFrequently: true });
478 if (probeContext) {
479 probeContext.drawImage(source, 0, 0, 8, 8);
480 if (
481 !probeContext
482 .getImageData(0, 0, 8, 8)
483 .data.some((value, index) => index % 4 !== 3 && value > 3)
484 )
485 return;
486 }
487 }
488 context.save();
489 context.fillStyle = "#080a10";
490 context.fillRect(0, 0, canvas.width, canvas.height);
491 context.translate(canvas.width / 2, canvas.height / 2);
492 // The inner panel is mounted a quarter turn from the native framebuffer.
493 const rotate = id === 3;
494 if (rotate) context.rotate(Math.PI / 2);
495 const scale = Math.min(
496 canvas.width / (rotate ? source.height : source.width),
497 canvas.height / (rotate ? source.width : source.height),
498 );
499 context.drawImage(
500 source,
501 (-source.width * scale) / 2,
502 (-source.height * scale) / 2,
503 source.width * scale,
504 source.height * scale,
505 );
506 context.restore();
507 surface.texture.needsUpdate = true;
508 readyKey = duoDisplayKey(screen);
509 if (primary) primaryKey = readyKey;
510 scheduler.invalidate();
511 },
512 resize(width, height, ratio) {
513 if (disposed || ![width, height, ratio].every(Number.isFinite) || width <= 0 || height <= 0)
514 return;
515 viewport = { width, height, ratio: Math.min(2, Math.max(1, ratio)) };
516 fit(true);
517 scheduler.invalidate();
518 },
519 screenPoint(x, y, captured = false) {
520 if (
521 disposed ||
522 moving() ||
523 previewAngle !== null ||
524 requestedOrientation !== null ||
525 requestedPanel !== null
526 )
527 return null;
528 applyPose();
529 return model?.screenPoint(x, y, camera, screen, readyKey, captured) ?? null;
530 },
531 cancelInput() {
532 hingeLeaf = null;
533 model?.cancelInput();
534 },
535 orbit(x, y) {
536 if (disposed) return;
537 hingeLeaf = null;
538 orbit.orbit(x * viewport.width, y * viewport.height, performance.now());
539 scheduler.invalidate();
540 },
541 beginHinge(x, y) {
542 if (disposed || !model || !screen) return false;
543 applyPose();
544 const leaf = model.hingeLeafAt(x, y, camera);
545 if (!leaf) return false;
546 clearHandoff();
547 requestedOrientation = null;
548 hingeLeaf = screen.screenId === 1 && angle > 20 ? "left" : "right";
549 orbit.setPose(orbit.rotation.clone(), performance.now(), true);
550 model.cancelInput();
551 return true;
552 },
553 resetPose() {
554 if (disposed) return;
555 hingeLeaf = null;
556 requestedOrientation = null;
557 clearHandoff();
558 const snap = activeSnaps().find((snap) => snap.orientation === screen?.orientation);
559 if (snap && snap.orientation !== viewOrientation && options.onOrientationRequested) {
560 viewOrientation = snap.orientation;
561 requestedOrientation = snap.orientation;
562 options.onOrientationRequested(snap.orientation);
563 }
564 orbit.reset(snap?.rotation ?? targetPresentation, performance.now());
565 applyPose();
566 fit();
567 scheduler.invalidate();
568 },
569 dispose() {
570 clearHandoff();
571 if (disposed) return;
572 disposed = true;
573 scheduler.dispose();
574 slot.dispose();
575 options.canvas.removeEventListener("webglcontextlost", lost);
576 for (const surface of surfaces) surface.texture.dispose();
577 scene.environment = null;
578 environment.dispose();
579 renderer.dispose();
580 renderer.forceContextLoss();
581 },
582 };
583}