Open History to see the last commit
1import {
2 AmbientLight,
3 Box3,
4 CanvasTexture,
5 DirectionalLight,
6 LinearFilter,
7 Matrix4,
8 PerspectiveCamera,
9 Quaternion,
10 Scene,
11 SRGBColorSpace,
12 Vector3,
13 WebGLRenderer,
14} from "three";
15import {
16 createDeviceModelSlot,
17 type DeviceModelSource,
18 type DeviceAccessorySource,
19} from "./model.ts";
20import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
21import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
22import {
23 createAndroidFoldScene,
24 DEFAULT_FOLD_INNER_ASPECT,
25 isFoldInnerAspect,
26} from "./androidFoldScene.ts";
27import { createRenderScheduler } from "./renderScheduler.ts";
28import { createDeviceMotion } from "./deviceMotion.ts";
29import { createDeviceFraming } from "./deviceFraming.ts";
30import { nearestDeviceView } from "./deviceViewSnap.ts";
31import type { DeviceScreenSize } from "./stream.ts";
32import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts";
33
34export interface PhoneViewer {
35 readonly setModel: (source: DeviceModelSource | null) => void;
36 readonly setAccessory: (source: DeviceAccessorySource | null) => void;
37 readonly frameUpdated: () => void;
38 readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void;
39 readonly setFoldAngle: (angle: number | null) => void;
40 readonly resize: (width: number, height: number, pixelRatio: number) => void;
41 readonly screenPoint: (
42 x: number,
43 y: number,
44 captured?: boolean,
45 ) => { x: number; y: number } | null;
46 readonly orbit: (deltaX: number, deltaY: number) => void;
47 readonly setInteractionActive: (active: boolean, mode: "touch" | "orbit") => void;
48 readonly resetPose: () => void;
49 readonly dispose: () => void;
50}
51
52const ANDROID_ORIENTATION_TURN_MS = 450;
53const ANDROID_FOLD_TURN_MS = 850;
54
55/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */
56export function createPhoneViewer(options: {
57 readonly canvas: HTMLCanvasElement;
58 readonly source: HTMLCanvasElement;
59 readonly onUnavailable: () => void;
60 readonly onModelError?: (cause: unknown) => void;
61 readonly onFramingAspect?: (aspect: number) => void;
62 readonly profile?: DeviceShapeProfile;
63 readonly model?: DeviceModelSource | null;
64 readonly accessory?: DeviceAccessorySource | null;
65 readonly foldAngle?: number | null;
66}): PhoneViewer {
67 const renderer = new WebGLRenderer({
68 canvas: options.canvas,
69 alpha: true,
70 antialias: true,
71 powerPreference: "low-power",
72 });
73 renderer.outputColorSpace = SRGBColorSpace;
74 const makeTexture = () => {
75 const next = new CanvasTexture(options.source);
76 next.colorSpace = SRGBColorSpace;
77 next.minFilter = LinearFilter;
78 next.magFilter = LinearFilter;
79 next.generateMipmaps = false;
80 return next;
81 };
82 let texture = makeTexture();
83 let textureWidth = options.source.width;
84 let textureHeight = options.source.height;
85 const scene = new Scene();
86 const camera = new PerspectiveCamera(32, 1, 0.1, 30);
87 camera.position.z = 5.5;
88 const ambient = new AmbientLight(0xffffff, 2.4);
89 const key = new DirectionalLight(0xe4edff, 5);
90 key.position.set(-3, 4, 5);
91 const rim = new DirectionalLight(0xffffff, 4);
92 rim.position.set(3, 1, -3);
93 const fill = new DirectionalLight(0x9facd4, 2);
94 fill.position.set(-2, -2, -4);
95 scene.add(ambient, key, rim, fill);
96
97 let screen: DeviceScreenSize | null = null;
98 let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);
99 let profile = options.profile ?? IOS_PHONE_SHAPE;
100 let foldAngle = options.foldAngle ?? null;
101 let orientationAngle =
102 foldAngle !== null && profile.id.startsWith("android") ? 0 : layout.rotation;
103 let orientationTurn: { from: number; to: number; startedAt: number } | null = null;
104 let imported: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
105 let modelSource = options.model ?? null;
106 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
107 let accessoryBounds: Box3 | null = null;
108 let foldTurn: { from: number; to: number; startedAt: number } | null = null;
109 const rawAspect = () => options.source.width / options.source.height;
110 let foldAspect = isFoldInnerAspect(rawAspect()) ? rawAspect() : DEFAULT_FOLD_INNER_ASPECT;
111 const createFoldScene = (angle: number, displayLayout = layout) =>
112 createAndroidFoldScene(texture, displayLayout, angle, foldAspect);
113 const visibleFoldAngle = (fallback: number) => {
114 if (!foldTurn) return fallback;
115 const progress = Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
116 const eased = progress * progress * (3 - 2 * progress);
117 return foldTurn.from + (foldTurn.to - foldTurn.from) * eased;
118 };
119 let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =
120 foldAngle !== null && profile.id.startsWith("android")
121 ? createFoldScene(foldAngle)
122 : createPhoneScene(texture, layout, profile);
123 scene.add(phone.root);
124 let disposed = false;
125 const rest = new Quaternion();
126 const motion = createDeviceMotion({
127 choose: (rotation) =>
128 nearestDeviceView(rotation, [{ rotation: new Quaternion(), yawLimit: Math.PI / 3 }])!
129 .rotation,
130 });
131 motion.setPose(rest, performance.now(), true);
132 const framing = createDeviceFraming();
133 const reducedMotion = () =>
134 globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false;
135 let viewport = { width: 0, height: 0, pixelRatio: 1 };
136 let drawingBuffer = { width: 0, height: 0, pixelRatio: 0 };
137
138 let framingAspect: number | null = null;
139 const fit = (immediate = false) => {
140 if (!viewport.width || !viewport.height) return;
141 camera.aspect = viewport.width / viewport.height;
142 const bounds = new Box3(
143 new Vector3(-phone.width / 2, -phone.height / 2, 0),
144 new Vector3(phone.width / 2, phone.height / 2, 0),
145 );
146 if (imported && accessoryBounds) bounds.union(accessoryBounds);
147 bounds.applyMatrix4(new Matrix4().makeRotationZ(orientationAngle));
148 const size = bounds.getSize(new Vector3());
149 const aspect = size.x / size.y;
150 if (aspect !== framingAspect) {
151 framingAspect = aspect;
152 options.onFramingAspect?.(aspect);
153 }
154 phone.root.updateMatrixWorld(true);
155 framing.setBounds(
156 new Box3().setFromObject(phone.root),
157 (camera.fov * Math.PI) / 360,
158 camera.aspect,
159 performance.now(),
160 immediate,
161 );
162 applyCamera();
163 };
164 const applyCamera = () => {
165 camera.position.set(framing.center.x, framing.center.y, framing.distance());
166 camera.lookAt(framing.center.x, framing.center.y, 0);
167 camera.updateProjectionMatrix();
168 };
169 const applyPose = () => {
170 phone.root.quaternion.copy(motion.rotation);
171 phone.orientation.rotation.z = orientationAngle;
172 };
173 const scheduler = createRenderScheduler(() => {
174 if (disposed || !viewport.width || !viewport.height) return;
175 try {
176 if (
177 drawingBuffer.width !== viewport.width ||
178 drawingBuffer.height !== viewport.height ||
179 drawingBuffer.pixelRatio !== viewport.pixelRatio
180 ) {
181 // Canvas allocation clears the previous image. Commit it with the redraw,
182 // rather than exposing an empty buffer between ResizeObserver and the next frame.
183 renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.pixelRatio);
184 drawingBuffer = viewport;
185 }
186 const now = performance.now();
187 if (motion.advance(now, reducedMotion())) {
188 applyPose();
189 fit(reducedMotion());
190 }
191 if (orientationTurn) {
192 const progress = Math.min(
193 1,
194 (now - orientationTurn.startedAt) / ANDROID_ORIENTATION_TURN_MS,
195 );
196 const eased = progress * progress * (3 - 2 * progress);
197 orientationAngle =
198 orientationTurn.from + (orientationTurn.to - orientationTurn.from) * eased;
199 if (progress === 1) orientationTurn = null;
200 applyPose();
201 fit(reducedMotion());
202 }
203 if (foldTurn && "setAngle" in phone) {
204 const progress = Math.min(1, (now - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
205 const eased = progress * progress * (3 - 2 * progress);
206 phone.setAngle(foldTurn.from + (foldTurn.to - foldTurn.from) * eased);
207 if (progress === 1) foldTurn = null;
208 fit(reducedMotion());
209 }
210 framing.advance(now, reducedMotion());
211 applyCamera();
212 renderer.render(scene, camera);
213 if (motion.needsFrame() || framing.needsFrame() || orientationTurn || foldTurn)
214 scheduler.invalidate();
215 } catch {
216 options.onUnavailable();
217 }
218 });
219 const updateLayout = (nextProfile = profile) => {
220 const next = phoneDisplayLayout(screen, options.source.width, options.source.height);
221 const resized =
222 textureWidth !== options.source.width || textureHeight !== options.source.height;
223 if (
224 resized ||
225 nextProfile !== profile ||
226 next.aspect !== layout.aspect ||
227 next.rawLandscape !== layout.rawLandscape ||
228 next.rotation !== layout.rotation
229 ) {
230 if (resized) {
231 const previous = texture;
232 texture = makeTexture();
233 textureWidth = options.source.width;
234 textureHeight = options.source.height;
235 phone.setDisplay(texture, next);
236 previous.dispose();
237 }
238 const frameAspect = rawAspect();
239 const innerChanged = isFoldInnerAspect(frameAspect) && frameAspect !== foldAspect;
240 if (innerChanged) foldAspect = frameAspect;
241 if (!imported && "setAngle" in phone && innerChanged) {
242 const angle = visibleFoldAngle(foldAngle ?? 180);
243 scene.remove(phone.root);
244 phone.dispose();
245 phone = createFoldScene(angle, next);
246 scene.add(phone.root);
247 } else if (
248 !imported &&
249 !("setAngle" in phone) &&
250 (nextProfile !== profile || next.aspect !== layout.aspect)
251 ) {
252 scene.remove(phone.root);
253 phone.dispose();
254 phone = createPhoneScene(texture, next, nextProfile);
255 scene.add(phone.root);
256 } else {
257 phone.setDisplay(texture, next);
258 }
259 if (next.rotation !== layout.rotation) {
260 if (nextProfile.id.startsWith("android") && !("setAngle" in phone) && !reducedMotion()) {
261 const difference = Math.atan2(
262 Math.sin(next.rotation - orientationAngle),
263 Math.cos(next.rotation - orientationAngle),
264 );
265 orientationTurn = {
266 from: orientationAngle,
267 to: orientationAngle + difference,
268 startedAt: performance.now(),
269 };
270 } else {
271 orientationTurn = null;
272 orientationAngle = "setAngle" in phone ? 0 : next.rotation;
273 }
274 }
275 layout = next;
276 profile = nextProfile;
277 applyPose();
278 fit(!orientationTurn);
279 }
280 applyPose();
281 };
282 const modelSlot = createDeviceModelSlot({
283 load: loadDeviceModel,
284 onError: options.onModelError,
285 install(model) {
286 const next = disposed
287 ? null
288 : model
289 ? createImportedPhoneScene(model.asset, texture, layout)
290 : foldAngle !== null && profile.id.startsWith("android")
291 ? createFoldScene(foldAngle)
292 : createPhoneScene(texture, layout, profile);
293 foldTurn = null;
294 scene.remove(phone.root);
295 phone.dispose();
296 imported = model;
297 if (!next) return;
298 phone = next;
299 scene.add(phone.root);
300 if (accessory) {
301 if (imported) phone.orientation.add(accessory.asset);
302 else accessory.asset.removeFromParent();
303 }
304 applyPose();
305 fit(true);
306 scheduler.invalidate();
307 },
308 });
309 const accessorySlot = createDeviceModelSlot({
310 load: loadDeviceModel,
311 onError: options.onModelError,
312 install(model) {
313 const bounds = model ? new Box3().setFromObject(model.asset) : null;
314 if (
315 bounds &&
316 (bounds.isEmpty() ||
317 ![...bounds.min.toArray(), ...bounds.max.toArray()].every(Number.isFinite))
318 )
319 throw new Error("Device accessory has invalid bounds");
320 accessory?.asset.removeFromParent();
321 accessory = model;
322 accessoryBounds = bounds;
323 if (disposed) return;
324 if (imported && accessory) phone.orientation.add(accessory.asset);
325 fit(true);
326 scheduler.invalidate();
327 },
328 });
329 const setAccessory = (source: DeviceAccessorySource | null) => {
330 accessorySlot.set(source?.modelId === modelSource?.id ? source : null);
331 };
332 const contextLost = (event: Event) => {
333 event.preventDefault();
334 options.onUnavailable();
335 };
336 options.canvas.addEventListener("webglcontextlost", contextLost);
337 applyPose();
338 modelSlot.set(options.model ?? null);
339 setAccessory(options.accessory ?? null);
340 return {
341 setModel(source) {
342 if (source?.id !== modelSource?.id || source?.url !== modelSource?.url)
343 accessorySlot.set(null);
344 modelSource = source;
345 modelSlot.set(source);
346 },
347 setAccessory,
348 setFoldAngle(next) {
349 if (disposed || next === foldAngle) return;
350 const previous = foldAngle;
351 foldAngle = next;
352 if (imported) return;
353 if (next === null || !("setAngle" in phone)) {
354 scene.remove(phone.root);
355 phone.dispose();
356 phone = next === null ? createPhoneScene(texture, layout, profile) : createFoldScene(next);
357 scene.add(phone.root);
358 orientationTurn = null;
359 orientationAngle = next === null ? layout.rotation : 0;
360 foldTurn = null;
361 applyPose();
362 fit(true);
363 } else {
364 const from = visibleFoldAngle(previous ?? next);
365 if (reducedMotion()) {
366 foldTurn = null;
367 phone.setAngle(next);
368 fit(true);
369 } else {
370 foldTurn = { from, to: next, startedAt: performance.now() };
371 }
372 }
373 scheduler.invalidate();
374 },
375 frameUpdated() {
376 if (disposed) return;
377 updateLayout();
378 texture.needsUpdate = true;
379 scheduler.invalidate();
380 },
381 setScreen(next, nextProfile = profile) {
382 if (disposed) return;
383 screen = next;
384 updateLayout(nextProfile);
385 scheduler.invalidate();
386 },
387 resize(width, height, pixelRatio) {
388 if (disposed) return;
389 if (![width, height, pixelRatio].every(Number.isFinite) || width <= 0 || height <= 0) return;
390 const ratio = Math.min(2, Math.max(1, pixelRatio));
391 if (viewport.width === width && viewport.height === height && viewport.pixelRatio === ratio)
392 return;
393 viewport = { width, height, pixelRatio: ratio };
394 fit(true);
395 scheduler.invalidate();
396 },
397 screenPoint(x, y, captured = false) {
398 if (disposed) return null;
399 applyPose();
400 return phone.screenPoint(x, y, camera, captured);
401 },
402 orbit(deltaX, deltaY) {
403 if (disposed) return;
404 motion.orbit(deltaX * viewport.width, deltaY * viewport.height, performance.now());
405 scheduler.invalidate();
406 },
407 setInteractionActive(active, mode) {
408 if (disposed) return;
409 const now = performance.now();
410 if (mode === "orbit") motion.dragActive(active, now);
411 else {
412 motion.hold(active, now);
413 framing.hold(active, now);
414 }
415 scheduler.invalidate();
416 },
417 resetPose() {
418 if (disposed) return;
419 motion.reset(rest, performance.now());
420 scheduler.invalidate();
421 },
422 dispose() {
423 if (disposed) return;
424 disposed = true;
425 scheduler.dispose();
426 const hadImported = imported !== null;
427 accessorySlot.dispose();
428 modelSlot.dispose();
429 options.canvas.removeEventListener("webglcontextlost", contextLost);
430 scene.remove(phone.root);
431 if (!hadImported) phone.dispose();
432 texture.dispose();
433 renderer.dispose();
434 renderer.forceContextLoss();
435 },
436 };
437}