examples/webxr_vr_layers.html

<!DOCTYPE html>
<html lang="en">
	<head>
		<title>three.js vr - layers</title>
		<meta charset="utf-8">
		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
		<link type="text/css" rel="stylesheet" href="main.css">
	</head>
	<body>

		<div id="info">
			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> media and projection layers<br/>
			(Oculus Browser 16.1+)

			<p> This example demonstrates the use of <a href="https://www.w3.org/TR/webxrlayers-1/">WebXR Layers</a> to render high quality text and video.
				  For static content such as text, using layers increases the usable resolution of the content by avoiding the extra resampling pass that occurs during normal VR rendering.
				  For dynamic content such as video, using layers also improves performance by only copying data when new frames are available.	 </p>
			<br />
			<p><i>See the comments in the code for more information.</i></p>

		</div>

		<script type="importmap">
			{
				"imports": {
					"three": "../build/three.module.js",
					"three/addons/": "./jsm/"
				}
			}
		</script>

		<script type="module">

			import * as THREE from 'three';
			import { GUI } from 'three/addons/libs/lil-gui.module.min.js';
			import { HTMLMesh } from 'three/addons/interactive/HTMLMesh.js';
			import { InteractiveGroup } from 'three/addons/interactive/InteractiveGroup.js';
			import { VRButton } from 'three/addons/webxr/VRButton.js';
			import { XRControllerModelFactory } from 'three/addons/webxr/XRControllerModelFactory.js';
			import { XRHandModelFactory } from 'three/addons/webxr/XRHandModelFactory.js';

			let camera, scene, renderer;
			let video;

			// Four eye charts are rendered to demonstrate the differences in text quality.
			// The two charts on the top are rendered into XRQuadLayers ( substantially more legible ).
			// The two charts on the bottom are rendered to the eye buffer.
			//
			// The two charts on the left are rendered without mipmaps and have aliasing artifacts.
			// The two charts on the right are with mipmaps an don't twinkle but are blurrier.
			// To maximize text legibility, it's important to choose a texture size optimized for the
			// distance of the text. (This example intentionally uses incorrectly large textures to
			// demonstrate this issue.) If the optimal text size can't be determined beforehand, then
			// mipmaps are required to avoid aliasing.
			//
			// The background of the scene is an equirectangular layer. It uses an XRMediaBinding to
			// render the contents of a video element into the scene. This example uses a low resolution
			// video to avoid large files, but using media layers allows video that is higher resolution than normal rendering.

			let snellenTexture;
			let quadLayerPlain;
			let quadLayerMips;
			let guiLayer;
			let guiMesh;
			let errorMesh;

			// Set via GUI.
			const parameters = {
				eyeChartDistanceFt: 20,
			};

			// Data shared between the THREE.Meshes on the bottom and WebXR Layers on the top for the eye
			// charts. See https://en.wikipedia.org/wiki/Snellen_chart for details about the math.
			//
			// The image was designed so that each 2x2px block on the 20/20 line subtends 1 minute of
			// arc. That is
			//    tan(1/60 deg) * 6.1m * 160px/142mm = 2px
			// per block on line 8.
			//
			// This fidelity is beyond any modern consumer headset since it would require ~60px/deg of
			// resolution. The Quest has ~16ppd and the Quest 2 has ~20ppd so only lines 3 or 4 will be
			// legible when using layers. Without layers, you lose ~sqrt(2) in resolution due to the
			// extra resampling.
			const snellenConfig = {
				// The texture is a power of two so that mipmaps can be generated.
				textureSizePx: 512,
				// This is the valid part of the image.
				widthPx: 320,
				heightPx: 450,

				x: 0,
				y: 1.5,
				z: - 6.1, // 20/20 vision @ 20ft = 6.1m

				// This is the size of mesh and the visible part of the quad layer.
				widthMeters: .268, // 320px image * (142mm/160px scale factor)
				heightMeters: .382 // 450px image * (142mm/160px scale factor)
			};

			snellenConfig.cropX = snellenConfig.widthPx / snellenConfig.textureSizePx;
			snellenConfig.cropY = snellenConfig.heightPx / snellenConfig.textureSizePx;

			// The quad layer is a [-1, 1] quad but only a part of it has image data. Scale the layer so
			// that the part with image data is the same size as the mesh.
			snellenConfig.quadWidth = .5 * snellenConfig.widthMeters / snellenConfig.cropX;
			snellenConfig.quadHeight = .5 * snellenConfig.heightMeters / snellenConfig.cropY;

			init();

			function init() {

				scene = new THREE.Scene();

				camera = new THREE.PerspectiveCamera( 50, window.innerWidth / window.innerHeight, 0.1, 10 );
				camera.position.set( 0, 1.6, 3 );

				const hemLight = new THREE.HemisphereLight( 0x808080, 0x606060, 3 );
				const light = new THREE.DirectionalLight( 0xffffff, 3 );
				scene.add( hemLight, light );

				//

				renderer = new THREE.WebGLRenderer( { antialias: false } );
				renderer.setPixelRatio( window.devicePixelRatio );
				renderer.setSize( window.innerWidth, window.innerHeight );
				renderer.setAnimationLoop( animate );
				renderer.setClearAlpha( 1 );
				renderer.setClearColor( new THREE.Color( 0 ), 0 );
				renderer.xr.enabled = true;

				document.body.appendChild( renderer.domElement );
				document.body.appendChild( VRButton.createButton( renderer ) );

				// controllers

				const lineGeometry = new THREE.BufferGeometry().setFromPoints( [
					new THREE.Vector3( 0, 0, 0 ),
					new THREE.Vector3( 0, 0, - 10 )
				] );
				const line = new THREE.Line( lineGeometry, new THREE.LineBasicMaterial( { color: 0x5555ff } ) );

				// The invisible dummyMesh quads and the guiMesh need to be rendered before the controller lines so that they
				// leave a hole in the depth buffer that the lines can intersect.
				line.renderOrder = 1;

				const controllerModelFactory = new XRControllerModelFactory();
				const handModelFactory = new XRHandModelFactory().setPath( './models/fbx/' );

				//

				const controllers = [
					renderer.xr.getController( 0 ),
					renderer.xr.getController( 1 )
				];

				controllers.forEach( ( controller, i ) => {

					const controllerGrip = renderer.xr.getControllerGrip( i );
					controllerGrip.add( controllerModelFactory.createControllerModel( controllerGrip ) );
					scene.add( controllerGrip );

					const hand = renderer.xr.getHand( i );
					hand.add( handModelFactory.createHandModel( hand ) );

					controller.add( line.clone() );
					scene.add( controller, controllerGrip, hand );

				} );

				// Eye charts

				const eyeCharts = new THREE.Group();
				eyeCharts.position.z = snellenConfig.z;
				scene.add( eyeCharts );

				snellenTexture = new THREE.TextureLoader().load( 'textures/snellen.png' );
				snellenTexture.repeat.x = snellenConfig.cropX;
				snellenTexture.repeat.y = snellenConfig.cropY;
				snellenTexture.generateMipmaps = false;
				snellenTexture.minFilter = THREE.LinearFilter;
				const snellenMeshPlain = new THREE.Mesh(
					new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
					new THREE.MeshBasicMaterial( { map: snellenTexture } ) );
				snellenMeshPlain.position.x = snellenConfig.x - snellenConfig.widthMeters;
				snellenMeshPlain.position.y = snellenConfig.y - snellenConfig.heightMeters;
				eyeCharts.add( snellenMeshPlain );

				snellenTexture = new THREE.TextureLoader().load( 'textures/snellen.png' );
				snellenTexture.repeat.x = snellenConfig.cropX;
				snellenTexture.repeat.y = snellenConfig.cropY;
				const snellenMeshMipMap = new THREE.Mesh(
					new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
					new THREE.MeshBasicMaterial( { map: snellenTexture } ) );
				snellenMeshMipMap.position.x = snellenConfig.x + snellenConfig.widthMeters;
				snellenMeshMipMap.position.y = snellenConfig.y - snellenConfig.heightMeters;
				eyeCharts.add( snellenMeshMipMap );

				// The layers don't participate depth testing between each other. Since the projection
				// layer is rendered last, any 3D object will incorrectly overlap layers. To avoid this,
				// invisible quads can be placed into the scene to participate in depth testing when the
				// projection layer is rendered.
				const dummyMeshLeft = new THREE.Mesh(
					new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
					new THREE.MeshBasicMaterial( { colorWrite: false } ) );
				dummyMeshLeft.position.x = snellenConfig.x - snellenConfig.widthMeters;
				dummyMeshLeft.position.y = snellenConfig.y + snellenConfig.heightMeters;
				eyeCharts.add( dummyMeshLeft );

				const dummyMeshRight = dummyMeshLeft.clone( true );
				dummyMeshRight.position.x = snellenConfig.x + snellenConfig.widthMeters;
				eyeCharts.add( dummyMeshRight );

				// The GUI is rendered into an invisible HTMLMesh and the backing canvas's data is copied
				// into a layer as required. Hit testing and interaction is done using standard HTMLMesh
				// behavior, but since the layer is in the same place as the invisible mesh, the user
				// thinks they're directly interacting with the layer.
				const gui = new GUI( { width: 300 } );
				gui.add( parameters, 'eyeChartDistanceFt', 1.0, 20.0 ).onChange( onChange );
				gui.domElement.style.visibility = 'hidden';

				function onChange() {

					eyeCharts.position.z = - parameters.eyeChartDistanceFt * 0.3048;
					snellenConfig.z = eyeCharts.position.z;

					if ( quadLayerPlain ) {

						quadLayerPlain.transform = new XRRigidTransform( {
							x: snellenConfig.x - snellenConfig.widthMeters,
							y: snellenConfig.y + snellenConfig.heightMeters,
							z: eyeCharts.position.z
						} );

					}

					if ( quadLayerMips ) {

						quadLayerMips.transform = new XRRigidTransform( {
							x: snellenConfig.x + snellenConfig.widthMeters,
							y: snellenConfig.y + snellenConfig.heightMeters,
							z: eyeCharts.position.z
						} );

					}

					guiLayer.needsUpdate = true;

				}

				const group = new InteractiveGroup();
				group.listenToXRControllerEvents( controllers[ 0 ] );
				group.listenToXRControllerEvents( controllers[ 1 ] );
				scene.add( group );

				guiMesh = new HTMLMesh( gui.domElement );
				guiMesh.position.set( 1.0, 1.5, - 1.0 );
				guiMesh.rotation.y = - Math.PI / 4;
				guiMesh.scale.setScalar( 2 );
				guiMesh.material.colorWrite = false;
				guiMesh.material.transparent = false;
				group.add( guiMesh );

				// Error message if layer initialization fails.
				const errorCanvas = document.createElement( 'canvas' );
				errorCanvas.width = 400;
				errorCanvas.height = 40;
				const errorContext = errorCanvas.getContext( '2d' );
				errorContext.fillStyle = '#FF0000';
				errorContext.fillRect( 0, 0, errorCanvas.width, errorCanvas.height );
				errorContext.fillStyle = '#000000';
				errorContext.font = '28px sans-serif';
				errorContext.fillText( 'ERROR: Layers not initialized!', 10, 30 );

				errorMesh = new THREE.Mesh(
					new THREE.PlaneGeometry( 1, .1 ),
					new THREE.MeshBasicMaterial( { map: new THREE.CanvasTexture( errorCanvas ) } )
				);
				errorMesh.position.z = - 1;
				errorMesh.position.y = 1.5;
				errorMesh.visible = false;
				scene.add( errorMesh );

				window.addEventListener( 'resize', onWindowResize, false );

				video = document.createElement( 'video' );
				video.loop = true;
				video.src = 'textures/MaryOculus.webm';

			}

			function onWindowResize() {

				camera.aspect = window.innerWidth / window.innerHeight;
				camera.updateProjectionMatrix();

				renderer.setSize( window.innerWidth, window.innerHeight );

			}
			//

			function animate( t, frame ) {

				const xr = renderer.xr;
				const session = xr.getSession();
				const gl = renderer.getContext();

				// Init layers once in immersive mode and video is ready.
				const useLayers = session &&
					session.enabledFeatures !== undefined &&
					session.enabledFeatures.includes( 'layers' ) &&
					XRMediaBinding !== undefined;

				if ( ! useLayers ) {

					errorMesh.visible = true;

				}

				if (
					session &&
					useLayers &&
					session.hasMediaLayer === undefined &&
					video.readyState >= 2
				) {

					session.hasMediaLayer = true;
					session.requestReferenceSpace( 'local-floor' ).then( ( refSpace ) => {

						// Create Quad layers for Snellen chart.
						const glBinding = xr.getBinding();
						const quadLayerConfig = {
							width: snellenConfig.quadWidth,
							height: snellenConfig.quadHeight,
							viewPixelWidth: snellenConfig.textureSizePx,
							viewPixelHeight: snellenConfig.textureSizePx,
							isStatic: true,
							space: refSpace,
							layout: 'mono',
							transform: new XRRigidTransform( {
								x: snellenConfig.x - snellenConfig.widthMeters,
								y: snellenConfig.y + snellenConfig.heightMeters,
								z: snellenConfig.z
							} )
						};

						quadLayerPlain = glBinding.createQuadLayer( quadLayerConfig );

						quadLayerConfig.mipLevels = 3;
						quadLayerConfig.transform = new XRRigidTransform( {
							x: snellenConfig.x + snellenConfig.widthMeters,
							y: snellenConfig.y + snellenConfig.heightMeters,
							z: snellenConfig.z
						} );
						quadLayerMips = glBinding.createQuadLayer( quadLayerConfig );

						// Create GUI layer.
						guiLayer = glBinding.createQuadLayer( {
							width: guiMesh.geometry.parameters.width,
							height: guiMesh.geometry.parameters.height,
							viewPixelWidth: guiMesh.material.map.image.width,
							viewPixelHeight: guiMesh.material.map.image.height,
							space: refSpace,
							transform: new XRRigidTransform( guiMesh.position, guiMesh.quaternion )
						} );

						// Create background EQR video layer.
						const mediaBinding = new XRMediaBinding( session );
						const equirectLayer = mediaBinding.createEquirectLayer(
							video,
							{
								space: refSpace,
								layout: 'stereo-left-right',
								// Rotate by 45 deg to avoid stereo conflict with the 3D geometry.
								transform: new XRRigidTransform(
									{},
									{ x: 0, y: .28, z: 0, w: .96 }
								)
							}
						);

						errorMesh.visible = false;

						session.updateRenderState( {
							layers: [
								equirectLayer,
								quadLayerPlain,
								quadLayerMips,
								guiLayer,
								session.renderState.layers[ 0 ]
							]
						} );

						video.play();

					} );

				}

				// Copy image to layers as required.
				// needsRedraw is set on creation or if the underlying GL resources of a layer are lost.
				if ( session && quadLayerPlain && quadLayerPlain.needsRedraw ) {

					const glayer = xr.getBinding().getSubImage( quadLayerPlain, frame );
					renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
					gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
					gl.texSubImage2D( gl.TEXTURE_2D, 0,
						( snellenConfig.textureSizePx - snellenConfig.widthPx ) / 2,
						( snellenConfig.textureSizePx - snellenConfig.heightPx ) / 2,
						snellenConfig.widthPx, snellenConfig.heightPx,
						gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );

				}

				// Same as above but also gl.generateMipmap.
				if ( session && quadLayerMips && quadLayerMips.needsRedraw ) {

					const glayer = xr.getBinding().getSubImage( quadLayerMips, frame );
					renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
					gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
					gl.texSubImage2D( gl.TEXTURE_2D, 0,
						( snellenConfig.textureSizePx - snellenConfig.widthPx ) / 2,
						( snellenConfig.textureSizePx - snellenConfig.heightPx ) / 2,
						snellenConfig.widthPx, snellenConfig.heightPx,
						gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );
					gl.generateMipmap( gl.TEXTURE_2D );

				}

				// Same as above, but guiLayer.needsUpdate is set when the user interacts with the GUI.
				if ( session && guiLayer && ( guiLayer.needsRedraw || guiLayer.needsUpdate ) ) {

					const glayer = xr.getBinding().getSubImage( guiLayer, frame );
					renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
					gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
					const canvas = guiMesh.material.map.image;
					gl.texSubImage2D( gl.TEXTURE_2D, 0, 0, 0, canvas.width, canvas.height, gl.RGBA, gl.UNSIGNED_BYTE, canvas );
					guiLayer.needsUpdate = false;

				}

				renderer.render( scene, camera );

			}


		</script>
	</body>
</html>