feat(xr): WebGPU multiview frame pass, XR bridge, and clustered lights

examples/src/examples/test/xr-views.example.mjs

@@ -43,6 +43,85 @@ createOptions.resourceHandlers = [
 const app = new pc.AppBase(canvas);
 app.init(createOptions);
 
+// Composite pass that samples a 4-layer texture array and writes the layers into a 2x2 grid on
+// the canvas backbuffer. Used by the WebGPU branch to visualise the per-eye renders produced by
+// FramePassMultiView.
+class CompositeArrayPass extends pc.RenderPassShaderQuad {
+    constructor(graphicsDevice, sourceTexture, numViews) {
+        super(graphicsDevice);
+        this.name = 'CompositeArrayPass';
+        this.sourceTexture = sourceTexture;
+        this.numViews = numViews;
+
+        this.shader = pc.ShaderUtils.createShader(graphicsDevice, {
+            uniqueName: 'XrViewsCompositeShader',
+            attributes: { aPosition: pc.SEMANTIC_POSITION },
+            vertexChunk: 'quadVS',
+
+            fragmentWGSL: /* wgsl */ `
+                var sourceTexture: texture_2d_array<f32>;
+                var sourceTextureSampler: sampler;
+                varying uv0: vec2f;
+
+                @fragment fn fragmentMain(input: FragmentInput) -> FragmentOutput {
+                    var output: FragmentOutput;
+                    let q = floor(input.uv0 * 2.0);
+                    // clamp layer: at uv edge (1,1) q can be 2, giving layer 4 for a 4-layer array
+                    let layer = clamp(i32(q.x + q.y * 2.0), 0, 3);
+                    let localUV = input.uv0 * 2.0 - q;
+                    output.color = textureSample(sourceTexture, sourceTextureSampler, localUV, layer);
+                    return output;
+                }
+            `
+        });
+    }
+
+    // Called once per frame during frame graph construction, after frameStart() but before any
+    // GPU commands are recorded. This is the safe point to resize the array texture: the previous
+    // frame's GPU commands have already been submitted and deferred-destroys flushed, so
+    // destroying the old GPU texture here won't conflict with any pending submit.
+    frameUpdate() {
+        super.frameUpdate();
+
+        const tex = this.sourceTexture;
+        if (!tex) return;
+
+        // resize to match the current backbuffer dimensions
+        const { width, height } = this.device.backBuffer;
+        if (width > 0 && height > 0) {
+            tex.resize(width, height);
+        }
+
+        // re-populate device.xrSubImages with the current (possibly new) GPU texture reference.
+        // this must happen after resize() so the GPU texture handle is up-to-date.
+        const gpuTexture = tex.impl?.gpuTexture;
+        if (gpuTexture) {
+            const viewFormat = gpuTexture.format;
+            const subImages = [];
+            for (let i = 0; i < this.numViews; i++) {
+                subImages.push({
+                    colorTexture: gpuTexture,
+                    viewDescriptor: {
+                        dimension: '2d',
+                        baseArrayLayer: i,
+                        arrayLayerCount: 1,
+                        baseMipLevel: 0,
+                        mipLevelCount: 1
+                    },
+                    viewport: { x: 0, y: 0, width, height },
+                    viewFormat
+                });
+            }
+            this.device.xrSubImages = subImages;
+        }
+    }
+
+    execute() {
+        this.device.scope.resolve('sourceTexture').setValue(this.sourceTexture);
+        super.execute();
+    }
+}
+
 const assetListLoader = new pc.AssetListLoader(Object.values(assets), app.assets);
 assetListLoader.load(() => {
     app.start();
@@ -77,8 +156,6 @@ assetListLoader.load(() => {
         shadowBias: 0.3,
         normalOffsetBias: 0.2,
         intensity: 1.0,
-
-        // enable shadow casting
         castShadows: false,
         shadowDistance: 1000
     });
@@ -109,10 +186,13 @@ assetListLoader.load(() => {
     camera.script.create('orbitCameraInputTouch');
     app.root.addChild(camera);
 
-    // Create XR views using a loop
+    // Create mock XR views using a loop. The number of views differs between backends because
+    // each backend uses a different visualisation:
+    // - WebGL: a single canvas-sized backbuffer with 4 sub-rect viewports (2x2 grid).
+    // - WebGPU: a 4-layer array texture, one full-canvas-size view per layer, then composited
+    //   into a 2x2 grid as a separate post-render pass.
+    const numViews = 4;
     const viewsList = [];
-    const numViews = 4; // 2x2 grid
-
     for (let i = 0; i < numViews; i++) {
         viewsList.push({
             updateTransforms(transform) {
@@ -136,11 +216,53 @@ assetListLoader.load(() => {
         }
     };
 
+    // ----------------------------------------------------------------------------------------
+    // WebGPU-only setup: drive FramePassMultiView via a fake bridge - we provide the per-view
+    // sub-image entries on the device that the wrapper consumes (mirroring what
+    // WebgpuXrBridge.beginFrame does on a real headset).
+    // ----------------------------------------------------------------------------------------
+    let arrayTex = null;
+    let compositeCamera = null;
+    if (device.isWebGPU) {
+
+        const createArrayTexture = (w, h) => new pc.Texture(device, {
+            name: 'XrViewsArrayTexture',
+            format: device.backBufferFormat,
+            arrayLength: numViews,
+            width: w,
+            height: h,
+            mipmaps: false,
+            addressU: pc.ADDRESS_CLAMP_TO_EDGE,
+            addressV: pc.ADDRESS_CLAMP_TO_EDGE,
+            minFilter: pc.FILTER_LINEAR,
+            magFilter: pc.FILTER_LINEAR
+        });
+
+        arrayTex = createArrayTexture(Math.max(canvas.width, 1), Math.max(canvas.height, 1));
+
+        // composite camera renders second (higher priority) and only runs the composite pass that
+        // samples the four rendered layers and lays them out as a 2x2 grid on the canvas
+        compositeCamera = new pc.Entity('XrViewsCompositeCamera');
+        compositeCamera.addComponent('camera', {
+            priority: 1,
+            clearColor: new pc.Color(0, 0, 0, 0),
+            clearColorBuffer: false,
+            clearDepthBuffer: false,
+            clearStencilBuffer: false
+        });
+        app.root.addChild(compositeCamera);
+
+        const compositePass = new CompositeArrayPass(device, arrayTex, numViews);
+        compositePass.init(null);
+        compositeCamera.camera.framePasses = [compositePass];
+    }
+
     const cameraComponent = camera.camera;
     app.on('update', (/** @type {number} */ dt) => {
 
         const width = canvas.width;
         const height = canvas.height;
+        const isWebgpu = device.isWebGPU;
 
         // update all views - supply some matrices to make pre view rendering possible
         // note that this is not complete set up, view frustum does not get updated and so
@@ -166,13 +288,25 @@ assetListLoader.load(() => {
 
             view.projViewOffMat.mul2(view.projMat, viewMat);
 
-            // adjust viewport for a 2x2 grid layout
             const viewport = view.viewport;
-            viewport.x = (view.viewIndex % 2 === 0) ? 0 : width / 2;
-            viewport.y = (view.viewIndex < 2) ? 0 : height / 2;
-            viewport.z = width / 2;
-            viewport.w = height / 2;
+            if (isWebgpu) {
+                // each view writes into its own array layer at full size; the composite pass
+                // arranges the four layers into a 2x2 grid on the canvas
+                viewport.x = 0;
+                viewport.y = 0;
+                viewport.z = width;
+                viewport.w = height;
+            } else {
+                // WebGL: 4 sub-viewports of a single canvas-sized backbuffer (2x2 grid).
+                // WebGL viewport y=0 is the bottom of the canvas, so views 0,1 go in the top
+                // row (y = height/2) to match the WebGPU composite's top-down layout.
+                viewport.x = (view.viewIndex % 2 === 0) ? 0 : width / 2;
+                viewport.y = (view.viewIndex < 2) ? height / 2 : 0;
+                viewport.z = width / 2;
+                viewport.w = height / 2;
+            }
         });
+
     });
 });
 

src/platform/graphics/webgpu/webgpu-graphics-device.js

@@ -170,6 +170,36 @@ class WebgpuGraphicsDevice extends GraphicsDevice {
      */
     xrColorTextureViewFormat = null;
 
+    /**
+     * Optional `GPUTextureViewDescriptor` describing how the framebuffer's color attachment view
+     * should be created from {@link WebgpuGraphicsDevice#xrColorTexture}. Used to pick the right
+     * array layer / mip when XR provides a layered (texture array) projection layer. Set per eye
+     * by {@link FramePassMultiView}; cleared back to `null` outside the per-view loop.
+     *
+     * @type {any} // `GPUTextureViewDescriptor | null`; using `any` to avoid exporting WebGPU types in published typings.
+     * @ignore
+     */
+    xrColorTextureViewDescriptor = null;
+
+    /**
+     * Per-view XR sub-image entries populated each frame by the WebGPU XR bridge. Each entry
+     * describes one XR view: the underlying GPU color texture, the view descriptor that selects the
+     * right slice, the viewport, and the view's GPU format. Empty outside immersive WebGPU XR.
+     *
+     * @type {{ colorTexture: any, viewDescriptor: any, viewport: any, viewFormat: any }[]}
+     * @ignore
+     */
+    xrSubImages = [];
+
+    /**
+     * Active XR view index for the multi-view rendering wrapper, or `-1` when not iterating views.
+     * Read by the forward renderer's per-view inner loop to render only the active eye.
+     *
+     * @type {number}
+     * @ignore
+     */
+    xrCurrentViewIndex = -1;
+
     /**
      * When set, used as the main color attachment in {@link WebgpuGraphicsDevice#frameStart} if there is
      * no XR color texture and no canvas {@link GPUCanvasContext#getCurrentTexture} (for example headless
@@ -238,14 +268,27 @@ class WebgpuGraphicsDevice extends GraphicsDevice {
         this.resolver.destroy();
         this.resolver = null;
 
-        this.xrColorTexture = null;
-        this.xrColorTextureViewFormat = null;
+        this._clearXrState();
 
         this.externalBackbuffer = null;
 
         super.destroy();
     }
 
+    /**
+     * Reset all per-frame WebGPU XR render state to its inactive defaults. Called by the XR bridge
+     * at the end of each XR frame and on session teardown, and by the graphics device on destroy.
+     *
+     * @ignore
+     */
+    _clearXrState() {
+        this.xrColorTexture = null;
+        this.xrColorTextureViewFormat = null;
+        this.xrColorTextureViewDescriptor = null;
+        this.xrSubImages.length = 0;
+        this.xrCurrentViewIndex = -1;
+    }
+
     initDeviceCaps() {
 
         const limits = this.wgpu?.limits;

src/platform/graphics/webgpu/webgpu-render-target.js

@@ -196,9 +196,15 @@ class WebgpuRenderTarget {
         Debug.assert(gpuTexture);
         this.assignedColorTexture = gpuTexture;
 
-        // create view (optionally handles srgb conversion)
-        const view = gpuTexture.createView({ format: viewFormat });
-        DebugHelper.setLabel(view, 'Framebuffer.assignedColor');
+        const wgpuDevice = /** @type {WebgpuGraphicsDevice} */ (this.renderTarget.device);
+        const xrViewDesc = wgpuDevice?.xrColorTextureViewDescriptor;
+        // WebXR may supply a per-eye view descriptor (e.g. array layer); merge in our view format
+        // for sRGB / reinterpret when it matches the session color texture.
+        const xrSlice = xrViewDesc && gpuTexture === wgpuDevice.xrColorTexture;
+        const view = gpuTexture.createView(
+            xrSlice ? { ...xrViewDesc, format: viewFormat } : { format: viewFormat }
+        );
+        DebugHelper.setLabel(view, xrSlice ? 'Framebuffer.xrColorTextureView' : 'Framebuffer.contextColorTextureView');
 
         // use it as render buffer or resolve target
         const colorAttachment = this.renderPassDescriptor.colorAttachments[0];

src/platform/graphics/webgpu/webgpu-xr-bridge.js

@@ -48,8 +48,7 @@ class WebgpuXrBridge {
         this._binding = null;
         this._layer = null;
         this._cachedFramebufferSize.set(0, 0);
-        device.xrColorTexture = null;
-        device.xrColorTextureViewFormat = null;
+        device._clearXrState();
     }
 
     /**
@@ -59,8 +58,7 @@ class WebgpuXrBridge {
     beginFrame(frame, referenceSpace) {
         const device = this.xrBridge.device;
 
-        device.xrColorTexture = null;
-        device.xrColorTextureViewFormat = null;
+        device._clearXrState();
 
         if (!this._binding || !this._layer || !referenceSpace) {
             return;
@@ -71,35 +69,55 @@ class WebgpuXrBridge {
             return;
         }
 
-        /** @type {XRGPUSubImage|null} */
-        let firstSub = null;
+        const subImages = device.xrSubImages;
         for (let i = 0; i < pose.views.length; i++) {
+            let sub;
             try {
-                const sub = this._binding.getViewSubImage(this._layer, pose.views[i]);
-                // TODO: stereo / multiview WebGPU — drive per-eye color (e.g. texture-array projection layer);
-                // v1 only keeps the first view for xrColorTexture (see createProjectionLayer TODO).
-                if (i === 0) {
-                    firstSub = sub;
-                }
+                sub = this._binding.getViewSubImage(this._layer, pose.views[i]);
             } catch (e) {
                 this.xrBridge._onBindingError?.(e);
                 return;
             }
+
+            const colorTexture = sub?.colorTexture;
+            if (!colorTexture) continue;
+
+            // pull the per-view GPUTextureViewDescriptor from the sub-image when available; this is
+            // required when the runtime returns a layered texture (texture-array) for stereo so each
+            // eye binds the right slice
+            let viewDescriptor = null;
+            if (typeof sub.getViewDescriptor === 'function') {
+                try {
+                    const desc = sub.getViewDescriptor();
+                    if (desc) {
+                        viewDescriptor = { ...desc };
+                    }
+                } catch (e) {
+                    // descriptor optional - fall through to default view
+                }
+            }
+
+            subImages.push({
+                colorTexture,
+                viewDescriptor,
+                viewport: sub.viewport,
+                viewFormat: colorTexture.format
+            });
         }
 
-        if (firstSub?.colorTexture) {
-            device.xrColorTexture = firstSub.colorTexture;
-            device.xrColorTextureViewFormat = firstSub.colorTexture.format;
-            this._cachedFramebufferSize.set(firstSub.colorTexture.width, firstSub.colorTexture.height);
+        // First view: seeds device.xr* for WebGPU frameStart (runs before per-eye rendering) and
+        // caches texture size for getFramebufferSize when the projection layer omits dimensions.
+        const first = subImages[0];
+        if (first) {
+            device.xrColorTexture = first.colorTexture;
+            device.xrColorTextureViewFormat = first.viewFormat;
+            this._cachedFramebufferSize.set(first.colorTexture.width, first.colorTexture.height);
         }
     }
 
     endFrame() {
         const device = this.xrBridge.device;
-        if (device) {
-            device.xrColorTexture = null;
-            device.xrColorTextureViewFormat = null;
-        }
+        device?._clearXrState();
     }
 
     /**
@@ -270,8 +288,7 @@ class WebgpuXrBridge {
         this._binding = null;
         this._layer = null;
         this._cachedFramebufferSize.set(0, 0);
-        device.xrColorTexture = null;
-        device.xrColorTextureViewFormat = null;
+        device._clearXrState();
 
         session.updateRenderState({
             layers: [],