From 25292d1860b6d67be8fdc63a9f30655b6dbf8df9 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Tue, 1 Sep 2026 14:31:59 +0100 Subject: [PATCH 01/13] Add SceneDepthReader, reading a camera's scene depth back to the CPU The depth a camera renders for its effects is only available to shaders. SceneDepthReader renders a requested region of it through the same chunk the effects sample it with and reads the result back, so an application can use it too - autofocus being the obvious case. Reads are asynchronous and any number may be in flight, so one can be issued every frame. The depth is recorded on the camera as it is published, because the uniform holding it is global: with more than one camera rendering a depth, the last to render owns that uniform, so anything wanting a particular camera's depth has to read it from the camera instead. The scene pass now accumulates the depth in reciprocal space. Previously it stored a coverage weighted mean of the depths with the background pinned at the far clip, which returns a depth no surface is at wherever coverage is partial - a half covered pixel over a surface at 5 units with a far clip of 1000 stored 502. Averaging the reciprocals instead keeps the background's contribution small, so such a pixel falls off toward the background smoothly rather than being dragged out to it. Consumers all go through getLinearScreenDepth, which inverts the average; the encoding is declared by the producer through a new shader parameter, so the prepass keeps writing the depth outright. Adds two examples: gaussian-splatting/depth-effects, where volumetric fog and depth of field both read a depth the splats write themselves with no proxy geometry, and test/scene-depth-read, which checks the read against boxes at known distances. Co-Authored-By: Claude Opus 5 --- .../depth-effects.controls.jsx | 160 +++++++ .../depth-effects.example.mjs | 414 +++++++++++++++++ .../test/scene-depth-read.example.mjs | 177 +++++++ ...aussian-splatting_depth-effects_large.webp | Bin 0 -> 7216 bytes ...aussian-splatting_depth-effects_small.webp | Bin 0 -> 726 bytes .../test_scene-depth-read_large.webp | Bin 0 -> 496 bytes .../test_scene-depth-read_small.webp | Bin 0 -> 226 bytes .../render-passes/frame-pass-camera-frame.js | 34 +- .../render-passes/render-pass-prepass.js | 8 +- src/framework/graphics/scene-depth-reader.js | 438 ++++++++++++++++++ src/index.js | 1 + src/scene/camera-shader-params.js | 24 +- src/scene/camera.js | 34 ++ src/scene/graphics/frame-pass-depth-grab.js | 3 +- src/scene/renderer/render-pass-forward.js | 27 +- .../glsl/chunks/common/frag/scene-textures.js | 15 +- .../glsl/chunks/common/frag/screenDepth.js | 6 + .../glsl/chunks/gsplat/frag/gsplat.js | 6 +- .../render-pass/frag/scene-depth-read.js | 33 ++ src/scene/shader-lib/shader-utils.js | 15 +- .../wgsl/chunks/common/frag/scene-textures.js | 15 +- .../wgsl/chunks/common/frag/screenDepth.js | 10 +- .../wgsl/chunks/gsplat/frag/gsplat.js | 6 +- .../render-pass/frag/scene-depth-read.js | 37 ++ 24 files changed, 1408 insertions(+), 55 deletions(-) create mode 100644 examples/src/examples/gaussian-splatting/depth-effects.controls.jsx create mode 100644 examples/src/examples/gaussian-splatting/depth-effects.example.mjs create mode 100644 examples/src/examples/test/scene-depth-read.example.mjs create mode 100644 examples/thumbnails/gaussian-splatting_depth-effects_large.webp create mode 100644 examples/thumbnails/gaussian-splatting_depth-effects_small.webp create mode 100644 examples/thumbnails/test_scene-depth-read_large.webp create mode 100644 examples/thumbnails/test_scene-depth-read_small.webp create mode 100644 src/framework/graphics/scene-depth-reader.js create mode 100644 src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js create mode 100644 src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js diff --git a/examples/src/examples/gaussian-splatting/depth-effects.controls.jsx b/examples/src/examples/gaussian-splatting/depth-effects.controls.jsx new file mode 100644 index 00000000000..0f86c3ad9be --- /dev/null +++ b/examples/src/examples/gaussian-splatting/depth-effects.controls.jsx @@ -0,0 +1,160 @@ +import { + BindingTwoWay, + BooleanInput, + LabelGroup, + Panel, + SelectInput, + SliderInput +} from '@playcanvas/pcui/react'; + +/** + * @import { Observer } from '@playcanvas/observer' + * @import { ReactElement } from 'react' + */ + +/** + * @param {{ observer: Observer }} props - The control panel props. + * @returns {ReactElement} The control panel. + */ +export function Controls({ observer }) { + return ( + <> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs new file mode 100644 index 00000000000..646d92f67da --- /dev/null +++ b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs @@ -0,0 +1,414 @@ +// @config +// +// Volumetric fog and depth of field in a Gaussian Splat scene, with no proxy geometry of any kind - +// both read a scene depth the splats write themselves, see {accent:Scene#gsplat.sceneDepthWrite}. +// +// @flag NO_MINISTATS +// +// @credit +// title: SplatGen_demo_addon +// author: shehab mekky +// source: https://superspl.at/scene/c1e6297e +// license: CC BY 4.0 (http://creativecommons.org/licenses/by/4.0/) + +import { + AppBase, + AppOptions, + Asset, + AssetListLoader, + CameraComponentSystem, + CameraFrame, + Entity, + FILLMODE_FILL_WINDOW, + GSplatComponentSystem, + GSplatHandler, + Keyboard, + LightComponentSystem, + Mouse, + MiniStats, + RESOLUTION_AUTO, + RenderComponentSystem, + SHADOWUPDATE_REALTIME, + SHADOW_PCF3_32F, + SceneDepthReader, + ScriptComponentSystem, + ScriptHandler, + TONEMAP_LINEAR, + TextureHandler, + TouchDevice, + Vec3, + Vec4, + createGraphicsDevice, + platform +} from 'playcanvas'; +import { CameraControls } from 'playcanvas/scripts/esm/camera-controls.mjs'; +import { ProceduralSky } from 'playcanvas/scripts/esm/sky/procedural-sky.mjs'; + +import { data, deviceType } from 'examples/context'; + +/** + * @import { LightComponent } from 'playcanvas' + */ + +const canvas = /** @type {HTMLCanvasElement} */ (document.getElementById('application-canvas')); +window.focus(); + +const gfxOptions = { + deviceTypes: [deviceType], + + // Disable antialiasing as gaussian splats do not benefit from it and it's expensive. Scene + // texture depth also requires it to be off, as resolving a depth attachment would average + // depths across a silhouette. + antialias: false +}; + +const device = await createGraphicsDevice(canvas, gfxOptions); +device.maxPixelRatio = Math.min(window.devicePixelRatio, 2); + +// The scene depth is stored in linear view space units, in R32F where a float render target can be +// blended into and R16F where it cannot - and half float quantizes with distance, stepping by a quarter +// of a unit at 500. So the far clip is generous on the format which can carry it and stays tight on the +// fallback, which costs the far mountains there rather than the precision the fog and the DOF need. +const FAR_CLIP = device.textureFloatBlendable ? 1000 : 200; + +const createOptions = new AppOptions(); +createOptions.graphicsDevice = device; +createOptions.mouse = new Mouse(document.body); +createOptions.touch = new TouchDevice(document.body); +createOptions.keyboard = new Keyboard(document.body); + +createOptions.componentSystems = [ + RenderComponentSystem, + CameraComponentSystem, + LightComponentSystem, + ScriptComponentSystem, + GSplatComponentSystem +]; +createOptions.resourceHandlers = [TextureHandler, ScriptHandler, GSplatHandler]; + +const app = new AppBase(canvas); +app.init(createOptions); + +app.setCanvasFillMode(FILLMODE_FILL_WINDOW); +app.setCanvasResolution(RESOLUTION_AUTO); + +const resize = () => app.resizeCanvas(); +window.addEventListener('resize', resize); +app.on('destroy', () => { + window.removeEventListener('resize', resize); +}); + +// The capture is Y down and reconstructed at its own scale, so everything with a world space unit +// attached to it is gathered here. +const CAPTURE = { + // A copy of the credited capture with part of the scene - a logo - removed, which is what the + // CC BY 4.0 it is under asks to be noted. The author's page is https://superspl.at/user/shehabmekky + splatUrl: 'https://s3.eu-west-1.amazonaws.com/code.playcanvas.com/examples_data/splatgen_01/editted.sog', + + // a single SOG capture roughly 5 units across, framed from just off its axis with the sun turned + // round behind it so the shafts rake across the frame + cameraPosition: [1.9, 3.66, 7.49], + focusPoint: [0.21, 3.18, -0.22], + moveSpeed: 0.25, + sceneSize: 4, + shadowDistance: 15, + + // The furthest the autofocus is allowed to focus. Where the splats thin out, their depth fades + // toward the far clip, so a focus distance landing inside that fade would put a thin in focus + // contour through an otherwise blurred background - the depth crosses the whole focus range within + // a pixel or two there. A read past this is clamped to it rather than discarded, so looking into + // the distance pulls the focus out to the limit instead of leaving it wherever it happened to be, + // which is what the reticle turning orange-red indicates. The capture sits about 8 units away and + // is 5 across, so this clears it comfortably. + focusMaxDistance: 15, + + // the capture spans about y -1.5 to 1.8, so the density is held constant past its top and only + // thins above that - the fog reaches through the whole of it rather than pooling in the bottom + fogHeightBase: 2.2, + fogHeightFalloff: 0.4 +}; + +const splatAsset = new Asset('splat', 'gsplat', { url: CAPTURE.splatUrl }); + +await new Promise((resolve) => { + new AssetListLoader([splatAsset], app.assets).load(resolve); +}); + +app.start(); + +const miniStats = new MiniStats(app, MiniStats.getDefaultOptions(['gsplats'])); // eslint-disable-line no-unused-vars + +// The sun climbs from the horizon at 06:00 to this elevation at noon +const MAX_SUN_ELEVATION = 65; + +// Initial UI state +data.set('settings', { + // a CameraFrame.debug mode, which displays one of the values the frame generates in place of the + // composed result. 'none' for the composed frame itself. + debug: 'none', + + fog: { + enabled: true, + density: 0.071, + anisotropy: 0.68, + intensity: 2, + maxDistance: 20, + steps: 32, + scale: 0.5 + }, + dof: { + enabled: true, + + // the diameter of the circle the autofocus samples, as a percentage of the view + focusSize: 2, + focusRange: 1, + blurRadius: 4 + }, + sky: { + // late afternoon, so the sun rakes across the capture and the shafts are at their longest + time: 17.8, + + // turns the sky, the sun and its shadows around Y, to orient them over the capture + rotation: 302, + + // the scene is rendered without a tone curve, so this is what holds the brightest parts of the + // capture off the clip - the default 1 works at this sun angle, a lower one as it comes round + exposure: 1 + } +}); + +// ------ Camera ------ +const camera = new Entity('camera'); +camera.addComponent('camera', { + fov: 70, + farClip: FAR_CLIP, + + // the splats are captured in daylight and need no tone curve on top of that + toneMapping: TONEMAP_LINEAR +}); +app.root.addChild(camera); + +camera.addComponent('script'); + +// orbit, pan and fly all enabled - CameraControls starts in orbit and switches to fly on the fly input +const cc = /** @type {CameraControls} */ (/** @type {any} */ (camera.script).create(CameraControls)); + +// ------ Sun ------ +// A single directional light kept in sync with the procedural sky below, so the time of day drives +// its direction, color and intensity at once. The splats cast into its shadow map, which is what +// carves the shafts out of the fog. +const sun = new Entity('sun'); +sun.addComponent('light', { + type: 'directional', + + // captured by the procedural sky as the daytime peak, then faded across the day / night cycle + intensity: 6, + castShadows: true, + shadowType: SHADOW_PCF3_32F, + shadowResolution: 2048, + + // gaussian splats do not cast into cascaded shadow maps, and the fog needs their shadows to have + // any shafts in it, so a single cascade is used + numCascades: 1, + shadowBias: 0.3, + normalOffsetBias: 0.2, + + // the sun moves with the time of day, so the shadow map cannot be rendered just once + shadowUpdateMode: SHADOWUPDATE_REALTIME +}); +app.root.addChild(sun); + +// ------ Procedural sky ------ +const sky = new Entity('sky'); +sky.addComponent('script'); +const skyScript = /** @type {ProceduralSky} */ (/** @type {any} */ (sky.script).create(ProceduralSky)); +skyScript.sunLight = sun; +app.root.addChild(sky); + +// ------ Splats ------ +// The capture is Y down, so it needs the usual 180 degree rotation around X to stand up in the +// engine's Y up world. +const splat = new Entity('capture'); +splat.addComponent('gsplat', { + asset: splatAsset, + castShadows: true +}); +splat.setLocalEulerAngles(180, 0, 0); +app.root.addChild(splat); + +app.scene.gsplat.radialSorting = true; +app.scene.gsplat.splatBudget = (platform.mobile ? 1 : 6) * 1000000; + +// The splats are what the fog and the DOF stop at, and only their forward pass can produce that depth - +// the prepass it would otherwise come from renders opaque meshes only, of which this scene has none. +// Whether the device can do it at all is CameraFrame.isSplatSceneDepthSupported. +app.scene.gsplat.sceneDepthWrite = true; + +// ------ Camera frame with volumetric fog ------ +const cameraFrame = new CameraFrame(app, camera.camera); +cameraFrame.rendering.sharpness = 0.5; +cameraFrame.volumetricFog.light = /** @type {LightComponent} */ (sun.light); +cameraFrame.volumetricFog.tint.set(1, 0.93, 0.83); +cameraFrame.volumetricFog.ambientColor.set(0.55, 0.68, 0.9); +cameraFrame.volumetricFog.ambientIntensity = 0.02; + +// the DOF settings which are not exposed as controls, see graphics/depth-of-field for those +cameraFrame.dof.nearBlur = true; +cameraFrame.dof.blurRings = 4; +cameraFrame.dof.blurRingPoints = 5; +cameraFrame.dof.highQuality = true; + +// ------ Autofocus ------ +// The engine has no autofocus, so this reads the scene depth the DOF pass consumes back to the CPU and +// eases the focus distance toward what is in the middle of the screen. SceneDepthReader renders the +// region it is asked for through the same chunk the effects sample the depth with, so this works +// whichever format the depth ended up in, and the read landing a frame or two later is hidden by the +// easing. +const FOCUS_TAU = 0.15; +const FOCUS_SAMPLES = 8; + +const depthReader = new SceneDepthReader(camera.camera); +const focusRect = new Vec4(); +const focusSamples = new Float32Array(FOCUS_SAMPLES * FOCUS_SAMPLES); + +/** Marks the region the autofocus samples. */ +const reticle = document.createElement('div'); +reticle.style.cssText = + 'position:absolute;left:50%;top:50%;border:1px solid rgba(255,255,255,0.9);border-radius:50%;' + + 'box-shadow:0 0 3px rgba(0,0,0,0.8);pointer-events:none;display:none;'; +document.body.appendChild(reticle); + +/** The distance the last read resolved to, or null while nothing has been in focus range. */ +let focusTarget = null; + +/** The eased value driving the DOF pass, null until the first read lands. */ +let smoothedFocus = null; + +/** Whether the last read was further than the focus is allowed to go, and so was clamped to it. */ +let focusClamped = false; + +app.on('destroy', () => { + depthReader.destroy(); + reticle.remove(); +}); + +app.on('update', (/** @type {number} */ dt) => { + const { dof } = data.get('settings'); + + if (!dof.enabled) { + reticle.style.display = 'none'; + return; + } + + // the sampled region as a fraction of the view, and the reticle covering the same area of the screen. + // The region is given in normalized units, which are not square on screen, so its height is corrected + // by the aspect ratio - otherwise the round reticle would not mark what is actually read. + const size = dof.focusSize / 100; + const height = (size * canvas.clientWidth) / canvas.clientHeight; + focusRect.set(0.5 - size / 2, 0.5 - height / 2, size, height); + const cssSize = size * canvas.clientWidth; + reticle.style.width = `${cssSize}px`; + reticle.style.height = `${cssSize}px`; + reticle.style.margin = `${-cssSize / 2}px 0 0 ${-cssSize / 2}px`; + reticle.style.display = focusTarget === null ? 'none' : 'block'; + + // orange-red while the read is further than the focus is allowed to go, so it is clear that the + // focus is sitting at its limit rather than on what the reticle covers + reticle.style.borderColor = focusClamped ? 'rgba(255, 90, 30, 0.95)' : 'rgba(255, 255, 255, 0.9)'; + + // Read every frame, with no regard for whether earlier reads have landed - several can be in flight. + // The median rather than the nearest sample, as captures are full of faint floaters which would + // otherwise grab the focus, and thinly covered pixels read too far - the splat depth is weighted by + // transmittance, so it blends toward the value the depth was cleared to where coverage is partial. + depthReader.read(focusRect, FOCUS_SAMPLES, FOCUS_SAMPLES, focusSamples)?.then((samples) => { + const finite = samples.filter(Number.isFinite).sort(); + if (finite.length) { + const median = finite[finite.length >> 1]; + focusClamped = median > CAPTURE.focusMaxDistance; + focusTarget = Math.min(median, CAPTURE.focusMaxDistance); + } + }); + + if (focusTarget !== null) { + // ease toward the read distance, frame rate independent, snapping on the first one + smoothedFocus = + smoothedFocus === null + ? focusTarget + : smoothedFocus + (focusTarget - smoothedFocus) * (1 - Math.exp(-dt / FOCUS_TAU)); + cameraFrame.dof.focusDistance = smoothedFocus; + cameraFrame.update(); + } +}); + +// Everything with a world space unit attached to it comes from CAPTURE +camera.setLocalPosition(.../** @type {[number, number, number]} */ (CAPTURE.cameraPosition)); +Object.assign(cc, { + sceneSize: CAPTURE.sceneSize, + moveSpeed: CAPTURE.moveSpeed, + moveFastSpeed: CAPTURE.moveSpeed * 4, + focusPoint: new Vec3(.../** @type {[number, number, number]} */ (CAPTURE.focusPoint)) +}); +sun.light.shadowDistance = CAPTURE.shadowDistance; + +cameraFrame.volumetricFog.heightBase = CAPTURE.fogHeightBase; +cameraFrame.volumetricFog.heightFalloff = CAPTURE.fogHeightFalloff; + +const applySettings = () => { + const { debug, dof, fog, sky: skySettings } = data.get('settings'); + + cameraFrame.debug = debug === 'none' ? null : debug; + + cameraFrame.volumetricFog.enabled = fog.enabled; + cameraFrame.volumetricFog.density = fog.density; + cameraFrame.volumetricFog.anisotropy = fog.anisotropy; + cameraFrame.volumetricFog.intensity = fog.intensity; + cameraFrame.volumetricFog.maxDistance = fog.maxDistance; + cameraFrame.volumetricFog.steps = fog.steps; + cameraFrame.volumetricFog.scale = fog.scale; + + // the autofocus drives the focus distance itself, see above + cameraFrame.dof.enabled = dof.enabled; + cameraFrame.dof.focusRange = dof.focusRange; + cameraFrame.dof.blurRadius = dof.blurRadius; + + cameraFrame.update(); + + skyScript.rotation = skySettings.rotation; + app.scene.exposure = skySettings.exposure; +}; + +applySettings(); +data.on('*:set', () => applySettings()); + +// ------ Time of day ------ +// The sun sweeps 15 degrees of azimuth per hour, so 06:00 is due east, noon due south and 18:00 +// due west, and its elevation follows a sine that puts sunrise and sunset at 06:00 and 18:00. +app.on('update', () => { + const hour = data.get('settings.sky.time'); + skyScript.azimuth = hour * 15; + skyScript.elevation = MAX_SUN_ELEVATION * Math.sin(((hour - 6) / 12) * Math.PI); +}); + +// ------ Hold the loading screen until the capture is on screen ------ +// The examples loading screen is dismissed once this module finishes evaluating, so waiting here for the +// first frame with nothing left to load keeps it up until the splats are being rendered - instead of +// handing over an empty sky and popping them in a moment later. Rendering runs during the wait, and +// everything above is already set up, so the first frame handed over is a composed one. The timeout is +// only there so that a capture which never finishes loading cannot leave the example on the loading +// screen for good. +await new Promise((resolve) => { + const timer = setTimeout(resolve, 20000); + const onFrameReady = ( + /** @type {any} */ cam, + /** @type {any} */ layer, + /** @type {boolean} */ ready, + /** @type {number} */ loadingCount + ) => { + if (ready && loadingCount === 0) { + app.systems.gsplat.off('frame:ready', onFrameReady); + clearTimeout(timer); + resolve(); + } + }; + app.systems.gsplat.on('frame:ready', onFrameReady); +}); diff --git a/examples/src/examples/test/scene-depth-read.example.mjs b/examples/src/examples/test/scene-depth-read.example.mjs new file mode 100644 index 00000000000..01f3919ad04 --- /dev/null +++ b/examples/src/examples/test/scene-depth-read.example.mjs @@ -0,0 +1,177 @@ +// @config +// +// Functional test of {accent:SceneDepthReader}. Three boxes stand at known distances from the camera, +// the depth of each is read back over a patch of samples centred on it, and a fourth patch is read from +// the empty space above them. The samples are compared against what the placement says they should be. +// The camera requests the scene depth itself, so what is read is the depth the grab pass produces - the +// non-linear encoding, which is the case a CPU side decode cannot handle at all. +// +// @flag HIDDEN + +import { + AppBase, + AppOptions, + CameraComponentSystem, + Color, + Entity, + FILLMODE_FILL_WINDOW, + LightComponentSystem, + PROJECTION_ORTHOGRAPHIC, + RESOLUTION_AUTO, + RenderComponentSystem, + SceneDepthReader, + Vec3, + Vec4, + createGraphicsDevice +} from 'playcanvas'; + +import { deviceType } from 'examples/context'; + +const canvas = /** @type {HTMLCanvasElement} */ (document.getElementById('application-canvas')); +window.focus(); + +const device = await createGraphicsDevice(canvas, { deviceTypes: [deviceType], antialias: false }); + +const createOptions = new AppOptions(); +createOptions.graphicsDevice = device; +createOptions.componentSystems = [RenderComponentSystem, CameraComponentSystem, LightComponentSystem]; + +const app = new AppBase(canvas); +app.init(createOptions); +app.setCanvasFillMode(FILLMODE_FILL_WINDOW); +app.setCanvasResolution(RESOLUTION_AUTO); + +const resize = () => app.resizeCanvas(); +window.addEventListener('resize', resize); +app.on('destroy', () => window.removeEventListener('resize', resize)); + +app.start(); + +// ------ Three boxes, at 10, 15 and 20 units from the camera ------ +const DISTANCES = [10, 15, 20]; +const BOX_SIZE = 2; + +DISTANCES.forEach((distance, i) => { + const box = new Entity(`box-${i}`); + box.addComponent('render', { type: 'box' }); + box.setLocalScale(BOX_SIZE, BOX_SIZE, BOX_SIZE); + + // spread across the view, with the front face at the distance being tested + box.setLocalPosition((i - 1) * 3, 0, -distance - BOX_SIZE / 2); + app.root.addChild(box); +}); + +const light = new Entity('light'); +light.addComponent('light', { type: 'directional' }); +light.setLocalEulerAngles(45, 30, 0); +app.root.addChild(light); + +const camera = new Entity('camera'); +camera.addComponent('camera', { + clearColor: new Color(0.1, 0.1, 0.15), + nearClip: 0.1, + farClip: 100, + + // orthographic, so a sample's depth is its distance along the view direction wherever the box sits + // on screen, which makes the expected values exact + projection: PROJECTION_ORTHOGRAPHIC, + orthoHeight: 6 +}); +app.root.addChild(camera); + +// nothing else here consumes the depth, so it is requested outright - which is what puts the grab pass +// in the frame +camera.camera.requestSceneDepthMap(true); + +// ------ Read the depth of each box, and of the empty space above them ------ +const reader = new SceneDepthReader(camera.camera); + +// how much of the view a box covers depends on the aspect ratio, so each region is derived from the +// projection rather than assumed - a small patch of samples centred on the box, well inside its edges +const SAMPLES = 4; +const PATCH = 0.02; +const boxRegions = DISTANCES.map(() => new Vec4()); +const emptyRegion = new Vec4(); +const worldPoint = new Vec3(); +const screenPoint = new Vec3(); + +/** + * Centres a normalized region on a world position. + * + * @param {Vec4} region - The region to place. + * @param {number} x - The world x to centre it on. + * @param {number} y - The world y to centre it on. + * @param {number} z - The world z to centre it on. + */ +const placeRegion = (region, x, y, z) => { + camera.camera.worldToScreen(worldPoint.set(x, y, z), screenPoint); + const { width, height } = app.graphicsDevice.clientRect; + + // screen coordinates run down from the top, the regions up from the bottom + region.set(screenPoint.x / width - PATCH / 2, 1 - screenPoint.y / height - PATCH / 2, PATCH, PATCH); +}; + +const report = document.createElement('pre'); +report.style.cssText = + 'position:absolute;left:16px;bottom:16px;margin:0;padding:12px;background:rgba(0,0,0,0.7);' + + 'color:#fff;font:13px monospace;white-space:pre;pointer-events:none;'; +document.body.appendChild(report); +app.on('destroy', () => { + reader.destroy(); + report.remove(); +}); + +/** + * Compares the samples against the box placement. + * + * @param {Float32Array[]} results - A patch of samples per box, then the patch above them. + * @returns {string} The result, one line per check. + */ +const check = (results) => { + const lines = []; + let passed = true; + + // the patch sits entirely on the box, so every sample in it is that box's distance + DISTANCES.forEach((distance, i) => { + const samples = Array.from(results[i]); + const ok = samples.every((value) => Math.abs(value - distance) < 0.5); + const range = `${Math.min(...samples).toFixed(2)} to ${Math.max(...samples).toFixed(2)}`; + passed = passed && ok; + lines.push(`box at ${distance}: samples ${range} ${ok ? 'ok' : 'FAILED'}`); + }); + + // and nothing above them, so every sample there is infinite + const allEmpty = Array.from(results[DISTANCES.length]).every((value) => value === Infinity); + passed = passed && allEmpty; + lines.push(`above the boxes: all infinite ${allEmpty ? 'ok' : 'FAILED'}`); + lines.push('', passed ? 'PASSED' : 'FAILED'); + + return lines.join('\n'); +}; + +let pending = false; +app.on('update', () => { + if (pending) { + return; + } + + // one read per box, plus one clear of them - which also exercises several reads in flight at once + DISTANCES.forEach((distance, i) => { + placeRegion(boxRegions[i], (i - 1) * 3, 0, -distance - BOX_SIZE / 2); + }); + placeRegion(emptyRegion, 0, 4.5, -15); + + const reads = boxRegions.map((region) => reader.read(region, SAMPLES, SAMPLES)); + reads.push(reader.read(emptyRegion, SAMPLES, SAMPLES)); + + if (reads.some((read) => !read)) { + report.textContent = 'waiting for the depth to be rendered'; + return; + } + + pending = true; + Promise.all(reads).then((results) => { + report.textContent = check(results); + pending = false; + }); +}); diff --git a/examples/thumbnails/gaussian-splatting_depth-effects_large.webp b/examples/thumbnails/gaussian-splatting_depth-effects_large.webp new file mode 100644 index 0000000000000000000000000000000000000000..1f76e8c242658672defb527b4d84f7b655cc5ea9 GIT binary patch literal 7216 zcmV-09M9uYNk&E}8~^}UMM6+kP&gnQ8~^}ti2$7eDnJ4706uLrl}IEaqAjGf3c;`q ziDfO6n653R8)HhZPQbF*0QB_%HJP0^|F_$MApiJ%az9e~tBp|AWbg?H;0?Dt~uHVsz?9K!J>Vxa`YbY|NPv_YAR9QUK)cWP4j>Q0=!X1T(WTY3fs?v9l{3a;QLObyd~Lh4eq!O{KInR5T#?U!LV;1nY_KztzG}!sNElHsxe=s znS_zzmfT?(@a9dE6?&MV*0J}mdn#a|@z1uTanv`{0B8T*?#`itMhNlZ$eNMudTk5{ zxIU7!n<&;(jL%EdGd2+%O=M3s_lB?Fs#TL1#SrMWbWB#FKgy|p)_$F-N8^j)N8Rci z1nbd|o6+Lv?WX;&3l2yL@zd;NrHAa`3tB-LDyQ4^>*OD}5(D4oQW}86MNEM7VaqfO z1>=mwqvCtuHl>cm_?7%?Fi)N5?0|AUxE3JXhc=Z_}wlz)Y%vD|C;xny%G2$-Ck2k21??fK2AbdTS6mV znB|WtlN5~^_?irCn=E50yeo%D3}ywo_#`%17cLIm^R_524hQ?TE5kDtOtRyo+%$=s z{bRMs%VNjUM}jCFdHi`Xn+uthhmp?Cy;sbolgpui8D|qYTrlbkUDs@tU^tzAjuv*% z0XP`^;4RR}DTI?MolF2!}qnpO6~yTcU^N>=&_0oSlUppc0HuXylogGI`bvgLwmbaWx~km_Vt*M>X|@I$TJiWC6SzA9{h4IU%jc3>`dDVf^ZeQw(E zG)ylNnL?mH~hO{_gf`l)}5( zhB$_DCe9kP*IfIUhI@9cGC$Mzg2xpV;j@40VP-Wo%|Gha2)3W@jKgnQ`m6)Qk-x-A z2Vet#?|8`Zf#f*6GUBJ%=lqHR8U%0!w0Fl1 z-#9J(%x8&@2FBhQvQG0EfiUU`6kM@F?fb=&tcf@j0f`bNgrM zeXz3E!wPnhv<1InKOLK9Ft^bG{IN9*?);w+~B^_8P`z z3s|%p)<1wTRGU!_ZojSbSXB(-X^{uupoVoHyAC--ukqQOgcWoN?^uy%rW^f@l{HP9 zv5oK?GP~mG94J@|gspC{AG!=EFM4+{?@EX898=iF&}@^+I4w882XF{E&Z->7tirhyhjRx;|iqch}{hM(UDy-+J@J> zGaTl>EIH{0$FYKnA4mL$L!O!IHzSh``0Y2deQUDw3eVSP8aJF#!DoUMoSU>?Pnm7d z)T(~q68)Gr#}0p0p`heRGY5xYKgFw93JTI6<)ama&uX)j7lMP&4}h@0=)^373q*wL z+m_Esi|z@>+@4sV`vhITpwUfUmi11LExx+v&-+F?2zjKPR2HKS!x&@cpyZ2d8+5xD zbdu5V%uCsV`Mh|rLFc%U-t*Rr72Zy^_>Ic|y_84H54LlMnP0ls7?VmtH^1)kawih? zp?+>axiSxt;o=I)a2Uji!&2kuX@H-K-ycUNp`4>U7WvD$H#XIWwbH$WIF<_BYd`5Z`E+078$3JN)-f$>q+fa56)G0 zh?{K4aJ_WG1`rwQ)FJB4{Jy5M!Ti+IEd|1GyrhgXrBa!0Rqc(W+2x2ezoge_3dulH z;7hrw0%1+^R7$5Gm#8ajxDzUBpp3m7&b+W?Gs3uNRkaAw@9p1_;hV%07Brw@jFw_L z|7W+)VO#<6v#$>rT|_x-VzkM*pspZx>4Y;z5?QOV1bo2!UE|K#-5~qLOk@!+6VN2mLp;#{az_9g_9G_h5wB zsJDJzC?zfKjgInGBqE6g4PCj&e!lj6zlDcFfloI8i3GKuBI`g(r+Zg}PM|?lHwNDP zkMYE$((4W)(d4<4wZk4?gb0M=ZR?~3T(d@%whF&v14_Xsy{`5w7lXfEtTjnv&R>OCwp;`DS1sa^hKEn-s1o9q*J-5Dq3`qbY2u?21W;aGDm1)2tx}CIhR8+f zt8AVbJ>dIqQRqi_{snK@7}ZWX8Rm>-?5vY_P<4g$9og;kl0uZ@Hx~#Lw)zJV+zaZP zi1?Tly0-WHHn3QWt>tsux8VaJWQl%kn5D-R`*vT4eV+VVk6ix532L}9hT|5P26aiw zCkzZ%!cW)wnW|k(BHxaz3&gC~hy;6R_x6;MbUjk$)PCKVunkPjOH{|hU%P3V&fu}_u_oVFB^mC|Q&^3T|80^^76VUa6@$%NwrWecr2Y5P zvk9(?-w0cUK;)C{EVL>z8aQkRkH_CL{$G07*G9n5AZijGruNf=5qMACRHkt-`t08F z595g{eJ#GKx%T2OV-*>=M_;&M(--9r;MkA8Xp~FMtfOAExePd?byFDJo#8-pUX4(% zFG#xZ8+lO0o$C6DzMh@#jci2vINNsfb&h=;*l6^AwiUoU=(CUHleuv>!YuygS-)9l zoL8uPaD?uShO|JaXQ8|6)ypos`KQ}u4<)U%kIqL3_ti3mKk|X2{dTuOi6M6TDeAEq z6@k8gL-rHBPt3*=ad$<*03wfaKK=<M*@@E>$4eWR&Z6eKL9l4_BjJ5L{}YIFPomq?b4py2IAPu zM()PxRckC$PL=D~g@ohwt2#PJ9f>XWvc8?XG3-J0J+#r2?vPG~lZqm1lI|cWh;d&f z{m9yr1ZF*UcU^)efOSZpjRtXbG zcj2ll55nykBO?yY*$E9i7i<9h&Qe^yy5Ip0dzMK%M0VP=M+!}itNY2PaD|12&%zq4 z*&x3YR_An&KPR;(2`_qO+> za*hK}y7L50{nr$t#t?b`zy$pS?&}qmaqb$qw zVhHN3qn7ickKD0n^@5mPp}>}(A(3v?3~6JhX)xOmcrVQJN`L)4LN^VQK740Gkox-F36x|Vhxa58@*k;ah@s_W#XG0oC6+TK6Z}+l0%6+i zQ$Q_uk9~AVSgEacb7W@dH@Yi&srwPMoLGW+QYCF7^vn$-iHd1Ag(D~dDFQDQY==Q& z(e=~hg5M}!%Fk?2!3Wy?Lpn`}Z(DQ$IP+@NUmojM{j4%|fqKX@m51OG-)S&ls7%t< z^jZWUF>f{=_2ENipTW=L`ZoFDFDSafeJr1FaGeckMt+^Pi?`|KhL_z1i?P}kySHjV zl{SBAj9bwiE50fPps#8Q-GL=r09~*_Vx9Jf@8-x zV~z_P>jaMPdJ8rt6X$#@bHbW7g3W#jj*y#M43c0dHPsJ?1E+waPEk<6Q_JUwvY4LZ z`7t6BO3P_acD;4C^vsV5x7n)^=6$+n8p=7l=7lvcA2gMwRUtCWfePgh(S{S$Rq(Ly z2T2~>itTemYJY}Y0!UhWa!Ny>3MNa#x$VW4Cwuefn`p$+v}j5Yx6j``b?-j6$bWnqS@6@;#Tj7_$)w+3Mtly|+2oniS1@ z7j$s&KuQU?0RVkW3q1N)?AA-dn|RBca<#Ilz(1uXvD&2J&!<&%*gViPVX|VNwMTC6 zLsQx`t#Qb#=asl&n{yIg-4E3Lf%AxgP|~xztGL5yCWiH--0UA?B=s8DviH$rl;tUF^YnO?)Fqa6>KhD@Rh=eNA7T>|o5 zJe=fb-N_CB4^D2&HUIEi)fbV6FZ##9jC30lC>%1feuML7ef~t81h+9H>R>kr`(8y7(1Y`ngv;V<_ItdX zSGPtt0WvuxXvLEC*_AdEl~n9+whtVdbDM_RHnm3c3?UOw{RrAbw$?koH!t3HXf2MY z>ZSq!Jk@<&#Pv#D^OJvUh~OM<#&~MG`=V1SDazqbakr z#D62%KaxehO`VrZFf~q!o@PKAxw`T*N_GdvXNL0js4IvF>rGepDuhg1>|cLmUFw&b zKu%+0hSX->ng;&=vOis8cCns>-9!PNOv`)oH^(fjEI@k6lt6Xhy{WBuMN>6whwF^J z39p}L{2v!Q{UQYp@t%Q83v?Xiz6OgM0&oz~i@mF$7*2$kSW?Se*d|W1OwtFyJ26LV zQR3A6V$n8^qVIAi;G7{Dq?ZBHgi-&ksog#I9q|DkQl1Ycotp~1VbmfJ3hJTMW83-L z>hJ!_(Bc$*I;RyL3{*WId)R5 zW!F>Foad;ARE|D^`yK#FbY;Lo%;@6_is*d%ClUVXt~&QK!ei_9_cIrx3R0iJK^db$ z!#S(Mf?5EvA8`?~bzPD&>c;B{(Aj0_ZhE2gf-N7D6qkoLi}brzyfp1E z=d)6;Z`2i?+%MfC<9-7Jp#+th&bGPP(_Y! z&gLw}o@AEqCI4Ima;l}2ZS&L8$Xq5Y+FQQxVnQW2oXD3BL!i2J2^1roK7E208 zhOd!RNLtuYkfac985k{;b~>q5&-Pbf)COO7 zr!cKKBkt=SVBRSo!#tBKOO&RwMDC_cN;F}_jt!5|ciAhBGa7~MDXOJji;7L7J$8uQ zx&!?UuZQtm;4yK7a`Rs*7sJD zii?&CQN8y5=`ZLO6#1~>NQSKfvPuI2g}wpY;}I<1D@PK3Y}9%YUGcw6{0<(Ls_l#) zANecSGDB=nluB;O^2Zda#VEU|Q({V@Ia4S}PpeGg(n5d@`jf0mpnK6TtAaKb2T#&< zegKp4u8h)s&6sS4q&VIgFRYYW@&awm0G|B@-@-=ij#xK|J~;(f033DncEZG8`~AZ% z+m=Bee1wrN`g}vGL}4+pnd6Ey zU08v#+16RJb-Vd8VX~5+!rAaugoShoa=1*z9{=Ke5yw~--QG0ZbF{u~l5=tom{WOz zb=SphyI*TM--~AB=<=)Y#U>BG90KgMtxaapsh50Js7b%#o4bnetnppG=H4>U$E$vsYD0mDtdTQi_9UlJ91|D7tIGgY&7{_(;5+e z2Kv1Ala4tH8>mI`AAqI01ZYNv+MJ3unTmA*L|sQq=vIhjE;xKbl{Rqz8N2S5*%Hke zSU%7{%%%l(s44ys>dj$R{7w>e@u&1Xbtco>Z*h6Z92(WC$p4|H5dIT@GLjptS1{k% zPH2QrS(ROfQ2K`J8R6~JTa2&K9{k{P;f&b!S^Kk>qonBfpOR53fn5+Vn~7zV3K5^7 zhb9bwVh+i*Y7fE=vLcAp38h7UPA&muwiB9;!(I%1;W8bm*i&x`h;|a_Mw}T@A(5|< z>d|ML2)8Jd?`kaGcAM@Z0mT+wj~(-CUQKqJ9s+|)y=Z^=0fTQtf2aBrIB>(;MUFjn zFxj0p!KP(ImN&q5%h4hvmy>1)St~niOaRn$3P!bXJ90OK_LAZQApig#p`dGwUSON{ zo>HV6g2%nU<)~3*42FHB#3ARM^S;U{~}FA+B%H z0xvc_SzLD(es0han+gmYN}6j6?2jxh`x{~FPmr660&$7rud)^s^o5Bm>Z&<;y~=AA zc+)B??*5*E7CBtR!pM%Awc*c-)TwR4lCRRwr=O# z!2@|v=2GQ2Tn62h&jvtJO;X&QB<(UFTpI6JOH__DNMus*c8A^6Zy#KTQlOMD#Ald~ z3DkDZgP#!g#A({Fk&5U0Ep#|ou4znpOiZz*d*s)(@GjmBMnrZd$-hpHn4*_5wI{wx zPS3K|=D4tWDULIEKp?bnpk(v!UZBUpJ0-KyX4}4*CfNvg)i&TSsI zLpJsyd&_s-xpuTt_bUqXcA_SZhIXSMdLm6&-qR-;Zw3DaY9BX@mQ~co%6MePRxW_b z+mri*^s~MS*cew=s>}l#Mrny-FH!c@yNnRFZvumyT{H=N>p)j)Stc_jYC~pGC`j<< z`9VACpg~avvVo&Q@osRBM5*XCqn#Efe4*jIFSPj0`$x0ROvQ7+EZ5jsA#edZ4SMxF zl4d34S-^x*%R*pqo8J$fe-pE_d8~bqxh251al(BeGKyNGe>6F=k6e%mWnK#1XbC3y zYoT}6bAv*Gv$v4xpaQbyqe0@@V4F;K4)%zXhHYvZ$>d9vBvf0b|qvZ zt%ZHGSa&@pJaH%ptD3GxC?%b0q3=UFA2aP{D3jJs1_A8C9*dHoes~gW yA=yq2medz91m?BLy9qkGxX5G>*z!4QB{`~-6a361*e%kTjEAA&^c=Q3Z~y=iSse!e literal 0 HcmV?d00001 diff --git a/examples/thumbnails/gaussian-splatting_depth-effects_small.webp b/examples/thumbnails/gaussian-splatting_depth-effects_small.webp new file mode 100644 index 0000000000000000000000000000000000000000..82a22cf067b4d464c84f363caf35adcce75c32e0 GIT binary patch literal 726 zcmV;{0xA7cNk&G_0ssJ4MM6+kP&gpM0ssII4gj42DnI}*06uLnl}98aA|WUV$bb!r zVgRQ`?E`2aI=E}_r&+C{?dh#hJ>KAxC!=NuNi73`nt5$f5Gd3O{@Snr2WA|tc_Q2f zLz6{0my1ZoRS?B8*qZ&$Ttv-l#>1nDF`ZY{J+%;#Mr@M1840RHk5Ip3N5 z#)3Lk9@L_*4b>4!$5_&o0k0DAs<3g%S@3lvg_xiBx>|;=8Eu}!cWaj=ZP94lN8n!^ zCIC_+eElvkb2fhm$IJq!O^B7ynP;(!?pFDT?05hj9q(B~MD*Q!Xip5K-ILs#p-Dy$ znBVcIhZ=0Of(CSaL+_GD%2h>F$J8M_`oVelHXk%q5ngs^y)~UpYkwf#r&$E}gF@Q^ z92M|(2^Y>+_oABD$Ii{PBCFi{k!Gkh+6vMA`!#&aXs$#wqgNUyD8C^}qmK@Apu-0R zu9dgzmTO?Sv}+Rl8DU`kz{Pu0!<7`gRNO9|HNAuFGL0?j8CRxCEgaN-2@y>x;61T^ zJi&%_tX=s-rOMwRc`>8#S~P!bUfhne6HU2`a+T&AC3cwefnZ>BKK=iv@X!DQ6!wKC zTB?0Z&(7?Uc=ZRK4n`PRgi{dx0cGe*A03eG=$=J$(bV4-ja$PQ3on9Hc;!1#f|F4t^^hl;Y&!lfq?8hAT1^Bxu( z9#0Veb@rg)g!1Ck``*o5V;*n*^T>quD+0fo^-(KWoN~{@L^!==ReywBcAU};uMe41_hu=A zJi+{df#Ki(m_qB6_a3hw+}Onc0vTW0fBvsu{+0Fio_UiGwA!7kY}+x({bN~t^ob8! zPg)meZOrFhe?nPyZg4Gatbb+`2T literal 0 HcmV?d00001 diff --git a/examples/thumbnails/test_scene-depth-read_small.webp b/examples/thumbnails/test_scene-depth-read_small.webp new file mode 100644 index 0000000000000000000000000000000000000000..c5d11c5299474b6a672dd96219f67736b4e0522e GIT binary patch literal 226 zcmV<803H8QNk&H600012MM6+kP&gpY0001R2LPP`DnI}*06twLjzy!QAsh4nI0eMB zfM`lI1Mma!07DIHlN}VOF$d`7kB+wXUjIsfzcPW;%NB?jsDG`E^1}cC{`z@d6Lwee z1zlt`57J@wfDwQInSQqU9mfOtI+iJMPlbKcXtpk)|C;`nf-gZOG#PLHg(7YqumUvH z7PlQfM4)H?<%NHk^a&N=K&xPk?$Tub-n%&}{ZFaa;KSieM9`89KU);4HxIp}Icnb3 c+C-7}Apl3r(N-%BmoPmgnBYdp)*m1M06H6D7XSbN literal 0 HcmV?d00001 diff --git a/src/extras/render-passes/frame-pass-camera-frame.js b/src/extras/render-passes/frame-pass-camera-frame.js index d29c7f5f49f..6807d7e771d 100644 --- a/src/extras/render-passes/frame-pass-camera-frame.js +++ b/src/extras/render-passes/frame-pass-camera-frame.js @@ -82,14 +82,6 @@ const _defaultOptions = new CameraFrameOptions(); // the formats the scene depth can be rendered to, in the order of preference const _sceneDepthFormats = [PIXELFORMAT_R32F, PIXELFORMAT_R16F]; -// the largest value a half float can store, and so the furthest the scene depth can reach when it falls -// back to that format -const _maxHalfFloat = 65504; - -// how far inside the far clip the scene depth is cleared to, as a fraction of it. Large enough to survive -// being stored as a half float, which steps by around a thousandth of the value it holds. -const _sceneDepthClearEpsilon = 1e-3; - /** * Render pass implementation of a common camera frame rendering with integrated post-processing * effects. @@ -217,6 +209,7 @@ class FramePassCameraFrame extends FramePass { const { shaderParams } = this.cameraComponent; shaderParams.sceneDepthMapLinear = false; shaderParams.sceneDepthMapPacked = false; + shaderParams.sceneDepthMapReciprocal = false; } if (this.rtSceneColor) { @@ -364,13 +357,6 @@ class FramePassCameraFrame extends FramePass { return 'this camera does not clear the whole render target, and the clear it uses instead would also clear the scene depth'; } - // the depth is stored in linear view space units, so the far clip has to fit in the format. Half - // float stops being able to represent it at all beyond 65504, and steps by tens of units well - // before that, so the effects consuming it need the far clip considerably lower still. - if (this.sceneDepthFormat === PIXELFORMAT_R16F && this.cameraComponent.camera.farClip > _maxHalfFloat) { - return `the far clip of this camera is too far for a half float scene depth - keep it below ${_maxHalfFloat}, and well below that for the depth to stay precise enough`; - } - // The depth prepass, which this camera needs as well, publishes its depth to the same uniform // as the scene textures do, and so the two have to store it the same way - the shaders sampling // it are generated once, from a single declaration of the encoding. Where a float texture cannot @@ -564,6 +550,10 @@ class FramePassCameraFrame extends FramePass { shaderParams.sceneDepthMapLinear = true; shaderParams.sceneDepthMapPacked = false; + // the scene pass accumulates an average of the reciprocals, as the blended splats + // contribute to it - unlike the prepass, which writes the depth outright + shaderParams.sceneDepthMapReciprocal = true; + // the passes blending into the scene color after the scene pass sample the scene depth, so // they cannot render to the render target it is attached to this.rtSceneColor = new RenderTarget({ @@ -801,7 +791,7 @@ class FramePassCameraFrame extends FramePass { // expose an attachment of the render target the remaining passes still render into, which the // materials they render could then sample - reading a texture attached to the render target // being rendered into is not allowed. - (this.scenePassTransparent ?? this.scenePass).publishSceneTextures = true; + (this.scenePassTransparent ?? this.scenePass).sceneTexturesCamera = this.cameraComponent; // Without a prepass nothing has published the scene depth by the time the scene renders, so the // first pass clears the uniform it is published to. This has to happen as the pass renders rather @@ -938,12 +928,14 @@ class FramePassCameraFrame extends FramePass { // texture with the scene render target, which the scene pass resizes this.rtSceneColor.resize(this.rt.width, this.rt.height); - // pixels no geometry covers read as the far clip, nudged just inside it so that a consumer - // testing for the far plane is not caught by rounding. Only the first pass rendering to the - // scene render target clears it - the one rendering the transparent layers after the grab - // pass blends into what the first one accumulated. + // cleared to the reciprocal of the far clip, which makes the background a surface at + // that distance taking part in the average the blended geometry accumulates - whatever + // coverage the splats leave over falls to it. That keeps a pixel a splat covers only + // faintly reporting close to the background rather than the splat's own distance. Only the + // first pass rendering to the scene render target clears it - the one rendering the + // transparent layers after the grab pass blends into what the first one accumulated. const clearValue = this._sceneDepthClearValue; - clearValue.r = this.cameraComponent.camera.farClip * (1 - _sceneDepthClearEpsilon); + clearValue.r = 1 / this.cameraComponent.camera.farClip; this.scenePass.setClearColor(clearValue, this.sceneDepthSlot); } diff --git a/src/extras/render-passes/render-pass-prepass.js b/src/extras/render-passes/render-pass-prepass.js index a44a4244541..4054bcc66e7 100644 --- a/src/extras/render-passes/render-pass-prepass.js +++ b/src/extras/render-passes/render-pass-prepass.js @@ -55,6 +55,7 @@ class RenderPassPrepass extends RenderPass { super.destroy(); this.camera.shaderParams.sceneDepthMapLinear = false; this.camera.shaderParams.sceneDepthMapPacked = false; + this.camera.shaderParams.sceneDepthMapReciprocal = false; this.renderTarget?.destroy(); this.renderTarget = null; this.linearDepthTexture?.destroy(); @@ -90,6 +91,9 @@ class RenderPassPrepass extends RenderPass { shaderParams.sceneDepthMapLinear = true; shaderParams.sceneDepthMapPacked = this.linearDepthFormat === PIXELFORMAT_RGBA8; + // this pass writes the depth outright, rather than the reciprocal average the scene pass accumulates + shaderParams.sceneDepthMapReciprocal = false; + // the WGSL screenDepth chunk implements no packed decode, as WebGPU always supports // rendering to float textures Debug.assert(!(device.isWebGPU && shaderParams.sceneDepthMapPacked)); @@ -98,8 +102,10 @@ class RenderPassPrepass extends RenderPass { } after() { - // Assign the linear depth texture to the uniform + // Assign the linear depth texture to the uniform, and record it on the camera - the uniform is + // global, so anything after this frame wanting this camera's depth needs the camera's own record this.device.scope.resolve(DEPTH_UNIFORM_NAME).setValue(this.linearDepthTexture); + this.camera.camera.publishSceneDepthMap(this.linearDepthTexture, this.device.renderVersion); } execute() { diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js new file mode 100644 index 00000000000..7948ece440f --- /dev/null +++ b/src/framework/graphics/scene-depth-reader.js @@ -0,0 +1,438 @@ +import { Debug } from '../../core/debug.js'; +import { Vec4 } from '../../core/math/vec4.js'; +import { + ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_RGBA8, SEMANTIC_POSITION, + SHADERLANGUAGE_GLSL, SHADERLANGUAGE_WGSL +} from '../../platform/graphics/constants.js'; +import { RenderTarget } from '../../platform/graphics/render-target.js'; +import { Texture } from '../../platform/graphics/texture.js'; +import { EVENT_POSTRENDER } from '../../scene/constants.js'; +import { RenderPassShaderQuad } from '../../scene/graphics/render-pass-shader-quad.js'; +import { ShaderChunks } from '../../scene/shader-lib/shader-chunks.js'; +import { ShaderUtils } from '../../scene/shader-lib/shader-utils.js'; +import glslSceneDepthReadPS from '../../scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js'; +import wgslSceneDepthReadPS from '../../scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js'; + +/** + * @import { CameraComponent } from '../components/camera/component.js' + */ + +/** + * How close to the far clip a sample has to be to count as nothing having been rendered, as a fraction + * of the far clip. Both producers report the far clip itself for a pixel nothing covered - the prepass + * clears to it, and the scene pass clears to its reciprocal, which decodes back to it - so the margin + * is only here to absorb the rounding of that round trip. + */ +const _farLimitFraction = 1 - 1.5e-3; + +/** + * Reads the scene depth of a camera back to the CPU. + * + * A sample is the distance from the camera to the surface at that point, in world units, measured + * along the camera's view direction. + * + * Reads are asynchronous and land a frame or two later. Any number of them may be in flight at once, so + * a read can be issued every frame without waiting for the previous one to finish. + * + * Note that something has to be rendering the depth for there to be anything to read: an effect which + * consumes it, or {@link CameraComponent#requestSceneDepthMap}. + * + * ```javascript + * const reader = new pc.SceneDepthReader(camera.camera); + * const rect = new pc.Vec4(0.45, 0.45, 0.1, 0.1); + * + * app.on('update', () => { + * reader.read(rect, 8, 8)?.then((samples) => { + * const hit = samples.filter(Number.isFinite); + * console.log(hit.length ? Math.min(...hit) : 'nothing in view'); + * }); + * }); + * ``` + * + * @category Graphics + * @alpha + */ +class SceneDepthReader { + /** + * The camera whose depth is read. + * + * @type {CameraComponent} + * @private + */ + camera; + + /** + * Reads requested this frame, rendered and handed to the readback when the camera has finished. + * + * @type {object[]} + * @private + */ + _requests = []; + + /** + * Readback buffers not currently in flight, keyed by their byte length. Each entry holds the bytes + * and a view over them, so a read allocates neither. + * + * @type {Map} + * @private + */ + _buffers = new Map(); + + /** + * Whether the camera rendered a depth the last time it finished a frame. Assumed true until then, + * so that a read issued before the first frame is not turned away. + * + * @type {boolean} + * @private + */ + _depthRendered = true; + + /** @private */ + device; + + /** @private */ + pass; + + /** + * Sized to the largest read so far, and never shrunk. + * + * @private + */ + renderTarget; + + /** @private */ + viewport; + + /** @private */ + _shaderKey; + + /** @private */ + _onPostRender; + + // the uniform scope ids the pass writes, and the scratch values written through them + + /** @private */ + rectId; + + /** @private */ + rectValue; + + /** @private */ + gridId; + + /** @private */ + gridValue; + + /** @private */ + farId; + + /** @private */ + emptyId; + + /** @private */ + depthMapId; + + /** @private */ + cameraParamsId; + + /** @private */ + cameraParams; + + /** + * @param {CameraComponent} camera - The camera whose depth is read. + */ + constructor(camera) { + Debug.assert(camera, 'SceneDepthReader requires a camera component.'); + this.camera = camera; + + const device = this.device = camera.system.app.graphicsDevice; + ShaderChunks.get(device, SHADERLANGUAGE_GLSL).set('sceneDepthReadPS', glslSceneDepthReadPS); + ShaderChunks.get(device, SHADERLANGUAGE_WGSL).set('sceneDepthReadPS', wgslSceneDepthReadPS); + + this.pass = new RenderPassShaderQuad(device); + this.renderTarget = null; + this.viewport = new Vec4(); + this._shaderKey = null; + + const { scope } = device; + this.rectId = scope.resolve('uDepthReadRect'); + this.rectValue = new Float32Array(4); + this.gridId = scope.resolve('uDepthReadGrid'); + this.gridValue = new Float32Array(2); + this.farId = scope.resolve('uDepthReadFar'); + this.emptyId = scope.resolve('uDepthReadEmpty'); + this.depthMapId = scope.resolve('uSceneDepthMap'); + this.cameraParamsId = scope.resolve('camera_params'); + this.cameraParams = new Float32Array(4); + + // the depth of a camera is only current while that camera renders, so the reads are serviced as + // it finishes - by which point its producer has published, and before another camera has run + this._onPostRender = (cameraComponent) => { + if (cameraComponent === this.camera) { + this._process(); + } + }; + camera.system.app.scene.on(EVENT_POSTRENDER, this._onPostRender); + } + + /** + * Requests the depth of a region of the view, as `width * height` samples in row major order. The + * region is point sampled rather than averaged - one sample per cell, taken at its centre - so + * asking for more samples than the region resolves to repeats them. + * + * Samples where nothing was rendered read as `Infinity`. + * + * @param {Vec4} rect - The region of the view to sample, normalized, with its origin in the bottom + * left as {@link CameraComponent#rect}. + * @param {number} width - The number of samples across the region. Not pixels. + * @param {number} height - The number of samples down the region. + * @param {Float32Array} [target] - An array to fill, at least `width * height` long. One is + * allocated when not given. It is filled when the returned promise resolves, so an array must not + * be shared between reads which overlap in time. + * @returns {Promise|null} The samples, in world units, or null when the camera is not + * rendering a scene depth, leaving nothing to read. + */ + read(rect, width, height, target) { + + Debug.assert(width > 0 && height > 0, 'SceneDepthReader#read needs a positive number of samples.'); + const count = width * height; + Debug.assert(!target || target.length >= count, `SceneDepthReader#read needs an array of at least ${count} samples.`); + + if (!this._depthRendered) { + return null; + } + + const request = { + x: rect.x, + y: rect.y, + z: rect.z, + w: rect.w, + width, + height, + target: target ?? new Float32Array(count), + resolve: null + }; + + const promise = new Promise((resolve) => { + request.resolve = resolve; + }); + this._requests.push(request); + return promise; + } + + /** + * Renders and reads back everything requested this frame. + * + * @private + */ + _process() { + + const { camera, device, _requests: requests } = this; + const internal = camera.camera; + + // the camera has just finished, so its record is this frame's if it rendered a depth at all + this._depthRendered = !!internal.sceneDepthMap && internal.sceneDepthMapVersion === device.renderVersion; + + if (requests.length === 0) { + return; + } + + // nothing rendered the depth, so nothing was hit anywhere + if (!this._depthRendered) { + requests.forEach((request) => { + request.target.fill(Infinity, 0, request.width * request.height); + request.resolve(request.target); + }); + requests.length = 0; + return; + } + + this._updateShader(); + + for (let i = 0; i < requests.length; i++) { + this._readRequest(requests[i]); + } + requests.length = 0; + } + + /** + * Renders one request and starts its readback. + * + * @param {object} request - The request. + * @private + */ + _readRequest(request) { + + const { camera } = this; + const { width, height } = request; + const internal = camera.camera; + + this._updateRenderTarget(width, height); + + const { rectValue, gridValue } = this; + rectValue[0] = request.x; + rectValue[1] = request.y; + rectValue[2] = request.z; + rectValue[3] = request.w; + this.rectId.setValue(rectValue); + + gridValue[0] = width; + gridValue[1] = height; + this.gridId.setValue(gridValue); + + this.farId.setValue(internal.farClip * _farLimitFraction); + this.emptyId.setValue(Infinity); + this.cameraParamsId.setValue(internal.fillShaderParams(this.cameraParams)); + + // this camera's own depth, rather than whatever the global uniform happens to hold - with more + // than one camera rendering a depth, the last one to render owns that uniform + this.depthMapId.setValue(internal.sceneDepthMap); + + // only the region the samples land in is rendered, as the target is sized to the largest read + this.pass.viewport = this.viewport.set(0, 0, width, height); + this.pass.render(); + + const count = width * height; + const buffer = this._borrowBuffer(count * 4); + const { target, resolve } = request; + + this.renderTarget.colorBuffer.read(0, 0, width, height, { + renderTarget: this.renderTarget, + data: buffer.bytes + }).then(() => { + + // the chunk packs the float with its high byte in red, which is a big endian read + const view = buffer.view; + for (let i = 0; i < count; i++) { + target[i] = view.getFloat32(i * 4, false); + } + this._returnBuffer(buffer); + resolve(target); + + }).catch(() => { + + // a lost device, or a destroyed reader - report the region as empty rather than leaving the + // promise hanging + target.fill(Infinity, 0, count); + this._returnBuffer(buffer); + resolve(target); + }); + } + + /** + * Builds the shader, or rebuilds it when the encoding of the depth this camera renders has changed. + * + * @private + */ + _updateShader() { + + const defines = new Map(); + const key = ShaderUtils.addScreenDepthChunkDefines(this.camera.shaderParams, defines); + if (this._shaderKey !== key) { + this._shaderKey = key; + this.pass.shader = ShaderUtils.createShader(this.device, { + uniqueName: `SceneDepthRead${key}`, + attributes: { aPosition: SEMANTIC_POSITION }, + vertexChunk: 'quadVS', + fragmentChunk: 'sceneDepthReadPS', + fragmentDefines: defines + }); + } + } + + /** + * Grows the target to hold the requested number of samples. It is never shrunk, so a single large + * read does not cost every following one an allocation. + * + * @param {number} width - Samples across. + * @param {number} height - Samples down. + * @private + */ + _updateRenderTarget(width, height) { + + const current = this.renderTarget; + if (current && current.width >= width && current.height >= height) { + return; + } + + const targetWidth = Math.max(width, current?.width ?? 0); + const targetHeight = Math.max(height, current?.height ?? 0); + this._destroyRenderTarget(); + + const texture = new Texture(this.device, { + name: 'SceneDepthRead', + width: targetWidth, + height: targetHeight, + format: PIXELFORMAT_RGBA8, + mipmaps: false, + minFilter: FILTER_NEAREST, + magFilter: FILTER_NEAREST, + addressU: ADDRESS_CLAMP_TO_EDGE, + addressV: ADDRESS_CLAMP_TO_EDGE + }); + + this.renderTarget = new RenderTarget({ + name: 'SceneDepthRead', + colorBuffer: texture, + depth: false + }); + this.pass.init(this.renderTarget); + } + + /** @private */ + _destroyRenderTarget() { + if (this.renderTarget) { + this.renderTarget.colorBuffer.destroy(); + this.renderTarget.destroy(); + this.renderTarget = null; + } + } + + /** + * @param {number} byteLength - Bytes needed. + * @returns {object} A buffer and a view over it. + * @private + */ + _borrowBuffer(byteLength) { + const pool = this._buffers.get(byteLength); + const buffer = pool?.pop(); + if (buffer) { + return buffer; + } + + const bytes = new Uint8Array(byteLength); + return { bytes, view: new DataView(bytes.buffer) }; + } + + /** + * @param {object} buffer - A buffer no longer in flight. + * @private + */ + _returnBuffer(buffer) { + const byteLength = buffer.bytes.byteLength; + const pool = this._buffers.get(byteLength) ?? []; + pool.push(buffer); + this._buffers.set(byteLength, pool); + } + + /** + * Frees the resources the reader owns and stops reading for this camera. Reads still in flight + * report their region as empty. + */ + destroy() { + + this.camera.system.app.scene.off(EVENT_POSTRENDER, this._onPostRender); + + this._requests.forEach((request) => { + request.target.fill(Infinity, 0, request.width * request.height); + request.resolve(request.target); + }); + this._requests.length = 0; + + this.pass.destroy(); + this._destroyRenderTarget(); + this._buffers.clear(); + } +} + +export { SceneDepthReader }; diff --git a/src/index.js b/src/index.js index 6580c017934..3d397a87514 100644 --- a/src/index.js +++ b/src/index.js @@ -352,6 +352,7 @@ export { BundleRegistry } from './framework/bundle/bundle-registry.js'; // FRAMEWORK / GRAPHICS export { Picker } from './framework/graphics/picker.js'; export { RenderPassPicker } from './framework/graphics/render-pass-picker.js'; +export { SceneDepthReader } from './framework/graphics/scene-depth-reader.js'; // FRAMEWORK / HANDLERS export { basisInitialize } from './framework/handlers/basis.js'; diff --git a/src/scene/camera-shader-params.js b/src/scene/camera-shader-params.js index b0e2477e3a7..79f9beadcce 100644 --- a/src/scene/camera-shader-params.js +++ b/src/scene/camera-shader-params.js @@ -36,6 +36,16 @@ class CameraShaderParams { */ _sceneDepthMapPacked = false; + /** + * True when the linear scene depth map holds a coverage weighted average of the reciprocals of the + * depths, which a consumer inverts to recover the depth. This is how the scene pass accumulates a + * depth the blended gaussian splats contribute to, and a zero marks a pixel nothing was rendered + * to. Only meaningful when {@link CameraShaderParams#sceneDepthMapLinear} is set. + * + * @private + */ + _sceneDepthMapReciprocal = false; + /** * The names of the scene textures the scene pass renders alongside the scene color, in the order * of the color attachments they are rendered to - the name at index i goes to the attachment at @@ -84,7 +94,7 @@ class CameraShaderParams { */ get hash() { if (this._hash === undefined) { - const key = `${this.gammaCorrection}_${this.toneMapping}_${this.srgbRenderTarget}_${this.fog}_${this.ssaoEnabled}_${this.sceneDepthMapLinear}_${this.sceneDepthMapPacked}_${this._sceneTextures.join('-')}`; + const key = `${this.gammaCorrection}_${this.toneMapping}_${this.srgbRenderTarget}_${this.fog}_${this.ssaoEnabled}_${this.sceneDepthMapLinear}_${this.sceneDepthMapPacked}_${this.sceneDepthMapReciprocal}_${this._sceneTextures.join('-')}`; this._hash = hashCode(key); } return this._hash; @@ -104,6 +114,7 @@ class CameraShaderParams { // nested, so that the packed define never appears without the linear one, which is // what the decode in the screenDepth chunk relies on if (this._sceneDepthMapPacked) defines.set('SCENE_DEPTHMAP_PACKED', ''); + if (this._sceneDepthMapReciprocal) defines.set('SCENE_DEPTHMAP_RECIPROCAL', ''); } // each scene texture supplies a pair of defines - one enabling its write, and one giving @@ -205,6 +216,17 @@ class CameraShaderParams { return this._sceneDepthMapPacked; } + set sceneDepthMapReciprocal(value) { + if (this._sceneDepthMapReciprocal !== value) { + this._sceneDepthMapReciprocal = value; + this.markDirty(); + } + } + + get sceneDepthMapReciprocal() { + return this._sceneDepthMapReciprocal; + } + /** * Sets the names of the scene textures the scene pass renders alongside the scene color, for * example `['depth']`. Their order is the order of the color attachments they are rendered to, diff --git a/src/scene/camera.js b/src/scene/camera.js index 17f8d39b444..62f71a80579 100644 --- a/src/scene/camera.js +++ b/src/scene/camera.js @@ -18,6 +18,7 @@ import { CameraShaderParams } from './camera-shader-params.js'; * @import { FramePass } from '../platform/graphics/frame-pass.js' * @import { GraphicsDevice } from '../platform/graphics/graphics-device.js' * @import { RenderTarget } from '../platform/graphics/render-target.js' + * @import { Texture } from '../platform/graphics/texture.js' * @import { FogParams } from './fog-params.js' * @import { Layer } from './layer.js' * @import { RenderView } from './render-view.js' @@ -131,6 +132,25 @@ class Camera { */ beforePasses = []; + /** + * The scene depth texture most recently published for this camera, or null. The uniform it is + * published to is global - the last camera to render owns it - so anything wanting the depth of + * one camera in particular reads it from here instead. See {@link SceneDepthReader}. + * + * @type {Texture|null} + * @ignore + */ + sceneDepthMap = null; + + /** + * The render version {@link Camera#sceneDepthMap} was published in, so a consumer can tell a + * texture rendered this frame from one left over from an earlier one. + * + * @type {number} + * @ignore + */ + sceneDepthMapVersion = -1; + /** @type {number} */ jitter = 0; @@ -229,6 +249,20 @@ class Camera { this.framePasses.length = 0; this.beforePasses.length = 0; + this.sceneDepthMap = null; + } + + /** + * Records the scene depth texture a producer has published for this camera, alongside the render + * version it was published in. + * + * @param {Texture} texture - The texture the depth was rendered to. + * @param {number} renderVersion - The render version it was rendered in. + * @ignore + */ + publishSceneDepthMap(texture, renderVersion) { + this.sceneDepthMap = texture; + this.sceneDepthMapVersion = renderVersion; } /** diff --git a/src/scene/graphics/frame-pass-depth-grab.js b/src/scene/graphics/frame-pass-depth-grab.js index e3b4c689fa2..a9d23820641 100644 --- a/src/scene/graphics/frame-pass-depth-grab.js +++ b/src/scene/graphics/frame-pass-depth-grab.js @@ -113,9 +113,10 @@ class FramePassDepthGrab extends FramePass { this.depthRenderTarget = this.allocateRenderTarget(this.depthRenderTarget, camera.renderTarget, device, format, useDepthBuffer); } - // assign uniform + // assign uniform, and record it on the camera - see RenderPassPrepass for why both const colorBuffer = useDepthBuffer ? this.depthRenderTarget.depthBuffer : this.depthRenderTarget.colorBuffer; device.scope.resolve(_depthUniformName).setValue(colorBuffer); + camera.publishSceneDepthMap(colorBuffer, device.renderVersion); } execute() { diff --git a/src/scene/renderer/render-pass-forward.js b/src/scene/renderer/render-pass-forward.js index 99446eb94f4..96f804e79c9 100644 --- a/src/scene/renderer/render-pass-forward.js +++ b/src/scene/renderer/render-pass-forward.js @@ -6,7 +6,7 @@ import { BlendState } from '../../platform/graphics/blend-state.js'; import { DebugGraphics } from '../../platform/graphics/debug-graphics.js'; import { RenderPass } from '../../platform/graphics/render-pass.js'; import { LayerRenderStep } from './layer-render-step.js'; -import { EVENT_POSTRENDER, EVENT_POSTRENDER_LAYER, EVENT_PRERENDER, EVENT_PRERENDER_LAYER, SHADER_FORWARD, sceneTextureUniformNames } from '../constants.js'; +import { EVENT_POSTRENDER, EVENT_POSTRENDER_LAYER, EVENT_PRERENDER, EVENT_PRERENDER_LAYER, SCENETEXTURE_DEPTH, SHADER_FORWARD, sceneTextureUniformNames } from '../constants.js'; /** * @import { CameraComponent } from '../../framework/components/camera/component.js' @@ -79,15 +79,15 @@ class RenderPassForward extends RenderPass { sceneTextures; /** - * True if this pass publishes the scene textures it rendered when it finishes, making them - * available to the passes which consume them. Only the last pass rendering to the render target - * they are attached to sets this - publishing earlier would expose an attachment of a render target - * the remaining passes still render into, and the materials they render could then sample it, which - * is not allowed. + * The camera whose scene textures this pass publishes when it finishes, making them available to + * the passes which consume them, or null if it publishes none. Only the last pass rendering to the + * render target they are attached to sets this - publishing earlier would expose an attachment of a + * render target the remaining passes still render into, and the materials they render could then + * sample it, which is not allowed. * - * @type {boolean} + * @type {CameraComponent|null} */ - publishSceneTextures = false; + sceneTexturesCamera = null; /** * True if this pass clears the uniforms the scene textures are published to before it renders. Only @@ -293,7 +293,7 @@ class RenderPassForward extends RenderPass { // later, so the scene texture depth, which additionally covers the blended geometry, is what // the consumers sample. const sceneTextures = this.sceneTextures; - if (this.publishSceneTextures && sceneTextures?.length) { + if (this.sceneTexturesCamera && sceneTextures?.length) { const { renderTarget } = this; Debug.assert(renderTarget.colorBufferCount > sceneTextures.length, 'The render target of a pass rendering the scene textures needs an attachment for each of them, in addition to the one holding the scene color.'); @@ -301,7 +301,14 @@ class RenderPassForward extends RenderPass { for (let i = 0; i < sceneTextures.length; i++) { const uniformName = sceneTextureUniformNames[sceneTextures[i]]; Debug.assert(uniformName, `Scene texture '${sceneTextures[i]}' has no uniform to be published under, see sceneTextureUniformNames.`); - this.device.scope.resolve(uniformName).setValue(renderTarget.getColorBuffer(i + 1)); + const texture = renderTarget.getColorBuffer(i + 1); + this.device.scope.resolve(uniformName).setValue(texture); + + // the uniforms are global, so the depth is recorded on the camera as well - that is what + // anything wanting this camera's depth in particular reads, see SceneDepthReader + if (sceneTextures[i] === SCENETEXTURE_DEPTH) { + this.sceneTexturesCamera.camera.publishSceneDepthMap(texture, this.device.renderVersion); + } } } diff --git a/src/scene/shader-lib/glsl/chunks/common/frag/scene-textures.js b/src/scene/shader-lib/glsl/chunks/common/frag/scene-textures.js index a7d7588ac78..04fa3f5c06f 100644 --- a/src/scene/shader-lib/glsl/chunks/common/frag/scene-textures.js +++ b/src/scene/shader-lib/glsl/chunks/common/frag/scene-textures.js @@ -16,14 +16,17 @@ export default /* glsl */` // Writes the fragment's contribution to the scene depth texture. // -// The alpha parameter is the fragment's coverage: opaque geometry passes 1.0, while blended geometry -// (gaussian splats) passes its own alpha. The depth is pre-multiplied by it, so that under the -// premultiplied blending the splats already use, the attachment accumulates a transmittance weighted -// average of the depth - it composites as depth * alpha + depth * (1 - alpha), converging on the -// depth of the surface the splats form. Opaque geometry, passing an alpha of 1.0, simply overwrites. +// The alpha is the fragment's coverage: opaque geometry passes 1.0, blended geometry (gaussian splats) +// its own alpha. What is accumulated is the reciprocal of the depth, pre-multiplied by that coverage, so +// that under the premultiplied blending the splats already use the attachment holds a coverage weighted +// average of the reciprocal - the weights sum to one on their own, with whatever coverage is left over +// falling to the value the attachment was cleared to. Clearing it to the reciprocal of the far clip +// therefore makes the background a surface at that distance, and averaging reciprocals rather than +// depths is what stops it dragging a partly covered pixel most of the way out to it. The read inverts +// the average back into a depth. Opaque geometry, passing 1.0, simply overwrites. void writeSceneTextureDepth(float linearDepth, float alpha) { #ifdef SCENE_TEXTURE_DEPTH - pcFragColor{SCENE_TEXTURE_DEPTH_SLOT} = vec4(linearDepth * alpha, 0.0, 0.0, alpha); + pcFragColor{SCENE_TEXTURE_DEPTH_SLOT} = vec4(alpha / max(linearDepth, 1e-6), 0.0, 0.0, alpha); #endif } diff --git a/src/scene/shader-lib/glsl/chunks/common/frag/screenDepth.js b/src/scene/shader-lib/glsl/chunks/common/frag/screenDepth.js index 3fb81781f32..784adaced86 100644 --- a/src/scene/shader-lib/glsl/chunks/common/frag/screenDepth.js +++ b/src/scene/shader-lib/glsl/chunks/common/frag/screenDepth.js @@ -48,6 +48,12 @@ float getLinearScreenDepth(vec2 uv) { // any filtering to keep the individual bytes intact ivec2 texel = ivec2(uv * vec2(textureSize(uSceneDepthMap, 0))); return uint2float(texelFetch(uSceneDepthMap, texel, 0)); + #elif defined(SCENE_DEPTHMAP_RECIPROCAL) + + // a coverage weighted average of the reciprocals of the depths, inverted back into a + // depth. Zero is a pixel nothing was rendered to, which reads as the far clip. + float recip = texture2D(uSceneDepthMap, uv).r; + return recip > 0.0 ? 1.0 / recip : camera_params.y; #else return texture2D(uSceneDepthMap, uv).r; #endif diff --git a/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js b/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js index 2977a1c3d97..cd4f76d09ca 100644 --- a/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js +++ b/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js @@ -102,9 +102,9 @@ void main(void) { modifySplatColor(gaussianUV, fragColor); gl_FragColor = vec4(fragColor.xyz * fragColor.a, fragColor.a); - // The same premultiplied blending which composites the color accumulates a transmittance - // weighted depth, so the splats gain a depth without being rendered a second time. Dithered - // splats render as opaque, and the fragments which survive the dither have full coverage. + // The same premultiplied blending which composites the color accumulates the scene depth, so + // the splats gain a depth without being rendered a second time. Dithered splats render as + // opaque, and the fragments which survive the dither have full coverage. // Guarded by the define the write function tests internally, as vLinearDepth is only generated // when the depth is written. #ifdef SCENE_TEXTURE_DEPTH diff --git a/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js b/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js new file mode 100644 index 00000000000..e3dc0ee17bb --- /dev/null +++ b/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js @@ -0,0 +1,33 @@ +// Samples the scene depth over a region of the view, one sample per output pixel, and packs each one +// losslessly into RGBA8. Reading RGBA8 back is the one thing every device agrees on, and it is what +// makes the read work regardless of the format the depth itself was rendered to. +export default /* glsl */` + #include "screenDepthPS" + #include "floatAsUintPS" + + // the region of the view to sample, normalized, and the number of samples across and down it + uniform vec4 uDepthReadRect; + uniform vec2 uDepthReadGrid; + + // at or beyond this depth nothing was rendered, and the sample is reported as the value below. + // Infinity arrives as a uniform rather than being written here, as WGSL rejects one written as a + // constant expression, and both languages are kept the same. + uniform float uDepthReadFar; + uniform float uDepthReadEmpty; + + void main() { + + // the centre of this output pixel, as a fraction of the region + vec2 cell = (floor(gl_FragCoord.xy) + 0.5) / uDepthReadGrid; + vec2 uv = uDepthReadRect.xy + cell * uDepthReadRect.zw; + + // snap to the centre of the nearest depth texel. Depth cannot be filtered - the average of a + // near and a far surface unprojects to empty space - and sampling a texel centre is what makes + // the linear sampler of the unpacked formats return that texel exactly. + vec2 size = vec2(textureSize(uSceneDepthMap, 0)); + uv = (floor(uv * size) + 0.5) / size; + + float depth = getLinearScreenDepth(uv); + gl_FragColor = float2uint(depth >= uDepthReadFar ? uDepthReadEmpty : depth); + } +`; diff --git a/src/scene/shader-lib/shader-utils.js b/src/scene/shader-lib/shader-utils.js index 6e60ccda5ee..5ac017c5e91 100644 --- a/src/scene/shader-lib/shader-utils.js +++ b/src/scene/shader-lib/shader-utils.js @@ -201,8 +201,14 @@ class ShaderUtils { * @ignore */ static getScreenDepthChunkKey(cameraShaderParams) { - const { sceneDepthMapLinear, sceneDepthMapPacked } = cameraShaderParams; - return sceneDepthMapLinear ? (sceneDepthMapPacked ? '-linearPacked' : '-linear') : ''; + const { sceneDepthMapLinear, sceneDepthMapPacked, sceneDepthMapReciprocal } = cameraShaderParams; + if (!sceneDepthMapLinear) { + return ''; + } + if (sceneDepthMapPacked) { + return '-linearPacked'; + } + return sceneDepthMapReciprocal ? '-linearReciprocal' : '-linear'; } /** @@ -226,6 +232,11 @@ class ShaderUtils { if (cameraShaderParams.sceneDepthMapPacked) { defines.set('SCENE_DEPTHMAP_PACKED', ''); } + + // likewise nested - the reciprocal decode inverts what the linear map holds + if (cameraShaderParams.sceneDepthMapReciprocal) { + defines.set('SCENE_DEPTHMAP_RECIPROCAL', ''); + } } return ShaderUtils.getScreenDepthChunkKey(cameraShaderParams); diff --git a/src/scene/shader-lib/wgsl/chunks/common/frag/scene-textures.js b/src/scene/shader-lib/wgsl/chunks/common/frag/scene-textures.js index d53eeecae93..86ed0e6c92a 100644 --- a/src/scene/shader-lib/wgsl/chunks/common/frag/scene-textures.js +++ b/src/scene/shader-lib/wgsl/chunks/common/frag/scene-textures.js @@ -16,14 +16,17 @@ export default /* wgsl */` // Writes the fragment's contribution to the scene depth texture. // -// The alpha parameter is the fragment's coverage: opaque geometry passes 1.0, while blended geometry -// (gaussian splats) passes its own alpha. The depth is pre-multiplied by it, so that under the -// premultiplied blending the splats already use, the attachment accumulates a transmittance weighted -// average of the depth - it composites as depth * alpha + depth * (1 - alpha), converging on the -// depth of the surface the splats form. Opaque geometry, passing an alpha of 1.0, simply overwrites. +// The alpha is the fragment's coverage: opaque geometry passes 1.0, blended geometry (gaussian splats) +// its own alpha. What is accumulated is the reciprocal of the depth, pre-multiplied by that coverage, so +// that under the premultiplied blending the splats already use the attachment holds a coverage weighted +// average of the reciprocal - the weights sum to one on their own, with whatever coverage is left over +// falling to the value the attachment was cleared to. Clearing it to the reciprocal of the far clip +// therefore makes the background a surface at that distance, and averaging reciprocals rather than +// depths is what stops it dragging a partly covered pixel most of the way out to it. The read inverts +// the average back into a depth. Opaque geometry, passing 1.0, simply overwrites. fn writeSceneTextureDepth(output: ptr, linearDepth: f32, alpha: f32) { #ifdef SCENE_TEXTURE_DEPTH - (*output).color{SCENE_TEXTURE_DEPTH_SLOT} = vec4f(linearDepth * alpha, 0.0, 0.0, alpha); + (*output).color{SCENE_TEXTURE_DEPTH_SLOT} = vec4f(alpha / max(linearDepth, 1e-6), 0.0, 0.0, alpha); #endif } diff --git a/src/scene/shader-lib/wgsl/chunks/common/frag/screenDepth.js b/src/scene/shader-lib/wgsl/chunks/common/frag/screenDepth.js index 15b4a39e2e0..8b13b4020e1 100644 --- a/src/scene/shader-lib/wgsl/chunks/common/frag/screenDepth.js +++ b/src/scene/shader-lib/wgsl/chunks/common/frag/screenDepth.js @@ -47,7 +47,15 @@ fn getLinearScreenDepth(uv: vec2f) -> f32 { let texel: vec2i = vec2i(uv * vec2f(textureSize)); #ifdef SCENE_DEPTHMAP_LINEAR - return textureLoad(uSceneDepthMap, texel, 0).r; + #ifdef SCENE_DEPTHMAP_RECIPROCAL + + // a coverage weighted average of the reciprocals of the depths, inverted back into a + // depth. Zero is a pixel nothing was rendered to, which reads as the far clip. + let recip: f32 = textureLoad(uSceneDepthMap, texel, 0).r; + return select(uniform.camera_params.y, 1.0 / recip, recip > 0.0); + #else + return textureLoad(uSceneDepthMap, texel, 0).r; + #endif #else return linearizeDepth(textureLoad(uSceneDepthMap, texel, 0).r); #endif diff --git a/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js b/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js index cd7aa091ad9..c17763dc746 100644 --- a/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js +++ b/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js @@ -111,9 +111,9 @@ fn fragmentMain(input: FragmentInput) -> FragmentOutput { modifySplatColor(vec2f(gaussianUV), &fragColor); output.color = vec4f(fragColor.xyz * fragColor.a, fragColor.a); - // The same premultiplied blending which composites the color accumulates a transmittance - // weighted depth, so the splats gain a depth without being rendered a second time. Dithered - // splats render as opaque, and the fragments which survive the dither have full coverage. + // The same premultiplied blending which composites the color accumulates the scene depth, so + // the splats gain a depth without being rendered a second time. Dithered splats render as + // opaque, and the fragments which survive the dither have full coverage. // Guarded by the define the write function tests internally, as vLinearDepth is only generated // when the depth is written. #ifdef SCENE_TEXTURE_DEPTH diff --git a/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js b/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js new file mode 100644 index 00000000000..d7982ca5ae0 --- /dev/null +++ b/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js @@ -0,0 +1,37 @@ +// Samples the scene depth over a region of the view, one sample per output pixel, and packs each one +// losslessly into RGBA8. Reading RGBA8 back is the one thing every device agrees on, and it is what +// makes the read work regardless of the format the depth itself was rendered to. +export default /* wgsl */` + #include "screenDepthPS" + #include "floatAsUintPS" + + // the region of the view to sample, normalized, and the number of samples across and down it + uniform uDepthReadRect: vec4f; + uniform uDepthReadGrid: vec2f; + + // at or beyond this depth nothing was rendered, and the sample is reported as the value below. + // Infinity arrives as a uniform because WGSL rejects one written as a constant expression - a + // bitcast of its bit pattern does not survive const evaluation. + uniform uDepthReadFar: f32; + uniform uDepthReadEmpty: f32; + + @fragment + fn fragmentMain(input: FragmentInput) -> FragmentOutput { + var output: FragmentOutput; + + // the centre of this output pixel, as a fraction of the region + let cell = (floor(pcPosition.xy) + vec2f(0.5)) / uniform.uDepthReadGrid; + let uv = uniform.uDepthReadRect.xy + cell * uniform.uDepthReadRect.zw; + + // snap to the centre of the nearest depth texel. The WGSL depth chunk fetches texels rather + // than sampling, so this only has to land inside the intended texel, but it keeps both + // languages doing the same arithmetic. + let size = vec2f(textureDimensions(uSceneDepthMap, 0)); + let snapped = (floor(uv * size) + vec2f(0.5)) / size; + + let depth = getLinearScreenDepth(snapped); + let value = select(depth, uniform.uDepthReadEmpty, depth >= uniform.uDepthReadFar); + output.color = float2uint(value); + return output; + } +`; From c3576600709edb979456decb0512122230f777ac Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Tue, 1 Sep 2026 15:18:38 +0100 Subject: [PATCH 02/13] updates --- .../gaussian-splatting/depth-effects.example.mjs | 4 ++++ src/framework/graphics/scene-depth-reader.js | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs index 646d92f67da..5953aa5fc0c 100644 --- a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs +++ b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs @@ -317,6 +317,10 @@ app.on('update', (/** @type {number} */ dt) => { reticle.style.borderColor = focusClamped ? 'rgba(255, 90, 30, 0.95)' : 'rgba(255, 255, 255, 0.9)'; // Read every frame, with no regard for whether earlier reads have landed - several can be in flight. + // They all fill the same array, which SceneDepthReader asks callers not to do when reads overlap, so + // a read can resolve against samples a later one has already overwritten. That only ever means the + // focus eases toward a distance a frame or two stale, which the easing below absorbs, and it saves + // allocating an array per frame - a reader whose result mattered exactly would pass its own. // The median rather than the nearest sample, as captures are full of faint floaters which would // otherwise grab the focus, and thinly covered pixels read too far - the splat depth is weighted by // transmittance, so it blends toward the value the depth was cleared to where coverage is partial. diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 7948ece440f..5aef4d7e41d 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -249,6 +249,16 @@ class SceneDepthReader { this._updateShader(); + // sized to the largest request before any of them render, as growing it part way through would + // destroy the texture the readback of an already rendered one is still to be sourced from + let width = 0; + let height = 0; + for (let i = 0; i < requests.length; i++) { + width = Math.max(width, requests[i].width); + height = Math.max(height, requests[i].height); + } + this._updateRenderTarget(width, height); + for (let i = 0; i < requests.length; i++) { this._readRequest(requests[i]); } @@ -267,8 +277,6 @@ class SceneDepthReader { const { width, height } = request; const internal = camera.camera; - this._updateRenderTarget(width, height); - const { rectValue, gridValue } = this; rectValue[0] = request.x; rectValue[1] = request.y; From bf1944ac7a98c1daaeb5b0f7969da2fc25df26df Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Tue, 1 Sep 2026 15:33:47 +0100 Subject: [PATCH 03/13] updates --- .../render-passes/frame-pass-camera-frame.js | 6 ++--- .../render-passes/render-pass-prepass.js | 14 +++++----- src/framework/graphics/scene-depth-reader.js | 27 ++++++++++++------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/extras/render-passes/frame-pass-camera-frame.js b/src/extras/render-passes/frame-pass-camera-frame.js index 6807d7e771d..72e61b45b85 100644 --- a/src/extras/render-passes/frame-pass-camera-frame.js +++ b/src/extras/render-passes/frame-pass-camera-frame.js @@ -543,9 +543,9 @@ class FramePassCameraFrame extends FramePass { if (this.sceneDepthTexture) { - // declare how the depth is stored, for the shaders which sample it. Note that the prepass - // declares the same when both are rendered, as the scene textures are only used on a - // device which can render the depth to a float texture. + + // declare how the depth is stored, for the shaders which sample it. Declared at setup, + // because the post-processing passes resolve these defines when they are constructed. const { shaderParams } = cameraComponent; shaderParams.sceneDepthMapLinear = true; shaderParams.sceneDepthMapPacked = false; diff --git a/src/extras/render-passes/render-pass-prepass.js b/src/extras/render-passes/render-pass-prepass.js index 4054bcc66e7..0dd203fcf43 100644 --- a/src/extras/render-passes/render-pass-prepass.js +++ b/src/extras/render-passes/render-pass-prepass.js @@ -84,20 +84,18 @@ class RenderPassPrepass extends RenderPass { samples: 1 }); + // the WGSL screenDepth chunk implements no packed decode, as WebGPU always supports + // rendering to float textures + Debug.assert(!(device.isWebGPU && this.linearDepthFormat === PIXELFORMAT_RGBA8)); + // declare how this pass stores the depth, so that the shaders sampling it decode what was - // actually written instead of inferring it from the device capabilities - other producers of - // the scene depth map store it in other formats + // actually written instead of inferring it from the device capabilities. Declared at setup, + // because the post-processing passes resolve these defines when they are constructed. const { shaderParams } = this.camera; shaderParams.sceneDepthMapLinear = true; shaderParams.sceneDepthMapPacked = this.linearDepthFormat === PIXELFORMAT_RGBA8; - - // this pass writes the depth outright, rather than the reciprocal average the scene pass accumulates shaderParams.sceneDepthMapReciprocal = false; - // the WGSL screenDepth chunk implements no packed decode, as WebGPU always supports - // rendering to float textures - Debug.assert(!(device.isWebGPU && shaderParams.sceneDepthMapPacked)); - this.init(renderTarget, options); } diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 5aef4d7e41d..a5732a67117 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -1,7 +1,7 @@ import { Debug } from '../../core/debug.js'; import { Vec4 } from '../../core/math/vec4.js'; import { - ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_RGBA8, SEMANTIC_POSITION, + ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_R16F, PIXELFORMAT_RGBA8, SEMANTIC_POSITION, SHADERLANGUAGE_GLSL, SHADERLANGUAGE_WGSL } from '../../platform/graphics/constants.js'; import { RenderTarget } from '../../platform/graphics/render-target.js'; @@ -20,10 +20,14 @@ import wgslSceneDepthReadPS from '../../scene/shader-lib/wgsl/chunks/render-pass /** * How close to the far clip a sample has to be to count as nothing having been rendered, as a fraction * of the far clip. Both producers report the far clip itself for a pixel nothing covered - the prepass - * clears to it, and the scene pass clears to its reciprocal, which decodes back to it - so the margin - * is only here to absorb the rounding of that round trip. + * clears to it, and the scene pass clears to its reciprocal, which decodes back to it - so the margin is + * only here to absorb the rounding of that round trip. A whole float survives it to within a few parts + * in a million, a half float to a few parts in ten thousand, and the margin is picked to match, as a + * surface which happens to lie inside it reads as empty. */ -const _farLimitFraction = 1 - 1.5e-3; +const _farLimitFractionFull = 1 - 1e-5; + +const _farLimitFractionHalf = 1 - 1.5e-3; /** * Reads the scene depth of a camera back to the CPU. @@ -180,7 +184,8 @@ class SceneDepthReader { * region is point sampled rather than averaged - one sample per cell, taken at its centre - so * asking for more samples than the region resolves to repeats them. * - * Samples where nothing was rendered read as `Infinity`. + * Samples where nothing was rendered read as `Infinity`, as do the few which land within a hair of + * the far clip, that being the depth an empty pixel reports. * * @param {Vec4} rect - The region of the view to sample, normalized, with its origin in the bottom * left as {@link CameraComponent#rect}. @@ -189,8 +194,8 @@ class SceneDepthReader { * @param {Float32Array} [target] - An array to fill, at least `width * height` long. One is * allocated when not given. It is filled when the returned promise resolves, so an array must not * be shared between reads which overlap in time. - * @returns {Promise|null} The samples, in world units, or null when the camera is not - * rendering a scene depth, leaving nothing to read. + * @returns {Promise|null} The samples, in world units, or null when the camera is + * disabled or is not rendering a scene depth, leaving nothing to read. */ read(rect, width, height, target) { @@ -198,7 +203,10 @@ class SceneDepthReader { const count = width * height; Debug.assert(!target || target.length >= count, `SceneDepthReader#read needs an array of at least ${count} samples.`); - if (!this._depthRendered) { + // a disabled camera renders nothing, so it would never service the read - the request would sit + // in the queue unanswered rather than the caller being told there is nothing to read + const { camera } = this; + if (!this._depthRendered || !camera.enabled || !camera.entity.enabled) { return null; } @@ -288,7 +296,8 @@ class SceneDepthReader { gridValue[1] = height; this.gridId.setValue(gridValue); - this.farId.setValue(internal.farClip * _farLimitFraction); + const halfFloat = internal.sceneDepthMap.format === PIXELFORMAT_R16F; + this.farId.setValue(internal.farClip * (halfFloat ? _farLimitFractionHalf : _farLimitFractionFull)); this.emptyId.setValue(Infinity); this.cameraParamsId.setValue(internal.fillShaderParams(this.cameraParams)); From 3c8bf21e3dd5c1d6ab36ec5892d7207034ea32e0 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Tue, 1 Sep 2026 15:58:07 +0100 Subject: [PATCH 04/13] added half-float limit --- .../render-passes/frame-pass-camera-frame.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/extras/render-passes/frame-pass-camera-frame.js b/src/extras/render-passes/frame-pass-camera-frame.js index 72e61b45b85..b859fc9e8f9 100644 --- a/src/extras/render-passes/frame-pass-camera-frame.js +++ b/src/extras/render-passes/frame-pass-camera-frame.js @@ -82,6 +82,10 @@ const _defaultOptions = new CameraFrameOptions(); // the formats the scene depth can be rendered to, in the order of preference const _sceneDepthFormats = [PIXELFORMAT_R32F, PIXELFORMAT_R16F]; +// the smallest half float which is still normal, and so the smallest one whose precision is relative to +// its own magnitude. Below it the format is subnormal, where precision falls away in absolute terms +const _minHalfFloatNormal = 6.103515625e-5; + /** * Render pass implementation of a common camera frame rendering with integrated post-processing * effects. @@ -350,6 +354,17 @@ class FramePassCameraFrame extends FramePass { return 'multi-sampling is enabled on the CameraFrame, which the scene depth cannot be rendered with'; } + // The depth is stored as its reciprocal, so what has to be representable is one over the far + // clip rather than the far clip itself. A half float holds that to a few parts in ten thousand + // while it stays normal, but below that it turns subnormal and the precision falls away + // absolutely - far enough out that the reciprocal of the cleared far clip no longer decodes to + // anything near it, and a pixel nothing covered stops being recognisable as empty. + if (this.sceneDepthFormat === PIXELFORMAT_R16F && + 1 / this.cameraComponent.camera.farClip < _minHalfFloatNormal) { + const limit = Math.floor(1 / _minHalfFloatNormal); + return `the far clip of this camera is too far for a half float scene depth - keep it below ${limit}, as the depth is stored as its reciprocal`; + } + // a camera which does not clear the whole render target, or which is not the first one // rendering to it, clears from inside the render pass, and that clear is not attachment aware // - it would also clear the scene textures From d3554e5ab7e7a3691ec697c97d1ab6e7382fee3f Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Tue, 1 Sep 2026 16:04:59 +0100 Subject: [PATCH 05/13] updates --- src/extras/render-passes/frame-pass-camera-frame.js | 11 ++++++----- src/framework/graphics/scene-depth-reader.js | 11 ++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/extras/render-passes/frame-pass-camera-frame.js b/src/extras/render-passes/frame-pass-camera-frame.js index b859fc9e8f9..184c97b8577 100644 --- a/src/extras/render-passes/frame-pass-camera-frame.js +++ b/src/extras/render-passes/frame-pass-camera-frame.js @@ -374,9 +374,10 @@ class FramePassCameraFrame extends FramePass { // The depth prepass, which this camera needs as well, publishes its depth to the same uniform // as the scene textures do, and so the two have to store it the same way - the shaders sampling - // it are generated once, from a single declaration of the encoding. Where a float texture cannot - // be rendered to, the prepass falls back to packing the depth into RGBA8, which the scene - // textures cannot do, as packed values cannot be blended into. + // it are generated once, from a single declaration of the encoding. They do not: the prepass + // writes the depth outright, while the scene textures accumulate an average of its reciprocal, + // which is what lets the blended gaussian splats contribute to it. So the two cannot coexist, + // and the prepass wins - it is the one the materials rendering in the scene pass sample. // // This restriction could be lifted by giving the passes which consume the depth after the scene // pass a uniform of their own, separate from the one the prepass publishes to. Each would then @@ -387,8 +388,8 @@ class FramePassCameraFrame extends FramePass { // that the choice of which uniform to sample would have to be made per consuming pass rather // than per camera, as SSAO applied during shading runs before the scene pass and so has to keep // reading the depth of the prepass. - if (this.needsInSceneDepth(options) && !this.device.textureFloatRenderable) { - return 'the depth prepass this camera also needs stores the depth packed into RGBA8 on this device, which the scene depth cannot be stored as'; + if (this.needsInSceneDepth(options)) { + return 'the depth prepass this camera also needs stores the depth differently, and the two cannot be told apart by the shaders sampling them'; } return null; diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index a5732a67117..5d00225c63f 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -1,8 +1,8 @@ import { Debug } from '../../core/debug.js'; import { Vec4 } from '../../core/math/vec4.js'; import { - ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_R16F, PIXELFORMAT_RGBA8, SEMANTIC_POSITION, - SHADERLANGUAGE_GLSL, SHADERLANGUAGE_WGSL + ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_R16F, PIXELFORMAT_RGBA8, + RENDERTARGET_ORIGIN_BOTTOM, SEMANTIC_POSITION, SHADERLANGUAGE_GLSL, SHADERLANGUAGE_WGSL } from '../../platform/graphics/constants.js'; import { RenderTarget } from '../../platform/graphics/render-target.js'; import { Texture } from '../../platform/graphics/texture.js'; @@ -388,10 +388,15 @@ class SceneDepthReader { addressV: ADDRESS_CLAMP_TO_EDGE }); + // The target is only ever grown, so a read is generally smaller than it. The rendered region + // and the region read back therefore have to be the same rows, which they are not by default: + // the viewport is placed from the bottom on WebGL and from the top on WebGPU, while the readback + // addresses texels natively on both. Asking for the WebGL row order on every API settles it. this.renderTarget = new RenderTarget({ name: 'SceneDepthRead', colorBuffer: texture, - depth: false + depth: false, + origin: RENDERTARGET_ORIGIN_BOTTOM }); this.pass.init(this.renderTarget); } From 1bc2fe681bed08bfd95b3c10e641ec51b5c01f97 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Tue, 1 Sep 2026 18:52:50 +0100 Subject: [PATCH 06/13] updates --- src/framework/graphics/scene-depth-reader.js | 49 +++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 5d00225c63f..3729ac64577 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -91,6 +91,15 @@ class SceneDepthReader { */ _depthRendered = true; + /** + * False once the device has been destroyed, after which nothing further is rendered and there is + * nothing left to read from. + * + * @type {boolean} + * @private + */ + _deviceValid = true; + /** @private */ device; @@ -113,6 +122,9 @@ class SceneDepthReader { /** @private */ _onPostRender; + /** @private */ + _onDeviceDestroy; + // the uniform scope ids the pass writes, and the scratch values written through them /** @private */ @@ -177,6 +189,14 @@ class SceneDepthReader { } }; camera.system.app.scene.on(EVENT_POSTRENDER, this._onPostRender); + + // a destroyed device renders nothing further, so reads are turned away from then on and anything + // already queued is settled rather than left waiting on a frame which will never come + this._onDeviceDestroy = () => { + this._deviceValid = false; + this._settleRequests(); + }; + device.on('destroy', this._onDeviceDestroy); } /** @@ -206,7 +226,7 @@ class SceneDepthReader { // a disabled camera renders nothing, so it would never service the read - the request would sit // in the queue unanswered rather than the caller being told there is nothing to read const { camera } = this; - if (!this._depthRendered || !camera.enabled || !camera.entity.enabled) { + if (!this._deviceValid || !this._depthRendered || !camera.enabled || !camera.entity.enabled) { return null; } @@ -247,11 +267,7 @@ class SceneDepthReader { // nothing rendered the depth, so nothing was hit anywhere if (!this._depthRendered) { - requests.forEach((request) => { - request.target.fill(Infinity, 0, request.width * request.height); - request.resolve(request.target); - }); - requests.length = 0; + this._settleRequests(); return; } @@ -437,6 +453,20 @@ class SceneDepthReader { this._buffers.set(byteLength, pool); } + /** + * Reports every queued read as empty, for the cases where the frame which would have serviced them + * is never going to arrive. + * + * @private + */ + _settleRequests() { + this._requests.forEach((request) => { + request.target.fill(Infinity, 0, request.width * request.height); + request.resolve(request.target); + }); + this._requests.length = 0; + } + /** * Frees the resources the reader owns and stops reading for this camera. Reads still in flight * report their region as empty. @@ -444,12 +474,9 @@ class SceneDepthReader { destroy() { this.camera.system.app.scene.off(EVENT_POSTRENDER, this._onPostRender); + this.device.off('destroy', this._onDeviceDestroy); - this._requests.forEach((request) => { - request.target.fill(Infinity, 0, request.width * request.height); - request.resolve(request.target); - }); - this._requests.length = 0; + this._settleRequests(); this.pass.destroy(); this._destroyRenderTarget(); From 867c1bc8bbe2f3f17655d0bd0349b61d05aee932 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 11:39:58 +0100 Subject: [PATCH 07/13] Settle reads which no frame is going to service A read is answered when the camera it reads finishes a frame, so anything which stops that happening has to settle what is queued instead. The camera component going is one such thing, and it cannot be seen from the component itself - removing it disables the camera internally and destroys the internal camera, but leaves the enabled flag set, so reads would keep being accepted and queued against a camera which will never render again. It is picked up from the component's beforeremove event, which is fired before the component sheds its own listeners. A read which completes after any of that now reports its region as empty rather than unpacking through buffers which have been let go of, and one which fails for a reason of its own is warned about rather than being reported as an empty region with nothing to go on. Co-Authored-By: Claude Opus 5 --- src/framework/graphics/scene-depth-reader.js | 67 ++++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 3729ac64577..33d7c9884dc 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -92,13 +92,13 @@ class SceneDepthReader { _depthRendered = true; /** - * False once the device has been destroyed, after which nothing further is rendered and there is - * nothing left to read from. + * False once this reader, the camera it reads or the device has gone, after which nothing further + * is rendered for it and a read in flight has nothing meaningful left to report. * * @type {boolean} * @private */ - _deviceValid = true; + _valid = true; /** @private */ device; @@ -125,6 +125,9 @@ class SceneDepthReader { /** @private */ _onDeviceDestroy; + /** @private */ + _onCameraRemove; + // the uniform scope ids the pass writes, and the scratch values written through them /** @private */ @@ -190,13 +193,20 @@ class SceneDepthReader { }; camera.system.app.scene.on(EVENT_POSTRENDER, this._onPostRender); - // a destroyed device renders nothing further, so reads are turned away from then on and anything - // already queued is settled rather than left waiting on a frame which will never come + // A destroyed device renders nothing further, and neither does a camera whose component has + // been removed - which notably leaves its enabled flag set, so the check in read cannot see it. + // Either way reads are turned away from then on, and anything already queued is settled rather + // than left waiting on a frame which will never come. this._onDeviceDestroy = () => { - this._deviceValid = false; - this._settleRequests(); + this._invalidate(); }; device.on('destroy', this._onDeviceDestroy); + + // fired before the component sheds its own listeners, so this is still reached + this._onCameraRemove = () => { + this._invalidate(); + }; + camera.on('beforeremove', this._onCameraRemove); } /** @@ -226,7 +236,7 @@ class SceneDepthReader { // a disabled camera renders nothing, so it would never service the read - the request would sit // in the queue unanswered rather than the caller being told there is nothing to read const { camera } = this; - if (!this._deviceValid || !this._depthRendered || !camera.enabled || !camera.entity.enabled) { + if (!this._valid || !this._depthRendered || !camera.enabled || !camera.entity.enabled) { return null; } @@ -334,6 +344,14 @@ class SceneDepthReader { data: buffer.bytes }).then(() => { + // the reader, its camera or the device went while this was in flight, so the bytes mean + // nothing and the buffers they would be unpacked through have been let go of + if (!this._valid) { + target.fill(Infinity, 0, count); + resolve(target); + return; + } + // the chunk packs the float with its high byte in red, which is a big endian read const view = buffer.view; for (let i = 0; i < count; i++) { @@ -342,10 +360,17 @@ class SceneDepthReader { this._returnBuffer(buffer); resolve(target); - }).catch(() => { + }).catch((error) => { + + // The region is reported as empty rather than the promise being left hanging, which is the + // answer a read has when the device or the reader went away under it. A read can also fail + // for a reason of its own, though, and reporting that as an empty region alone would leave + // nothing to go on - so it is warned about, as reads are issued from an update loop and + // rejecting would ask every caller to handle what is usually not theirs to handle. + if (this._valid) { + Debug.warnOnce(`SceneDepthReader read failed: ${error?.message ?? error}`); + } - // a lost device, or a destroyed reader - report the region as empty rather than leaving the - // promise hanging target.fill(Infinity, 0, count); this._returnBuffer(buffer); resolve(target); @@ -453,6 +478,18 @@ class SceneDepthReader { this._buffers.set(byteLength, pool); } + /** + * Marks the reader as having nothing left to read, and settles what is queued. Called when the + * device is destroyed, when the camera component is removed, and when the reader itself is + * destroyed - each of which means no frame will ever service a read again. + * + * @private + */ + _invalidate() { + this._valid = false; + this._settleRequests(); + } + /** * Reports every queued read as empty, for the cases where the frame which would have serviced them * is never going to arrive. @@ -468,15 +505,17 @@ class SceneDepthReader { } /** - * Frees the resources the reader owns and stops reading for this camera. Reads still in flight - * report their region as empty. + * Frees the resources the reader owns and stops reading for this camera. Reads which have not been + * rendered yet report their region as empty, and one already in flight does the same once it + * completes, rather than writing samples read through resources this has let go of. */ destroy() { this.camera.system.app.scene.off(EVENT_POSTRENDER, this._onPostRender); + this.camera.off('beforeremove', this._onCameraRemove); this.device.off('destroy', this._onDeviceDestroy); - this._settleRequests(); + this._invalidate(); this.pass.destroy(); this._destroyRenderTarget(); From cda75f0e01520839de8497533b79cbc7c72fde4b Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 12:17:12 +0100 Subject: [PATCH 08/13] Read back every encoding the scene depth arrives in The test read the depth of the grab pass alone, which is the one encoding this class did not change, so the reciprocal average the scene pass accumulates and the depth the prepass writes outright had no numerical cover at all. Runs once per encoding now, each reached by configuring the camera - the depth requested outright for the grab pass, a camera frame without multi-sampling for the scene textures, and one with it for the prepass, as multi-sampling is what rules the scene textures out. The encoding which turned up is reported alongside the samples and checked against the one the phase set out to reach, so a device falling back to another cannot read as a pass for a case which never ran. Co-Authored-By: Claude Opus 5 --- .../test/scene-depth-read.example.mjs | 122 +++++++++++++++--- src/framework/graphics/scene-depth-reader.js | 6 +- 2 files changed, 112 insertions(+), 16 deletions(-) diff --git a/examples/src/examples/test/scene-depth-read.example.mjs b/examples/src/examples/test/scene-depth-read.example.mjs index 01f3919ad04..2be0197204f 100644 --- a/examples/src/examples/test/scene-depth-read.example.mjs +++ b/examples/src/examples/test/scene-depth-read.example.mjs @@ -3,8 +3,12 @@ // Functional test of {accent:SceneDepthReader}. Three boxes stand at known distances from the camera, // the depth of each is read back over a patch of samples centred on it, and a fourth patch is read from // the empty space above them. The samples are compared against what the placement says they should be. -// The camera requests the scene depth itself, so what is read is the depth the grab pass produces - the -// non-linear encoding, which is the case a CPU side decode cannot handle at all. +// +// Run once for each way the depth can be encoded, as the producers do not agree: the grab pass writes it +// non-linearly, the prepass writes it outright, and the scene pass accumulates an average of its +// reciprocal. Each is reached by configuring the camera rather than asked for directly, so the encoding +// which actually turned up is reported as well - a device which cannot render one falls back to another, +// and a silent fallback would otherwise read as a pass for a case which never ran. // // @flag HIDDEN @@ -12,6 +16,7 @@ import { AppBase, AppOptions, CameraComponentSystem, + CameraFrame, Color, Entity, FILLMODE_FILL_WINDOW, @@ -79,9 +84,69 @@ camera.addComponent('camera', { }); app.root.addChild(camera); -// nothing else here consumes the depth, so it is requested outright - which is what puts the grab pass -// in the frame -camera.camera.requestSceneDepthMap(true); +// A camera frame, left off until the phase which needs it. Depth of field is what makes the frame +// render a depth at all; which encoding that depth ends up in is decided by the rest of the settings. +const cameraFrame = new CameraFrame(app, camera.camera); +cameraFrame.dof.enabled = true; +cameraFrame.enabled = false; +cameraFrame.update(); + +/** + * The encoding the depth currently assigned to the camera is in, named as the shader defines describe it. + * + * @returns {string} The encoding. + */ +const encoding = () => { + const params = camera.camera.shaderParams; + if (!params.sceneDepthMapLinear) { + return 'non-linear'; + } + if (params.sceneDepthMapReciprocal) { + return 'reciprocal'; + } + return params.sceneDepthMapPacked ? 'linear packed' : 'linear'; +}; + +const PHASES = [ + { + // nothing consumes the depth, so it is requested outright - which is what puts the grab pass in + // the frame, and the grab pass keeps the depth buffer's own non-linear encoding + name: 'depth grab', + expect: 'non-linear', + setup: () => { + cameraFrame.enabled = false; + cameraFrame.update(); + camera.camera.requestSceneDepthMap(true); + } + }, + { + // the scene pass carries the depth as an additional attachment, which is the encoding the + // blended gaussian splats can contribute to. Multi-sampling rules it out, so this is the phase + // which leaves it off, and the splat setting is set because that is what the camera frame asks + // for before it will spend the bandwidth on a half float attachment + name: 'scene textures', + expect: 'reciprocal', + setup: () => { + camera.camera.requestSceneDepthMap(false); + app.scene.gsplat.sceneDepthWrite = true; + cameraFrame.rendering.samples = 1; + cameraFrame.enabled = true; + cameraFrame.update(); + } + }, + { + // multi-sampling rules the scene textures out, which leaves the prepass to render the depth - + // stored outright, either as a float or packed into RGBA8 where floats cannot be rendered to + name: 'depth prepass', + expect: 'linear', + setup: () => { + camera.camera.requestSceneDepthMap(false); + cameraFrame.rendering.samples = 4; + cameraFrame.enabled = true; + cameraFrame.update(); + } + } +]; // ------ Read the depth of each box, and of the empty space above them ------ const reader = new SceneDepthReader(camera.camera); @@ -125,11 +190,15 @@ app.on('destroy', () => { * Compares the samples against the box placement. * * @param {Float32Array[]} results - A patch of samples per box, then the patch above them. - * @returns {string} The result, one line per check. + * @param {{ name: string, expect: string }} phase - The phase the samples were read in. + * @returns {{ text: string, passed: boolean }} The result, one line per check, and whether all held. */ -const check = (results) => { +const check = (results, phase) => { const lines = []; - let passed = true; + const found = encoding(); + let passed = found === phase.expect; + + lines.push(`${phase.name}: ${found}${passed ? '' : ` - expected ${phase.expect}, FALLBACK`}`); // the patch sits entirely on the box, so every sample in it is that box's distance DISTANCES.forEach((distance, i) => { @@ -137,21 +206,36 @@ const check = (results) => { const ok = samples.every((value) => Math.abs(value - distance) < 0.5); const range = `${Math.min(...samples).toFixed(2)} to ${Math.max(...samples).toFixed(2)}`; passed = passed && ok; - lines.push(`box at ${distance}: samples ${range} ${ok ? 'ok' : 'FAILED'}`); + lines.push(` box at ${distance}: samples ${range} ${ok ? 'ok' : 'FAILED'}`); }); // and nothing above them, so every sample there is infinite const allEmpty = Array.from(results[DISTANCES.length]).every((value) => value === Infinity); passed = passed && allEmpty; - lines.push(`above the boxes: all infinite ${allEmpty ? 'ok' : 'FAILED'}`); - lines.push('', passed ? 'PASSED' : 'FAILED'); + lines.push(` above the boxes: all infinite ${allEmpty ? 'ok' : 'FAILED'}`); - return lines.join('\n'); + return { text: lines.join('\n'), passed }; }; +// A phase is set up, given a frame for the passes to be rebuilt and the depth to be rendered, and then +// read. Reads return null until there is a depth to read, which is the other half of the waiting. +const SETTLE_FRAMES = 2; +const reports = []; +let phaseIndex = 0; +let framesInPhase = 0; let pending = false; +let allPassed = true; + app.on('update', () => { - if (pending) { + if (pending || phaseIndex >= PHASES.length) { + return; + } + + const phase = PHASES[phaseIndex]; + if (framesInPhase === 0) { + phase.setup(); + } + if (++framesInPhase <= SETTLE_FRAMES) { return; } @@ -165,13 +249,21 @@ app.on('update', () => { reads.push(reader.read(emptyRegion, SAMPLES, SAMPLES)); if (reads.some((read) => !read)) { - report.textContent = 'waiting for the depth to be rendered'; + report.textContent = [...reports, `${phase.name}: waiting for the depth`].join('\n'); return; } pending = true; Promise.all(reads).then((results) => { - report.textContent = check(results); + const { text, passed } = check(results, phase); + reports.push(text); + allPassed = allPassed && passed; + + phaseIndex++; + framesInPhase = 0; pending = false; + + const done = phaseIndex >= PHASES.length; + report.textContent = [...reports, '', done ? (allPassed ? 'PASSED' : 'FAILED') : ''].join('\n'); }); }); diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 33d7c9884dc..b55ce8d668e 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -369,10 +369,14 @@ class SceneDepthReader { // rejecting would ask every caller to handle what is usually not theirs to handle. if (this._valid) { Debug.warnOnce(`SceneDepthReader read failed: ${error?.message ?? error}`); + + // handed back only while the reader is valid - the pool is let go of as it is + // invalidated, and a buffer arriving after that would build it again, which is why the + // path above drops its own the same way + this._returnBuffer(buffer); } target.fill(Infinity, 0, count); - this._returnBuffer(buffer); resolve(target); }); } From f1a0d2c80029a5c72f2aceb8cd097459c64ebcc8 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 12:35:24 +0100 Subject: [PATCH 09/13] Release the quad the read pass renders with, and correct two descriptions The pass has no teardown of its own - FramePass#destroy does nothing and RenderPassShaderQuad does not override it - so destroying the reader left the QuadRender it holds alive, along with what that holds on the GPU. Clearing the shader is what releases it, and leaves the shader itself with the program library which cached it. The scene depth encoding was described as marking an uncovered pixel with a zero. It does not: such a pixel holds the reciprocal of the far clip the map is cleared to, and reads back as the far clip. Zero is only the guard the decode takes before dividing. The far clip in the depth effects example was explained in terms of how a half float quantizes a distance, which is not what is stored there either. What caps it on that format is one over the far clip having to stay a normal half float. Co-Authored-By: Claude Opus 5 --- .../gaussian-splatting/depth-effects.example.mjs | 9 +++++---- src/framework/graphics/scene-depth-reader.js | 3 +++ src/scene/camera-shader-params.js | 5 +++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs index 5953aa5fc0c..4d0252e2a68 100644 --- a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs +++ b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs @@ -65,10 +65,11 @@ const gfxOptions = { const device = await createGraphicsDevice(canvas, gfxOptions); device.maxPixelRatio = Math.min(window.devicePixelRatio, 2); -// The scene depth is stored in linear view space units, in R32F where a float render target can be -// blended into and R16F where it cannot - and half float quantizes with distance, stepping by a quarter -// of a unit at 500. So the far clip is generous on the format which can carry it and stays tight on the -// fallback, which costs the far mountains there rather than the precision the fog and the DOF need. +// The scene depth is stored as the reciprocal of the distance, in the widest float format the device +// can render and blend into - R32F for preference, R16F where that is not available. One over the far +// clip has to stay a normal half float on the fallback, which caps the far clip at 16384 there, so it +// is kept well inside that: generous where the format can carry it, tight where it cannot, which costs +// the far mountains rather than the precision the fog and the DOF need. const FAR_CLIP = device.textureFloatBlendable ? 1000 : 200; const createOptions = new AppOptions(); diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index b55ce8d668e..03210ba5a61 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -521,6 +521,9 @@ class SceneDepthReader { this._invalidate(); + // clearing the shader is what releases the quad the pass renders with - the pass itself has + // no teardown of its own, and the shader stays owned by the program library which cached it + this.pass.shader = null; this.pass.destroy(); this._destroyRenderTarget(); this._buffers.clear(); diff --git a/src/scene/camera-shader-params.js b/src/scene/camera-shader-params.js index 79f9beadcce..9e06fd1b129 100644 --- a/src/scene/camera-shader-params.js +++ b/src/scene/camera-shader-params.js @@ -39,8 +39,9 @@ class CameraShaderParams { /** * True when the linear scene depth map holds a coverage weighted average of the reciprocals of the * depths, which a consumer inverts to recover the depth. This is how the scene pass accumulates a - * depth the blended gaussian splats contribute to, and a zero marks a pixel nothing was rendered - * to. Only meaningful when {@link CameraShaderParams#sceneDepthMapLinear} is set. + * depth the blended gaussian splats contribute to. A pixel nothing was rendered to holds the + * reciprocal of the far clip the map was cleared to, and so reads back as the far clip itself. Only + * meaningful when {@link CameraShaderParams#sceneDepthMapLinear} is set. * * @private */ From 6903801622b7d005705f88031118a2b65bec4464 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 14:04:21 +0100 Subject: [PATCH 10/13] Keep the splats in the scene depth on a half float device A half float carries the reciprocal of a distance well over a camera range of about 0.000015 to 16384, and past the far end of that a distant depth loses accuracy and a pixel nothing covers stops reading as far away as it is. Refusing the scene depth there costs far more than it saves though: a scene of splats and nothing else has no opaque geometry for the prepass to render, so the depth comes out empty and the fog and the depth of field are bounded by nothing at all rather than being slightly out at the far end. So the range is documented where an application decides whether to turn the splat depth on, and where a caller reads the samples back, and the choice of what to do about it is left to the application - which may know its far clip is nowhere near the limit, or would rather leave the effects off on such a device. Nothing watches the clip planes for it either. Which producer renders the depth is settled when the camera frame is built, and rebuilding that recompiles the shaders of every pass which reads the depth - not something to trigger because a near clip was tweened. Co-Authored-By: Claude Opus 5 --- .../render-passes/frame-pass-camera-frame.js | 15 ------------- src/framework/graphics/scene-depth-reader.js | 22 +++++++++++++++++-- src/scene/gsplat-unified/gsplat-params.js | 5 ++++- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/extras/render-passes/frame-pass-camera-frame.js b/src/extras/render-passes/frame-pass-camera-frame.js index 184c97b8577..a9a80411651 100644 --- a/src/extras/render-passes/frame-pass-camera-frame.js +++ b/src/extras/render-passes/frame-pass-camera-frame.js @@ -82,10 +82,6 @@ const _defaultOptions = new CameraFrameOptions(); // the formats the scene depth can be rendered to, in the order of preference const _sceneDepthFormats = [PIXELFORMAT_R32F, PIXELFORMAT_R16F]; -// the smallest half float which is still normal, and so the smallest one whose precision is relative to -// its own magnitude. Below it the format is subnormal, where precision falls away in absolute terms -const _minHalfFloatNormal = 6.103515625e-5; - /** * Render pass implementation of a common camera frame rendering with integrated post-processing * effects. @@ -354,17 +350,6 @@ class FramePassCameraFrame extends FramePass { return 'multi-sampling is enabled on the CameraFrame, which the scene depth cannot be rendered with'; } - // The depth is stored as its reciprocal, so what has to be representable is one over the far - // clip rather than the far clip itself. A half float holds that to a few parts in ten thousand - // while it stays normal, but below that it turns subnormal and the precision falls away - // absolutely - far enough out that the reciprocal of the cleared far clip no longer decodes to - // anything near it, and a pixel nothing covered stops being recognisable as empty. - if (this.sceneDepthFormat === PIXELFORMAT_R16F && - 1 / this.cameraComponent.camera.farClip < _minHalfFloatNormal) { - const limit = Math.floor(1 / _minHalfFloatNormal); - return `the far clip of this camera is too far for a half float scene depth - keep it below ${limit}, as the depth is stored as its reciprocal`; - } - // a camera which does not clear the whole render target, or which is not the first one // rendering to it, clears from inside the render pass, and that clear is not attachment aware // - it would also clear the scene textures diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 03210ba5a61..347e6bf0c87 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -217,6 +217,11 @@ class SceneDepthReader { * Samples where nothing was rendered read as `Infinity`, as do the few which land within a hair of * the far clip, that being the depth an empty pixel reports. * + * Note that on a device which stores the scene depth at a lower precision - see + * {@link GSplatParams#sceneDepthWrite} - a far clip beyond about 16384 leaves an empty pixel + * reporting a large distance rather than `Infinity`, as the two stop being far enough apart to + * tell one from the other. + * * @param {Vec4} rect - The region of the view to sample, normalized, with its origin in the bottom * left as {@link CameraComponent#rect}. * @param {number} width - The number of samples across the region. Not pixels. @@ -339,10 +344,23 @@ class SceneDepthReader { const buffer = this._borrowBuffer(count * 4); const { target, resolve } = request; - this.renderTarget.colorBuffer.read(0, 0, width, height, { + const read = this.renderTarget.colorBuffer.read(0, 0, width, height, { renderTarget: this.renderTarget, data: buffer.bytes - }).then(() => { + }); + + // a backend which implements no readback at all - the null device among them - hands back + // nothing rather than a promise, so the region is reported as empty instead of the request + // being left with no way to be answered + if (!read) { + Debug.warnOnce('SceneDepthReader: this device implements no texture readback, so the depth reads as empty.'); + target.fill(Infinity, 0, count); + this._returnBuffer(buffer); + resolve(target); + return; + } + + read.then(() => { // the reader, its camera or the device went while this was in flight, so the bytes mean // nothing and the buffers they would be unpacked through have been let go of diff --git a/src/scene/gsplat-unified/gsplat-params.js b/src/scene/gsplat-unified/gsplat-params.js index 670e01e7081..8881497c86c 100644 --- a/src/scene/gsplat-unified/gsplat-params.js +++ b/src/scene/gsplat-unified/gsplat-params.js @@ -886,7 +886,10 @@ class GSplatParams { * {@link CameraFrame} - see {@link CameraFrame.isSplatSceneDepthSupported}. * * On some devices enabling this stores the scene depth at a lower precision, which the other - * effects using it share. + * effects using it share. The depth stays accurate over camera clip distances of roughly 0.000015 + * to 16384 there; past the far end of that a distant depth loses accuracy, and the pixels nothing + * covers stop reading as far away as they are. Keep the far clip inside that range on those + * devices, or leave the effects which read the depth off. * * @type {boolean} */ From 64dbfc6503d35358cc091b614a972c68d6dd7ebe Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 14:18:31 +0100 Subject: [PATCH 11/13] Give back the depth map request once, not twice The request the grab phase makes is accumulative, and the phase after it already gives that one back. Giving it back a second time took the count the camera keeps below zero, which it asserts against, and left anything requesting the depth afterwards unable to turn it on. Nothing measured changed, which is why the test went on passing - by that phase the camera frame renders the depth whatever the count says. Co-Authored-By: Claude Opus 5 --- examples/src/examples/test/scene-depth-read.example.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/src/examples/test/scene-depth-read.example.mjs b/examples/src/examples/test/scene-depth-read.example.mjs index 2be0197204f..7ee31887823 100644 --- a/examples/src/examples/test/scene-depth-read.example.mjs +++ b/examples/src/examples/test/scene-depth-read.example.mjs @@ -140,7 +140,8 @@ const PHASES = [ name: 'depth prepass', expect: 'linear', setup: () => { - camera.camera.requestSceneDepthMap(false); + // the request the first phase made was already given back by the one before this, and the + // count it keeps must not go below zero cameraFrame.rendering.samples = 4; cameraFrame.enabled = true; cameraFrame.update(); From a6728080aae311ddb3859d6a300b2a9c8db81a95 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 14:42:24 +0100 Subject: [PATCH 12/13] Read the region asked for, not the one mirrored about the view The region is given with its origin in the bottom left, as the camera's own rect is, while the depth is addressed however the graphics API stores it. The read worked out its own coordinates and skipped the conversion every other image effect makes, so on WebGPU it sampled the region mirrored about the middle of the view. The functional test could not see it: three boxes at one height are mirrored onto themselves, and the patch of empty space above them is mirrored onto empty space below. They now stand at heights of their own, where a mirrored read lands on nothing at all - which fails the check on every one of the three encodings before the conversion is applied, and passes on both backends after. Two smaller ones alongside it. A region with nothing in it read as every sample infinite and was dropped, leaving the focus of the example wherever it happened to be, where the point of the clamp is that looking into the distance pulls it out to the limit. And a fractional number of samples passed the check at the API boundary and then went on to be the dimensions of a texture. Co-Authored-By: Claude Opus 5 --- .../gaussian-splatting/depth-effects.example.mjs | 11 ++++++----- .../src/examples/test/scene-depth-read.example.mjs | 14 ++++++++++++-- src/framework/graphics/scene-depth-reader.js | 3 ++- .../chunks/render-pass/frag/scene-depth-read.js | 6 +++++- .../chunks/render-pass/frag/scene-depth-read.js | 6 +++++- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs index 4d0252e2a68..670b071ff06 100644 --- a/examples/src/examples/gaussian-splatting/depth-effects.example.mjs +++ b/examples/src/examples/gaussian-splatting/depth-effects.example.mjs @@ -326,12 +326,13 @@ app.on('update', (/** @type {number} */ dt) => { // otherwise grab the focus, and thinly covered pixels read too far - the splat depth is weighted by // transmittance, so it blends toward the value the depth was cleared to where coverage is partial. depthReader.read(focusRect, FOCUS_SAMPLES, FOCUS_SAMPLES, focusSamples)?.then((samples) => { + // a region with nothing in it reads as every sample infinite, which is a read of the distance + // like any other and clamps the same way - rather than being dropped, which would leave the + // focus wherever it happened to be const finite = samples.filter(Number.isFinite).sort(); - if (finite.length) { - const median = finite[finite.length >> 1]; - focusClamped = median > CAPTURE.focusMaxDistance; - focusTarget = Math.min(median, CAPTURE.focusMaxDistance); - } + const median = finite.length ? finite[finite.length >> 1] : Infinity; + focusClamped = median > CAPTURE.focusMaxDistance; + focusTarget = Math.min(median, CAPTURE.focusMaxDistance); }); if (focusTarget !== null) { diff --git a/examples/src/examples/test/scene-depth-read.example.mjs b/examples/src/examples/test/scene-depth-read.example.mjs index 7ee31887823..361862ced32 100644 --- a/examples/src/examples/test/scene-depth-read.example.mjs +++ b/examples/src/examples/test/scene-depth-read.example.mjs @@ -56,13 +56,23 @@ app.start(); const DISTANCES = [10, 15, 20]; const BOX_SIZE = 2; +/** + * Where a box stands. Each sits at a height of its own, so that a read which mirrored the region + * vertically would land on empty space rather than back on the same box - which is what a placement + * symmetrical about the centre of the view would hide. + * + * @param {number} i - The index of the box. + * @returns {[number, number, number]} Its world position. + */ +const boxPosition = (i) => [(i - 1) * 3, (i - 1) * 2.5, -DISTANCES[i] - BOX_SIZE / 2]; + DISTANCES.forEach((distance, i) => { const box = new Entity(`box-${i}`); box.addComponent('render', { type: 'box' }); box.setLocalScale(BOX_SIZE, BOX_SIZE, BOX_SIZE); // spread across the view, with the front face at the distance being tested - box.setLocalPosition((i - 1) * 3, 0, -distance - BOX_SIZE / 2); + box.setLocalPosition(...boxPosition(i)); app.root.addChild(box); }); @@ -242,7 +252,7 @@ app.on('update', () => { // one read per box, plus one clear of them - which also exercises several reads in flight at once DISTANCES.forEach((distance, i) => { - placeRegion(boxRegions[i], (i - 1) * 3, 0, -distance - BOX_SIZE / 2); + placeRegion(boxRegions[i], ...boxPosition(i)); }); placeRegion(emptyRegion, 0, 4.5, -15); diff --git a/src/framework/graphics/scene-depth-reader.js b/src/framework/graphics/scene-depth-reader.js index 347e6bf0c87..8ac9f79ecf9 100644 --- a/src/framework/graphics/scene-depth-reader.js +++ b/src/framework/graphics/scene-depth-reader.js @@ -234,7 +234,8 @@ class SceneDepthReader { */ read(rect, width, height, target) { - Debug.assert(width > 0 && height > 0, 'SceneDepthReader#read needs a positive number of samples.'); + Debug.assert(width > 0 && height > 0 && Number.isInteger(width) && Number.isInteger(height), + 'SceneDepthReader#read needs a whole positive number of samples, as they become the dimensions of a texture.'); const count = width * height; Debug.assert(!target || target.length >= count, `SceneDepthReader#read needs an array of at least ${count} samples.`); diff --git a/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js b/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js index e3dc0ee17bb..23db43cfc25 100644 --- a/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js +++ b/src/scene/shader-lib/glsl/chunks/render-pass/frag/scene-depth-read.js @@ -19,7 +19,11 @@ export default /* glsl */` // the centre of this output pixel, as a fraction of the region vec2 cell = (floor(gl_FragCoord.xy) + 0.5) / uDepthReadGrid; - vec2 uv = uDepthReadRect.xy + cell * uDepthReadRect.zw; + + // The region is given with its origin in the bottom left, as the camera's own rect is, while + // the depth is addressed however the API stores it - so the same conversion every other image + // effect makes. Without it the region read is the one mirrored about the middle of the view. + vec2 uv = getImageEffectUV(uDepthReadRect.xy + cell * uDepthReadRect.zw); // snap to the centre of the nearest depth texel. Depth cannot be filtered - the average of a // near and a far surface unprojects to empty space - and sampling a texel centre is what makes diff --git a/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js b/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js index d7982ca5ae0..2ed109c28c3 100644 --- a/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js +++ b/src/scene/shader-lib/wgsl/chunks/render-pass/frag/scene-depth-read.js @@ -21,7 +21,11 @@ export default /* wgsl */` // the centre of this output pixel, as a fraction of the region let cell = (floor(pcPosition.xy) + vec2f(0.5)) / uniform.uDepthReadGrid; - let uv = uniform.uDepthReadRect.xy + cell * uniform.uDepthReadRect.zw; + + // The region is given with its origin in the bottom left, as the camera's own rect is, while + // the depth is addressed however the API stores it - so the same conversion every other image + // effect makes. Without it the region read is the one mirrored about the middle of the view. + let uv = getImageEffectUV(uniform.uDepthReadRect.xy + cell * uniform.uDepthReadRect.zw); // snap to the centre of the nearest depth texel. The WGSL depth chunk fetches texels rather // than sampling, so this only has to land inside the intended texel, but it keeps both From 346a36557c014ac44605406552403090ef44aff0 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Wed, 2 Sep 2026 15:06:11 +0100 Subject: [PATCH 13/13] Record the scene depth on the camera the scene level already has The pass which publishes the scene textures held the camera component to do it, which has the scene level holding an instance from the framework level above it. Every use of it was to reach the camera underneath, which is where the depth is recorded and which the scene level owns, so that is what it holds now and what the camera frame hands it. The other producer already records the depth this way. Co-Authored-By: Claude Opus 5 --- src/extras/render-passes/frame-pass-camera-frame.js | 2 +- src/scene/renderer/render-pass-forward.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/extras/render-passes/frame-pass-camera-frame.js b/src/extras/render-passes/frame-pass-camera-frame.js index a9a80411651..b3bd8c07654 100644 --- a/src/extras/render-passes/frame-pass-camera-frame.js +++ b/src/extras/render-passes/frame-pass-camera-frame.js @@ -792,7 +792,7 @@ class FramePassCameraFrame extends FramePass { // expose an attachment of the render target the remaining passes still render into, which the // materials they render could then sample - reading a texture attached to the render target // being rendered into is not allowed. - (this.scenePassTransparent ?? this.scenePass).sceneTexturesCamera = this.cameraComponent; + (this.scenePassTransparent ?? this.scenePass).sceneTexturesCamera = this.cameraComponent.camera; // Without a prepass nothing has published the scene depth by the time the scene renders, so the // first pass clears the uniform it is published to. This has to happen as the pass renders rather diff --git a/src/scene/renderer/render-pass-forward.js b/src/scene/renderer/render-pass-forward.js index 96f804e79c9..50a4c7807de 100644 --- a/src/scene/renderer/render-pass-forward.js +++ b/src/scene/renderer/render-pass-forward.js @@ -10,6 +10,7 @@ import { EVENT_POSTRENDER, EVENT_POSTRENDER_LAYER, EVENT_PRERENDER, EVENT_PREREN /** * @import { CameraComponent } from '../../framework/components/camera/component.js' + * @import { Camera } from '../camera.js' * @import { LayerComposition } from '../composition/layer-composition.js' * @import { Layer } from '../layer.js' * @import { Renderer } from './renderer.js' @@ -85,7 +86,7 @@ class RenderPassForward extends RenderPass { * render target the remaining passes still render into, and the materials they render could then * sample it, which is not allowed. * - * @type {CameraComponent|null} + * @type {Camera|null} */ sceneTexturesCamera = null; @@ -307,7 +308,7 @@ class RenderPassForward extends RenderPass { // the uniforms are global, so the depth is recorded on the camera as well - that is what // anything wanting this camera's depth in particular reads, see SceneDepthReader if (sceneTextures[i] === SCENETEXTURE_DEPTH) { - this.sceneTexturesCamera.camera.publishSceneDepthMap(texture, this.device.renderVersion); + this.sceneTexturesCamera.publishSceneDepthMap(texture, this.device.renderVersion); } } }