diff --git a/examples.html b/examples.html index ad7f17b..d2e9b63 100644 --- a/examples.html +++ b/examples.html @@ -374,6 +374,7 @@ GLSL Shaders Debug Coloring Depth of Field + Splat Texture Editor
diff --git a/examples/assets.json b/examples/assets.json index 1997065..98b03f3 100644 --- a/examples/assets.json +++ b/examples/assets.json @@ -102,5 +102,13 @@ "valley.spz": { "url": "https://sparkjs.dev/assets/splats/valley.spz", "directory": "splats" + }, + "star.png": { + "url": "https://sparkjs.dev/assets/images/star.png", + "directory": "images" + }, + "heart.png": { + "url": "https://sparkjs.dev/assets/images/heart.png", + "directory": "images" } } diff --git a/examples/depth-of-field/index.html b/examples/depth-of-field/index.html index 7769f4a..e155682 100644 --- a/examples/depth-of-field/index.html +++ b/examples/depth-of-field/index.html @@ -38,15 +38,29 @@ document.body.appendChild(renderer.domElement) const spark = new SparkRenderer({ - renderer, - apertureAngle: 0.02, - focalDistance: 5.0, + renderer, + apertureAngle: 0.02, + focalDistance: 5.0, }); scene.add(spark); + const apertureSize = { + apertureSize: 0.1, + }; + function updateApertureAngle() { + if (spark.focalDistance > 0) { + spark.apertureAngle = 2 * Math.atan(0.5 * apertureSize.apertureSize / spark.focalDistance); + } else { + spark.apertureAngle = 0.0; + } + } + updateApertureAngle(); + const gui = new GUI({ title: "DoF settings" }); - gui.add(spark, "focalDistance", 0.1, 15, 0.1); - gui.add(spark, "apertureAngle", 0.0, 0.01 * Math.PI, 0.001); + gui.add(spark, "focalDistance", 0, 15, 0.01).name("Focal plane dist") + .onChange(updateApertureAngle); + gui.add(apertureSize, "apertureSize", 0, 0.4, 0.01).name("Aperture size") + .onChange(updateApertureAngle); const splatURL = await getAssetFileURL("valley.spz"); const background = new SplatMesh({ url: splatURL }); diff --git a/examples/editor/index.html b/examples/editor/index.html index 8014c52..874b37f 100644 --- a/examples/editor/index.html +++ b/examples/editor/index.html @@ -68,7 +68,7 @@ import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { GUI } from "lil-gui"; - import { constructGrid, SparkControls, SparkRenderer, SplatMesh, textSplats, dyno, transcodeSpz, isPcSogs } from "@sparkjsdev/spark"; + import { constructGrid, SparkControls, SparkRenderer, SplatMesh, textSplats, dyno, transcodeSpz, isMobile, isPcSogs } from "@sparkjsdev/spark"; import { getAssetFileURL } from "/examples/js/get-asset-url.js"; const scene = new THREE.Scene(); @@ -113,7 +113,7 @@ const gui = new GUI({ title: "Settings", container: document.getElementById("main-gui") - }).close(); + }); const secondGui = new GUI({ title: "Splats", container: document.getElementById("second-gui") @@ -134,7 +134,7 @@ }; const guiOptions = { - highDevicePixel: false, + highDevicePixel: !isMobile(), stats: false, resetOnLoad: true, loadOffset: 0, @@ -462,30 +462,55 @@ controls.pointerControls.reverseSwipe = value; }); - gui.add(guiOptions, "highDevicePixel").name("High DPI").onChange((value) => { + function setHighDpi(value) { renderer.setPixelRatio(value ? window.devicePixelRatio : 1); const width = canvas.clientWidth; const height = canvas.clientHeight; renderer.setSize(width, height, false); console.log("Render size", canvas.width, canvas.height); + } + setHighDpi(guiOptions.highDevicePixel); + + gui.add(guiOptions, "highDevicePixel").name("High DPI").onChange((value) => { + setHighDpi(value); }); gui.add(guiOptions, "stats").name("Show frame stats").onChange((value) => { stats.dom.style.display = value ? "block" : "none"; }); gui.add(spark.defaultView, "sortRadial").name("Radial sort").listen(); gui.add(grid, "opacity", 0, 1, 0.01).name("Grid opacity").listen(); + gui.add({ + logFocalDistance: 0.0, + }, "logFocalDistance", -2, 2, 0.01).name("Ln(Focal distance)").onChange((value) => { + spark.focalDistance = Math.exp(value); + }); + gui.add(spark, "apertureAngle", 0, 0.01 * Math.PI, 0.001).name("Aperture angle").listen(); + const debugFolder = gui.addFolder("Debug").close(); const normalColor = dyno.dynoBool(false); - gui.add(normalColor, "value").name("Normal color").onChange(() => updateFrameSplats()); - - gui.add(spark, "maxStdDev", 0.1, 3.0, 0.01).name("Max Gsplat stddev").listen(); - gui.add(spark, "falloff", 0, 1, 0.01).name("Gaussian falloff").listen(); - gui.add(spark, "preBlurAmount", 0, 2, 0.1).name("Blur amount (no AA)"); - gui.add(spark, "blurAmount", 0, 2, 0.1).name("Blur amount (AA)"); + debugFolder.add(normalColor, "value").name("Normal color").onChange(() => updateFrameSplats()); + + debugFolder.add(spark, "maxStdDev", 0.1, 3.0, 0.01).name("Max Gsplat stddev").listen(); + debugFolder.add(spark, "falloff", 0, 1, 0.01).name("Gaussian falloff").listen(); + debugFolder.add(spark, "preBlurAmount", 0, 2, 0.1).name("Blur amount (no AA)").listen(); + debugFolder.add(spark, "blurAmount", 0, 2, 0.1).name("Blur amount (AA)").listen(); + debugFolder.add({ + nonAA: () => { + spark.preBlurAmount = 0.3; + spark.blurAmount = 0.0; + }, + }, "nonAA").name("Non-AA preset"); + debugFolder.add({ + AA: () => { + spark.preBlurAmount = 0.0; + spark.blurAmount = 0.3; + }, + }, "AA").name("AA preset"); + debugFolder.add(spark, "renderScale", 0.1, 2.0, 0.1).name("Render scale"); const splatsFolder = secondGui.addFolder("Files"); - const editFolder = gui.addFolder("Edit Splats").close(); + const clipFolder = gui.addFolder("Clip Splats").close(); function updateFrameSplats() { frame.children.forEach((child) => { @@ -502,13 +527,13 @@ const clipMaxY = dyno.dynoFloat(5); const clipMinZ = dyno.dynoFloat(-5); const clipMaxZ = dyno.dynoFloat(5); - editFolder.add(clipEnable, "value").name("Enable clip").onChange(() => updateFrameSplats()); - editFolder.add(clipMinX, "value", -5, 5, 0.01).name("Min X").onChange(() => updateFrameSplats()); - editFolder.add(clipMaxX, "value", -5, 5, 0.01).name("Max X").onChange(() => updateFrameSplats()); - editFolder.add(clipMinY, "value", -5, 5, 0.01).name("Min Y").onChange(() => updateFrameSplats()); - editFolder.add(clipMaxY, "value", -5, 5, 0.01).name("Max Y").onChange(() => updateFrameSplats()); - editFolder.add(clipMinZ, "value", -5, 5, 0.01).name("Min Z").onChange(() => updateFrameSplats()); - editFolder.add(clipMaxZ, "value", -5, 5, 0.01).name("Max Z").onChange(() => updateFrameSplats()); + clipFolder.add(clipEnable, "value").name("Enable clip").onChange(() => updateFrameSplats()); + clipFolder.add(clipMinX, "value", -50, 50, 0.01).name("Min X").onChange(() => updateFrameSplats()); + clipFolder.add(clipMaxX, "value", -50, 50, 0.01).name("Max X").onChange(() => updateFrameSplats()); + clipFolder.add(clipMinY, "value", -50, 50, 0.01).name("Min Y").onChange(() => updateFrameSplats()); + clipFolder.add(clipMaxY, "value", -50, 50, 0.01).name("Max Y").onChange(() => updateFrameSplats()); + clipFolder.add(clipMinZ, "value", -50, 50, 0.01).name("Min Z").onChange(() => updateFrameSplats()); + clipFolder.add(clipMaxZ, "value", -50, 50, 0.01).name("Max Z").onChange(() => updateFrameSplats()); function makeWorldModifier(mesh) { const context = mesh.context; @@ -548,7 +573,7 @@ }); } - const writeFolder = secondGui.addFolder("Write Gsplats").close(); + const exportFolder = secondGui.addFolder("Export Gsplats").close(); const writeOptions = { filename: "gsplats", trimOpacity: true, @@ -591,12 +616,12 @@ URL.revokeObjectURL(url); }, }; - writeFolder.add(writeOptions, "filename").name("Filename").listen(); - writeFolder.add(writeOptions, "trimOpacity").name("Trim low opacity"); - writeFolder.add(writeOptions, "trimOpacityThreshold").name("Trim opacity <= 0..1"); - writeFolder.add(writeOptions, "maxSh", 0, 3, 1).name("Max spherical harmonics"); - writeFolder.add(writeOptions, "fractionalBits", 6, 24, 1).name("Fractional bits"); - writeFolder.add(writeOptions, "writeSpz").name("Create .spz and download"); + exportFolder.add(writeOptions, "filename").name("Filename").listen(); + exportFolder.add(writeOptions, "trimOpacity").name("Trim low opacity"); + exportFolder.add(writeOptions, "trimOpacityThreshold").name("Trim opacity <= 0..1"); + exportFolder.add(writeOptions, "maxSh", 0, 3, 1).name("Max spherical harmonics"); + exportFolder.add(writeOptions, "fractionalBits", 6, 24, 1).name("Fractional bits"); + exportFolder.add(writeOptions, "writeSpz").name("Create .spz and download"); function makeInstructions() { const instructions = textSplats({ diff --git a/examples/splat-texture/index.html b/examples/splat-texture/index.html new file mode 100644 index 0000000..e5b408b --- /dev/null +++ b/examples/splat-texture/index.html @@ -0,0 +1,166 @@ + + + + + + + Spark • Splat Texture + + + + + + + + + diff --git a/index.html b/index.html index c241b3a..50e2edd 100644 --- a/index.html +++ b/index.html @@ -146,6 +146,7 @@

Examples

  • GLSL Shaders
  • Debug Coloring
  • Depth of Field
    • +
  • Splat Texture
  • Editor
  • Viewer
    • diff --git a/src/SparkRenderer.ts b/src/SparkRenderer.ts index 3596df3..69492c2 100644 --- a/src/SparkRenderer.ts +++ b/src/SparkRenderer.ts @@ -148,6 +148,17 @@ export class SparkRenderer extends THREE.Mesh { apertureAngle: number; falloff: number; clipXY: number; + renderScale = 1.0; + + splatTexture: null | { + enable?: boolean; + texture?: THREE.Data3DTexture; + multiply?: THREE.Matrix2; + add?: THREE.Vector2; + near?: number; + far?: number; + mid?: number; + } = null; time?: number; deltaTime?: number; @@ -198,6 +209,8 @@ export class SparkRenderer extends THREE.Mesh { } | null = null; private static pmrem: THREE.PMREMGenerator | null = null; + static EMPTY_SPLAT_TEXTURE = new THREE.Data3DTexture(); + constructor(options: SparkRendererOptions) { const uniforms = SparkRenderer.makeUniforms(); const shaders = getShaders(); @@ -281,6 +294,9 @@ export class SparkRenderer extends THREE.Mesh { const uniforms = { // Size of render viewport in pixels renderSize: { value: new THREE.Vector2() }, + // Near and far plane distances + near: { value: 0.1 }, + far: { value: 1000.0 }, // Total number of Gsplats in packedSplats to render numSplats: { value: 0 }, // SplatAccumulator to view transformation quaternion @@ -304,6 +320,22 @@ export class SparkRenderer extends THREE.Mesh { falloff: { value: 1.0 }, // Clip Gsplats that are clipXY times beyond the +-1 frustum bounds clipXY: { value: 1.4 }, + // Debug renderSize scale factor + renderScale: { value: 1.0 }, + // Enable splat texture rendering + splatTexEnable: { value: false }, + // Splat texture to render + splatTexture: { type: "t", value: SparkRenderer.EMPTY_SPLAT_TEXTURE }, + // Splat texture UV transform (multiply) + splatTexMul: { value: new THREE.Matrix2() }, + // Splat texture UV transform (add) + splatTexAdd: { value: new THREE.Vector2() }, + // Splat texture near plane distance + splatTexNear: { value: 0.1 }, + // Splat texture far plane distance + splatTexFar: { value: 1000.0 }, + // Splat texture mid plane distance, or 0.0 to disable + splatTexMid: { value: 0.0 }, // Gsplat collection to render packedSplats: { type: "t", value: PackedSplats.getEmpty() }, // Time in seconds for time-based effects @@ -433,6 +465,11 @@ export class SparkRenderer extends THREE.Mesh { } // Update uniforms from instance properties + const typedCamera = camera as + | THREE.PerspectiveCamera + | THREE.OrthographicCamera; + this.uniforms.near.value = typedCamera.near; + this.uniforms.far.value = typedCamera.far; this.uniforms.encodeLinear.value = viewpoint.encodeLinear; this.uniforms.maxStdDev.value = this.maxStdDev; this.uniforms.enable2DGS.value = this.enable2DGS; @@ -442,6 +479,36 @@ export class SparkRenderer extends THREE.Mesh { this.uniforms.apertureAngle.value = this.apertureAngle; this.uniforms.falloff.value = this.falloff; this.uniforms.clipXY.value = this.clipXY; + this.uniforms.renderScale.value = this.renderScale; + + if (this.splatTexture) { + const { enable, texture, multiply, add, near, far, mid } = + this.splatTexture; + if (enable && texture) { + this.uniforms.splatTexEnable.value = true; + this.uniforms.splatTexture.value = texture; + if (multiply) { + this.uniforms.splatTexMul.value.fromArray(multiply.elements); + } else { + this.uniforms.splatTexMul.value.set( + 0.5 / this.maxStdDev, + 0, + 0, + 0.5 / this.maxStdDev, + ); + } + this.uniforms.splatTexAdd.value.set(add?.x ?? 0.5, add?.y ?? 0.5); + this.uniforms.splatTexNear.value = near ?? this.uniforms.near.value; + this.uniforms.splatTexFar.value = far ?? this.uniforms.far.value; + this.uniforms.splatTexMid.value = mid ?? 0.0; + } else { + this.uniforms.splatTexEnable.value = false; + this.uniforms.splatTexture.value = SparkRenderer.EMPTY_SPLAT_TEXTURE; + } + } else { + this.uniforms.splatTexEnable.value = false; + this.uniforms.splatTexture.value = SparkRenderer.EMPTY_SPLAT_TEXTURE; + } // Calculate the transform from the accumulator to the current camera const accumToWorld = diff --git a/src/shaders/splatFragment.glsl b/src/shaders/splatFragment.glsl index f59aed0..c49de39 100644 --- a/src/shaders/splatFragment.glsl +++ b/src/shaders/splatFragment.glsl @@ -4,11 +4,21 @@ precision highp int; #include +uniform float near; +uniform float far; uniform bool encodeLinear; uniform float maxStdDev; uniform bool disableFalloff; uniform float falloff; +uniform bool splatTexEnable; +uniform sampler3D splatTexture; +uniform mat2 splatTexMul; +uniform vec2 splatTexAdd; +uniform float splatTexNear; +uniform float splatTexFar; +uniform float splatTexMid; + out vec4 fragColor; in vec4 vRgba; @@ -16,20 +26,45 @@ in vec2 vSplatUv; in vec3 vNdc; void main() { + vec4 rgba = vRgba; + float z = dot(vSplatUv, vSplatUv); - if (z > (maxStdDev * maxStdDev)) { - discard; + if (!splatTexEnable) { + if (z > (maxStdDev * maxStdDev)) { + discard; + } + } else { + vec2 uv = splatTexMul * vSplatUv + splatTexAdd; + float ndcZ = vNdc.z; + float depth = (2.0 * near * far) / (far + near - ndcZ * (far - near)); + float clampedFar = max(splatTexFar, splatTexNear); + float clampedDepth = clamp(depth, splatTexNear, clampedFar); + float logDepth = log2(clampedDepth + 1.0); + float logNear = log2(splatTexNear + 1.0); + float logFar = log2(clampedFar + 1.0); + + float texZ; + if (splatTexMid > 0.0) { + float clampedMid = clamp(splatTexMid, splatTexNear, clampedFar); + float logMid = log2(clampedMid + 1.0); + texZ = (clampedDepth <= clampedMid) ? + (0.5 * ((logDepth - logNear) / (logMid - logNear))) : + (0.5 * ((logDepth - logMid) / (logFar - logMid)) + 0.5); + } else { + texZ = (logDepth - logNear) / (logFar - logNear); + } + + vec4 modulate = texture(splatTexture, vec3(uv, 1.0 - texZ)); + rgba *= modulate; } - float alpha = vRgba.a; - alpha *= mix(1.0, exp(-0.5 * z), falloff); - if (alpha < MIN_ALPHA) { + rgba.a *= mix(1.0, exp(-0.5 * z), falloff); + + if (rgba.a < MIN_ALPHA) { discard; } - - vec3 rgb = vRgba.rgb; if (encodeLinear) { - rgb = srgbToLinear(rgb); + rgba.rgb = srgbToLinear(rgba.rgb); } - fragColor = vec4(rgb, alpha); + fragColor = rgba; } diff --git a/src/shaders/splatVertex.glsl b/src/shaders/splatVertex.glsl index c9e9893..2eb0312 100644 --- a/src/shaders/splatVertex.glsl +++ b/src/shaders/splatVertex.glsl @@ -25,6 +25,7 @@ uniform float preBlurAmount; uniform float focalDistance; uniform float apertureAngle; uniform float clipXY; +uniform float renderScale; uniform usampler2DArray packedSplats; @@ -111,7 +112,8 @@ void main() { mat3 cov3D = RS * transpose(RS); // Compute the Jacobian of the splat's projection at its center - vec2 focal = 0.5 * renderSize * vec2(projectionMatrix[0][0], projectionMatrix[1][1]); + vec2 scaledRenderSize = renderSize * renderScale; + vec2 focal = 0.5 * scaledRenderSize * vec2(projectionMatrix[0][0], projectionMatrix[1][1]); float invZ = 1.0 / viewCenter.z; vec2 J1 = focal * invZ; vec2 J2 = -(J1 * viewCenter.xy) * invZ; @@ -175,7 +177,7 @@ void main() { // Compute the NDC coordinates for the ellipsoid's diagonal axes. vec2 pixelOffset = eigenVec1 * scale1 + eigenVec2 * scale2; - vec2 ndcOffset = (2.0 / renderSize) * pixelOffset; + vec2 ndcOffset = (2.0 / scaledRenderSize) * pixelOffset; vec3 ndc = vec3(ndcCenter.xy + ndcOffset, ndcCenter.z); vRgba = rgba;