-
Notifications
You must be signed in to change notification settings - Fork 3.4k
/
gaussianSplattingMaterial.ts
300 lines (247 loc) · 10.3 KB
/
gaussianSplattingMaterial.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
import type { SubMesh } from "../../Meshes/subMesh";
import type { AbstractMesh } from "../../Meshes/abstractMesh";
import type { Mesh } from "../../Meshes/mesh";
import type { IEffectCreationOptions } from "../../Materials/effect";
import type { Scene } from "../../scene";
import type { Matrix } from "../../Maths/math.vector";
import type { GaussianSplattingMesh } from "core/Meshes";
import { SerializationHelper } from "../../Misc/decorators.serialization";
import { VertexBuffer } from "../../Buffers/buffer";
import { MaterialDefines } from "../../Materials/materialDefines";
import { PushMaterial } from "../../Materials/pushMaterial";
import { RegisterClass } from "../../Misc/typeStore";
import { addClipPlaneUniforms, bindClipPlane } from "../clipPlaneMaterialHelper";
import { Camera } from "core/Cameras/camera";
import "../../Shaders/gaussianSplatting.fragment";
import "../../Shaders/gaussianSplatting.vertex";
import {
BindFogParameters,
BindLogDepth,
PrepareAttributesForInstances,
PrepareDefinesForAttributes,
PrepareDefinesForFrameBoundValues,
PrepareDefinesForMisc,
PrepareUniformsAndSamplersList,
} from "../materialHelper.functions";
/**
* @internal
*/
class GaussianSplattingMaterialDefines extends MaterialDefines {
public FOG = false;
public THIN_INSTANCES = true;
public LOGARITHMICDEPTH = false;
public CLIPPLANE = false;
public CLIPPLANE2 = false;
public CLIPPLANE3 = false;
public CLIPPLANE4 = false;
public CLIPPLANE5 = false;
public CLIPPLANE6 = false;
/**
* Constructor of the defines.
*/
constructor() {
super();
this.rebuild();
}
}
/**
* GaussianSplattingMaterial material used to render Gaussian Splatting
* @experimental
*/
export class GaussianSplattingMaterial extends PushMaterial {
/**
* Instantiates a Gaussian Splatting Material in the given scene
* @param name The friendly name of the material
* @param scene The scene to add the material to
*/
constructor(name: string, scene?: Scene) {
super(name, scene);
this.backFaceCulling = false;
}
/**
* Gets a boolean indicating that current material needs to register RTT
*/
public override get hasRenderTargetTextures(): boolean {
return false;
}
/**
* Specifies whether or not this material should be rendered in alpha test mode.
* @returns false
*/
public override needAlphaTesting(): boolean {
return false;
}
/**
* Specifies whether or not this material should be rendered in alpha blend mode.
* @returns true
*/
public override needAlphaBlending(): boolean {
return true;
}
/**
* Checks whether the material is ready to be rendered for a given mesh.
* @param mesh The mesh to render
* @param subMesh The submesh to check against
* @returns true if all the dependencies are ready (Textures, Effects...)
*/
public override isReadyForSubMesh(mesh: AbstractMesh, subMesh: SubMesh): boolean {
const useInstances = true;
const drawWrapper = subMesh._drawWrapper;
if (drawWrapper.effect && this.isFrozen) {
if (drawWrapper._wasPreviouslyReady && drawWrapper._wasPreviouslyUsingInstances === useInstances) {
return true;
}
}
if (!subMesh.materialDefines) {
subMesh.materialDefines = new GaussianSplattingMaterialDefines();
}
const scene = this.getScene();
const defines = <GaussianSplattingMaterialDefines>subMesh.materialDefines;
if (this._isReadyForSubMesh(subMesh)) {
return true;
}
const engine = scene.getEngine();
// Misc.
PrepareDefinesForMisc(mesh, scene, this._useLogarithmicDepth, this.pointsCloud, this.fogEnabled, false, defines);
// Values that need to be evaluated on every frame
PrepareDefinesForFrameBoundValues(scene, engine, this, defines, useInstances, null, true);
// Attribs
PrepareDefinesForAttributes(mesh, defines, false, false);
// Get correct effect
if (defines.isDirty) {
defines.markAsProcessed();
scene.resetCachedMaterial();
//Attributes
const attribs = [VertexBuffer.PositionKind, "splatIndex"];
PrepareAttributesForInstances(attribs, defines);
const uniforms = ["world", "view", "projection", "vFogInfos", "vFogColor", "logarithmicDepthConstant", "viewport", "dataTextureSize", "focal"];
const samplers = ["covariancesATexture", "covariancesBTexture", "centersTexture", "colorsTexture"];
const uniformBuffers = ["Scene", "Mesh"];
PrepareUniformsAndSamplersList(<IEffectCreationOptions>{
uniformsNames: uniforms,
uniformBuffersNames: uniformBuffers,
samplers: samplers,
defines: defines,
});
addClipPlaneUniforms(uniforms);
const join = defines.toString();
const effect = scene.getEngine().createEffect(
"gaussianSplatting",
<IEffectCreationOptions>{
attributes: attribs,
uniformsNames: uniforms,
uniformBuffersNames: uniformBuffers,
samplers: samplers,
defines: join,
onCompiled: this.onCompiled,
onError: this.onError,
},
engine
);
subMesh.setEffect(effect, defines, this._materialContext);
}
if (!subMesh.effect || !subMesh.effect.isReady()) {
return false;
}
defines._renderId = scene.getRenderId();
drawWrapper._wasPreviouslyReady = true;
drawWrapper._wasPreviouslyUsingInstances = useInstances;
return true;
}
/**
* Binds the submesh to this material by preparing the effect and shader to draw
* @param world defines the world transformation matrix
* @param mesh defines the mesh containing the submesh
* @param subMesh defines the submesh to bind the material to
*/
public override bindForSubMesh(world: Matrix, mesh: Mesh, subMesh: SubMesh): void {
const scene = this.getScene();
const defines = <GaussianSplattingMaterialDefines>subMesh.materialDefines;
if (!defines) {
return;
}
const effect = subMesh.effect;
if (!effect) {
return;
}
this._activeEffect = effect;
// Matrices Mesh.
mesh.getMeshUniformBuffer().bindToEffect(effect, "Mesh");
mesh.transferToEffect(world);
// Bind data
const mustRebind = this._mustRebind(scene, effect, subMesh, mesh.visibility);
if (mustRebind) {
this.bindView(effect);
this.bindViewProjection(effect);
const engine = scene.getEngine();
const camera = this.getScene().activeCamera;
const renderWidth = engine.getRenderWidth();
const renderHeight = engine.getRenderHeight();
this._activeEffect.setFloat2("viewport", renderWidth, renderHeight);
let focal = 1000;
if (camera) {
if (camera.fovMode == Camera.FOVMODE_VERTICAL_FIXED) {
focal = renderHeight / 2.0 / Math.tan(camera.fov / 2.0);
} else {
focal = renderWidth / 2.0 / Math.tan(camera.fov / 2.0);
}
}
this._activeEffect.setFloat2("focal", focal, focal);
const gsMesh = mesh as GaussianSplattingMesh;
if (gsMesh.covariancesATexture) {
const textureSize = gsMesh.covariancesATexture.getSize();
effect.setFloat2("dataTextureSize", textureSize.width, textureSize.height);
effect.setTexture("covariancesATexture", gsMesh.covariancesATexture);
effect.setTexture("covariancesBTexture", gsMesh.covariancesBTexture);
effect.setTexture("centersTexture", gsMesh.centersTexture);
effect.setTexture("colorsTexture", gsMesh.colorsTexture);
}
// Clip plane
bindClipPlane(effect, this, scene);
} else if (scene.getEngine()._features.needToAlwaysBindUniformBuffers) {
this._needToBindSceneUbo = true;
}
// Fog
BindFogParameters(scene, mesh, effect);
// Log. depth
if (this.useLogarithmicDepth) {
BindLogDepth(defines, effect, scene);
}
this._afterBind(mesh, this._activeEffect, subMesh);
}
/**
* Clones the material.
* @param name The cloned name.
* @returns The cloned material.
*/
public override clone(name: string): GaussianSplattingMaterial {
return SerializationHelper.Clone(() => new GaussianSplattingMaterial(name, this.getScene()), this);
}
/**
* Serializes the current material to its JSON representation.
* @returns The JSON representation.
*/
public override serialize(): any {
const serializationObject = super.serialize();
serializationObject.customType = "BABYLON.GaussianSplattingMaterial";
return serializationObject;
}
/**
* Gets the class name of the material
* @returns "GaussianSplattingMaterial"
*/
public override getClassName(): string {
return "GaussianSplattingMaterial";
}
/**
* Parse a JSON input to create back a Gaussian Splatting material.
* @param source The JSON data to parse
* @param scene The scene to create the parsed material in
* @param rootUrl The root url of the assets the material depends upon
* @returns the instantiated GaussianSplattingMaterial.
*/
public static override Parse(source: any, scene: Scene, rootUrl: string): GaussianSplattingMaterial {
return SerializationHelper.Parse(() => new GaussianSplattingMaterial(source.name, scene), source, scene, rootUrl);
}
}
RegisterClass("BABYLON.GaussianSplattingMaterial", GaussianSplattingMaterial);