Improve material system data packing

src/engine/renderer/Material.h

@@ -70,14 +70,6 @@ struct DrawCommand {
 
 struct Material {
 	uint32_t materialsSSBOOffset = 0;
-	uint32_t staticMaterialsSSBOOffset = 0;
-	uint32_t dynamicMaterialsSSBOOffset = 0;
-	uint32_t totalDrawSurfCount = 0;
-	uint32_t totalStaticDrawSurfCount = 0;
-	uint32_t totalDynamicDrawSurfCount = 0;
-	uint32_t currentDrawSurfCount = 0;
-	uint32_t currentStaticDrawSurfCount = 0;
-	uint32_t currentDynamicDrawSurfCount = 0;
 
 	uint32_t globalID = 0;
 	uint32_t surfaceCommandBatchOffset = 0;
@@ -143,6 +135,59 @@ struct Material {
 	}
 };
 
+struct TexBundle {
+	vec_t textureMatrix[6];
+	GLuint64 textures[MAX_TEXTURE_BUNDLES];
+};
+
+struct TextureData {
+	const textureBundle_t* texBundles[MAX_TEXTURE_BUNDLES] = { nullptr, nullptr, nullptr, nullptr, nullptr };
+	// For ST_STYLELIGHTMAP stages
+	image_t* texBundlesOverride[MAX_TEXTURE_BUNDLES] = { nullptr, nullptr, nullptr, nullptr, nullptr };
+	int textureMatrixBundle = 0;
+
+	bool operator==( const TextureData& other ) const {
+		for ( int i = 0; i < MAX_TEXTURE_BUNDLES; i++ ) {
+			if ( texBundlesOverride[i] != other.texBundlesOverride[i] ) {
+				return false;
+			}
+
+			const textureBundle_t* bundle = texBundles[i];
+			const textureBundle_t* otherBundle = other.texBundles[i];
+
+			// Skip texBundles image check for ST_STYLELIGHTMAP
+			if ( !texBundlesOverride[i] ) {
+				if ( bundle->numImages != otherBundle->numImages ) {
+					return false;
+				}
+
+				if ( ( bundle->numImages > 1 ) && ( bundle->imageAnimationSpeed != otherBundle->imageAnimationSpeed ) ) {
+					return false;
+				}
+
+				const uint8_t numImages = bundle->numImages > 0 ? bundle->numImages : 1;
+				for ( int j = 0; j < numImages; j++ ) {
+					if ( bundle->image[j] != otherBundle->image[j] ) {
+						return false;
+					}
+				}
+			}
+
+			if ( bundle->numTexMods != otherBundle->numTexMods ) {
+				return false;
+			}
+
+			for ( size_t j = 0; j < bundle->numTexMods; j++ ) {
+				if ( bundle->texMods[j] != otherBundle->texMods[j] ) {
+					return false;
+				}
+			}
+		}
+
+		return true;
+	}
+};
+
 enum class MaterialDebugMode {
 	NONE,
 	DEPTH,
@@ -178,8 +223,19 @@ extern PortalView portalStack[MAX_VIEWS];
 #define INDIRECT_COMMAND_SIZE 5
 #define SURFACE_COMMAND_SIZE 4
 #define SURFACE_COMMAND_BATCH_SIZE 2
+#define TEX_BUNDLE_SIZE 16
+#define TEX_BUNDLE_BITS 12
+#define LIGHTMAP_SIZE 4
+#define LIGHTMAP_BITS 24
 #define PORTAL_SURFACE_SIZE 8
 
+// 64kb min
+#define MIN_MATERIAL_UBO_SIZE BIT( 16 )
+
+/* 64kb UBO : 54kb texBundles, 4kb lightmaps, 2kb map shader stages, 4kb entity shader stages
+Current mapss use up to ~38kb at max, without models */
+#define MAX_TEX_BUNDLES 54 * 1024 / 64
+
 #define MAX_FRAMES 2
 #define MAX_VIEWFRAMES MAX_VIEWS * MAX_FRAMES // Buffer 2 frames for each view
 
@@ -276,14 +332,19 @@ class MaterialSystem {
 
 	void GenerateDepthImages( const int width, const int height, imageParams_t imageParms );
 
-	void AddStageTextures( drawSurf_t* drawSurf, shaderStage_t* pStage, Material* material );
+	void AddStageTextures( drawSurf_t* drawSurf, const uint32_t stage, Material* material );
+	void AddStage( drawSurf_t* drawSurf, shaderStage_t* pStage, uint32_t stage,
+		const bool mayUseVertexOverbright, const bool vertexLit, const bool fullbright );
 	void ProcessStage( drawSurf_t* drawSurf, shaderStage_t* pStage, shader_t* shader, uint32_t* packIDs, uint32_t& stage,
 		uint32_t& previousMaterialID );
 	void GenerateWorldMaterials();
 	void GenerateWorldMaterialsBuffer();
 	void GenerateWorldCommandBuffer();
 	void GeneratePortalBoundingSpheres();
 
+	void GenerateMaterialsBuffer( std::vector<shaderStage_t*>& stages, const uint32_t size, uint32_t* materialsData );
+	void GenerateTexturesBuffer( std::vector<TextureData>& textures, TexBundle* textureBundles );
+
 	void AddAllWorldSurfaces();
 
 	void Free();
@@ -308,9 +369,18 @@ class MaterialSystem {
 	uint32_t surfaceCommandsCount = 0;
 	uint32_t surfaceDescriptorsCount = 0;
 
-	std::vector<drawSurf_t> dynamicDrawSurfs;
-	uint32_t dynamicDrawSurfsOffset = 0;
-	uint32_t dynamicDrawSurfsSize = 0;
+	std::vector<shaderStage_t*> materialStages;
+	std::vector<shaderStage_t*> dynamicStages;
+
+	GLenum texDataBufferType;
+	std::vector<TextureData> texData;
+	std::vector<TextureData> dynamicTexData;
+
+	uint32_t totalStageSize;
+	uint32_t dynamicStagesOffset = 0;
+	uint32_t dynamicStagesSize = 0;
+	uint32_t dynamicTexDataOffset = 0;
+	uint32_t dynamicTexDataSize = 0;
 
 	Frame frames[MAX_FRAMES];
 	uint32_t currentFrame = 0;
@@ -323,7 +393,9 @@ class MaterialSystem {
 	void UpdateFrameData();
 };
 
-extern GLSSBO materialsSSBO; // Global
+extern GLUBO materialsUBO; // Global
+extern GLBuffer texDataBuffer; // Global
+extern GLUBO lightMapDataUBO; // Global
 
 extern GLSSBO surfaceDescriptorsSSBO; // Global
 extern GLSSBO surfaceCommandsSSBO; // Per viewframe, GPU updated
@@ -336,16 +408,18 @@ extern GLSSBO debugSSBO; // Global
 
 extern MaterialSystem materialSystem;
 
-void UpdateSurfaceDataNONE( uint32_t*, Material&, drawSurf_t*, const uint32_t );
-void UpdateSurfaceDataNOP( uint32_t*, Material&, drawSurf_t*, const uint32_t );
-void UpdateSurfaceDataGeneric3D( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataLightMapping( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataReflection( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataSkybox( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataScreen( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataHeatHaze( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataLiquid( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
-void UpdateSurfaceDataFog( uint32_t* materials, Material& material, drawSurf_t* drawSurf, const uint32_t stage );
+void UpdateSurfaceDataNONE( uint32_t*, shaderStage_t*, bool, bool, bool );
+void UpdateSurfaceDataNOP( uint32_t*, shaderStage_t*, bool, bool, bool );
+void UpdateSurfaceDataGeneric3D( uint32_t* materials, shaderStage_t* pStage, bool mayUseVertexOverbright, bool, bool );
+void UpdateSurfaceDataLightMapping( uint32_t* materials, shaderStage_t* pStage, bool, bool vertexLit, bool fullbright );
+void UpdateSurfaceDataReflection( uint32_t* materials, shaderStage_t* pStage, bool, bool, bool );
+void UpdateSurfaceDataSkybox( uint32_t* materials, shaderStage_t* pStage, bool, bool, bool );
+void UpdateSurfaceDataScreen( uint32_t* materials, shaderStage_t* pStage, bool, bool, bool );
+void UpdateSurfaceDataHeatHaze( uint32_t* materials, shaderStage_t* pStage, bool, bool, bool );
+void UpdateSurfaceDataLiquid( uint32_t* materials, shaderStage_t* pStage, bool, bool, bool );
+void UpdateSurfaceDataFog( uint32_t* materials, shaderStage_t* pStage, bool, bool, bool );
+
+// void UpdateSurf( uint32)
 
 void BindShaderNONE( Material* );
 void BindShaderNOP( Material* );
@@ -360,7 +434,7 @@ void BindShaderFog( Material* material );
 
 void ProcessMaterialNONE( Material*, shaderStage_t*, drawSurf_t* );
 void ProcessMaterialNOP( Material*, shaderStage_t*, drawSurf_t* );
-void ProcessMaterialGeneric3D( Material* material, shaderStage_t* pStage, drawSurf_t* drawSurf );
+void ProcessMaterialGeneric3D( Material* material, shaderStage_t* pStage, drawSurf_t* /* drawSurf */ );
 void ProcessMaterialLightMapping( Material* material, shaderStage_t* pStage, drawSurf_t* drawSurf );
 void ProcessMaterialReflection( Material* material, shaderStage_t* pStage, drawSurf_t* /* drawSurf */ );
 void ProcessMaterialSkybox( Material* material, shaderStage_t* pStage, drawSurf_t* /* drawSurf */ );

src/engine/renderer/ShadeCommon.h

@@ -32,12 +32,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ===========================================================================
 */
 
-inline size_t GetLightMapNum( shaderCommands_t* tess )
+inline size_t GetLightMapNum( const shaderCommands_t* tess )
 {
 	return tess->lightmapNum;
 }
 
-inline size_t GetLightMapNum( drawSurf_t* drawSurf )
+inline size_t GetLightMapNum( const drawSurf_t* drawSurf )
 {
 	return drawSurf->lightmapNum();
 }

src/engine/renderer/gl_shader.cpp

@@ -1359,9 +1359,45 @@ std::string GLShaderManager::ShaderPostProcess( GLShader *shader, const std::str
 
 	std::string newShaderText;
 	std::string materialStruct = "\nstruct Material {\n";
-	std::string materialBlock = "layout(std430, binding = 0) readonly buffer materialsSSBO {\n"
-									 "  Material materials[];\n"
-									 "};\n\n";
+	// 6 kb for materials
+	const uint32_t count = ( 4096 + 2048 ) / shader->GetPaddedSize();
+	std::string materialBlock = "layout(std140, binding = 6) uniform materialsUBO {\n"
+	                            "	Material materials[" + std::to_string( count ) + "]; \n"
+	                            "};\n\n";
+
+	std::string texBuf = glConfig2.maxUniformBlockSize >= MIN_MATERIAL_UBO_SIZE ?
+		"layout(std140, binding = 7) uniform texDataUBO {\n"
+		"	TexData texData[" + std::to_string( MAX_TEX_BUNDLES ) + "]; \n"
+		"};\n\n"
+		: "layout(std430, binding = 7) restrict readonly buffer texDataSSBO {\n"
+		"	TexData texData[];\n"
+		"};\n\n";
+	// We have to store u_TextureMatrix as vec4 + vec2 because otherwise it would be aligned to a vec4 under std140
+	std::string texDataBlock = "struct TexData {\n"
+		                       "	vec4 u_TextureMatrix;\n"
+	                           "	vec2 u_TextureMatrix2;\n"
+	                           "	uvec2 u_DiffuseMap;\n"
+	                           "	uvec2 u_NormalMap;\n"
+	                           "	uvec2 u_HeightMap;\n"
+	                           "	uvec2 u_MaterialMap;\n"
+	                           "	uvec2 u_GlowMap;\n"
+	                           "};\n\n"
+	                           + texBuf +
+		                       "#define u_TextureMatrix mat3x2( texData[( baseInstance >> 12 ) & 0xFFF].u_TextureMatrix.xy, texData[( baseInstance >> 12 ) & 0xFFF].u_TextureMatrix.zw, texData[( baseInstance >> 12 ) & 0xFFF].u_TextureMatrix2 )\n"
+		                       "#define u_DiffuseMap_initial texData[( baseInstance >> 12 ) & 0xFFF].u_DiffuseMap\n"
+		                       "#define u_NormalMap_initial texData[( baseInstance >> 12 ) & 0xFFF].u_NormalMap\n"
+		                       "#define u_HeightMap_initial texData[( baseInstance >> 12 ) & 0xFFF].u_HeightMap\n"
+		                       "#define u_MaterialMap_initial texData[( baseInstance >> 12 ) & 0xFFF].u_MaterialMap\n"
+		                       "#define u_GlowMap_initial texData[( baseInstance >> 12 ) & 0xFFF].u_GlowMap\n\n"
+		                       "struct LightMapData {\n"
+	                           "	uvec2 u_LightMap;\n"
+	                           "	uvec2 u_DeluxeMap;\n"
+	                           "};\n\n"
+	                           "layout(std140, binding = 8) uniform lightMapDataUBO {\n"
+	                           "	LightMapData lightMapData[256];\n"
+	                           "};\n\n"
+		                       "#define u_LightMap_initial lightMapData[( baseInstance >> 24 ) & 0xFF].u_LightMap\n"
+		                       "#define u_DeluxeMap_initial lightMapData[( baseInstance >> 24 ) & 0xFF].u_DeluxeMap\n\n";
 	std::string materialDefines;
 
 	/* Generate the struct and defines in the form of:
@@ -1380,24 +1416,25 @@ std::string GLShaderManager::ShaderPostProcess( GLShader *shader, const std::str
 			continue;
 		}
 
-		if ( uniform->IsTexture() ) {
-			materialStruct += "  uvec2 ";
-			materialStruct += uniform->GetName();
-		} else {
-			materialStruct += "  " + uniform->GetType() + " " + uniform->GetName();
-		}
+		if ( !uniform->IsTexture() ) {
+			materialStruct += "	" + uniform->GetType() + " " + uniform->GetName();
 
-		if ( uniform->GetComponentSize() ) {
-			materialStruct += "[ " + std::to_string( uniform->GetComponentSize() ) + " ]";
+			if ( uniform->GetComponentSize() ) {
+				materialStruct += "[ " + std::to_string( uniform->GetComponentSize() ) + " ]";
+			}
+			materialStruct += ";\n";
+
+			// vec3 is aligned to 4 components, so just pad it with int
+			// TODO: Try to move 1 component uniforms here to avoid wasting memory
+			if ( uniform->GetSTD430Size() == 3 ) {
+				materialStruct += "	int ";
+				materialStruct += uniform->GetName();
+				materialStruct += "_padding;\n";
+			}
 		}
-		materialStruct += ";\n";
 
-		// vec3 is aligned to 4 components, so just pad it with int
-		// TODO: Try to move 1 component uniforms here to avoid wasting memory
-		if ( uniform->GetSTD430Size() == 3 ) {
-			materialStruct += "  int ";
-			materialStruct += uniform->GetName();
-			materialStruct += "_padding;\n";
+		if ( uniform->IsTexture() ) {
+			continue;
 		}
 
 		materialDefines += "#define ";
@@ -1407,7 +1444,7 @@ std::string GLShaderManager::ShaderPostProcess( GLShader *shader, const std::str
 			materialDefines += "_initial uvec2("; // We'll need this to create sampler objects later
 		}
 
-		materialDefines += " materials[baseInstance].";
+		materialDefines += " materials[baseInstance & 0xFFF].";
 		materialDefines += uniform->GetName();
 
 		if ( uniform->IsTexture() ) {
@@ -1419,7 +1456,7 @@ std::string GLShaderManager::ShaderPostProcess( GLShader *shader, const std::str
 
 	// Array of structs is aligned to the largest member of the struct
 	for ( uint i = 0; i < shader->padding; i++ ) {
-		materialStruct += "  int material_padding" + std::to_string( i );
+		materialStruct += "	int material_padding" + std::to_string( i );
 		materialStruct += ";\n";
 	}
 
@@ -1457,7 +1494,7 @@ std::string GLShaderManager::ShaderPostProcess( GLShader *shader, const std::str
 
 	materialDefines += "\n";
 
-	newShaderText = "#define USE_MATERIAL_SYSTEM\n" + materialStruct + materialBlock + materialDefines + shaderMain;
+	newShaderText = "#define USE_MATERIAL_SYSTEM\n" + materialStruct + materialBlock + texDataBlock + materialDefines + shaderMain;
 	return newShaderText;
 }
 
@@ -1978,7 +2015,7 @@ void GLShader::PostProcessUniforms() {
 
 	std::vector<GLUniform*> globalUniforms;
 	for ( GLUniform* uniform : _uniforms ) {
-		if ( uniform->IsGlobal() ) {
+		if ( uniform->IsGlobal() || uniform->IsTexture() ) {
 			globalUniforms.emplace_back( uniform );
 		}
 	}
@@ -1988,7 +2025,6 @@ void GLShader::PostProcessUniforms() {
 
 	// Sort uniforms from highest to lowest alignment so we don't need to pad uniforms (other than vec3s)
 	const uint numUniforms = _uniforms.size();
-	GLuint structAlignment = 0;
 	GLuint structSize = 0;
 	while ( tmp.size() < numUniforms ) {
 		// Higher-alignment uniforms first to avoid wasting memory
@@ -1999,9 +2035,7 @@ void GLShader::PostProcessUniforms() {
 				highestAlignment = _uniforms[i]->GetSTD430Alignment();
 				highestUniform = i;
 			}
-			if ( highestAlignment > structAlignment ) {
-				structAlignment = highestAlignment;
-			}
+
 			if ( highestAlignment == 4 ) {
 				break; // 4-component is the highest alignment in std430
 			}
@@ -2020,6 +2054,7 @@ void GLShader::PostProcessUniforms() {
 	}
 	_uniforms = tmp;
 
+	const GLuint structAlignment = 4; // Material buffer is now a UBO, so it uses std140 layout, which is aligned to vec4
 	if ( structSize > 0 ) {
 		padding = ( structAlignment - ( structSize % structAlignment ) ) % structAlignment;
 	}
@@ -2178,7 +2213,7 @@ void GLShader::SetRequiredVertexPointers()
 void GLShader::WriteUniformsToBuffer( uint32_t* buffer ) {
 	uint32_t* bufPtr = buffer;
 	for ( GLUniform* uniform : _uniforms ) {
-		if ( !uniform->IsGlobal() ) {
+		if ( !uniform->IsGlobal() && !uniform->IsTexture() ) {
 			bufPtr = uniform->WriteToBuffer( bufPtr );
 		}
 	}
@@ -2676,7 +2711,7 @@ GLShader_fogQuake3::GLShader_fogQuake3( GLShaderManager *manager ) :
 	u_FogMap( this ),
 	u_ModelMatrix( this ),
 	u_ModelViewProjectionMatrix( this ),
-	u_Color( this ),
+	u_ColorGlobal( this ),
 	u_Bones( this ),
 	u_VertexInterpolation( this ),
 	u_FogDistanceVector( this ),
@@ -2698,7 +2733,7 @@ GLShader_fogQuake3Material::GLShader_fogQuake3Material( GLShaderManager* manager
 	u_FogMap( this ),
 	u_ModelMatrix( this ),
 	u_ModelViewProjectionMatrix( this ),
-	u_Color( this ),
+	u_ColorGlobal( this ),
 	u_FogDistanceVector( this ),
 	u_FogDepthVector( this ),
 	u_FogEyeT( this ),