renderer: fix vertex lighting with terrain blending and with non-style lightmap with tcGen lightmap

src/engine/renderer/gl_shader.cpp

@@ -549,24 +549,25 @@ static std::string GenEngineConstants() {
 	if ( r_precomputedLighting->integer )
 		AddDefine( str, "r_precomputedLighting", 1 );
 
-	if ( r_showLightMaps->integer )
+	if ( r_showNormalMaps->integer )
 	{
-		AddDefine( str, "r_showLightMaps", 1 );
+		AddDefine( str, "r_showNormalMaps", 1 );
 	}
-
-	if ( r_showDeluxeMaps->integer )
+	else if ( r_showMaterialMaps->integer )
 	{
-		AddDefine( str, "r_showDeluxeMaps", 1 );
+		AddDefine( str, "r_showMaterialMaps", 1 );
 	}
-
-	if ( r_showNormalMaps->integer )
+	else if ( r_showLightMaps->integer )
 	{
-		AddDefine( str, "r_showNormalMaps", 1 );
+		AddDefine( str, "r_showLightMaps", 1 );
 	}
-
-	if ( r_showMaterialMaps->integer )
+	else if ( r_showDeluxeMaps->integer )
 	{
-		AddDefine( str, "r_showMaterialMaps", 1 );
+		AddDefine( str, "r_showDeluxeMaps", 1 );
+	}
+	else if ( r_showVertexColors.Get() )
+	{
+		AddDefine( str, "r_showVertexColors", 1 );
 	}
 
 	if ( glConfig2.vboVertexSkinningAvailable )

src/engine/renderer/glsl_source/generic_fp.glsl

@@ -59,4 +59,9 @@ void	main()
 #if defined(GENERIC_2D)
 	gl_FragDepth = 0;
 #endif
+
+// Debugging.
+#if defined(r_showVertexColors) && !defined(GENERIC_2D)
+	outputColor = vec4(0.0, 0.0, 0.0, 0.0);
+#endif
 }

src/engine/renderer/glsl_source/lightMapping_fp.glsl

@@ -186,15 +186,24 @@ void main()
 	outputColor = color;
 
 	// Debugging.
-	#if defined(r_showLightMaps) && defined(USE_LIGHT_MAPPING)
-		outputColor = texture2D(u_LightMap, var_TexLight);
-	#elif defined(r_showDeluxeMaps) && defined(USE_DELUXE_MAPPING)
-		outputColor = texture2D(u_DeluxeMap, var_TexLight);
-	#elif defined(r_showNormalMaps)
+	#if defined(r_showNormalMaps)
 		// Convert normal to [0,1] color space.
 		normal = normal * 0.5 + 0.5;
 		outputColor = vec4(normal, 1.0);
 	#elif defined(r_showMaterialMaps)
 		outputColor = material;
+	#elif defined(r_showLightMaps) && defined(USE_LIGHT_MAPPING)
+		outputColor = texture2D(u_LightMap, var_TexLight);
+	#elif defined(r_showDeluxeMaps) && defined(USE_DELUXE_MAPPING)
+		outputColor = texture2D(u_DeluxeMap, var_TexLight);
+	#elif defined(r_showVertexColors)
+		/* We need to keep the texture alpha channel so impact
+		marks like creep don't fully overwrite the world texture. */
+		#if defined(USE_BSP_SURFACE)
+			outputColor.rgb = vec3(1.0, 1.0, 1.0);
+			outputColor *= var_Color;
+		#else
+			outputColor.rgb = vec3(0.0, 0.0, 0.0);
+		#endif
 	#endif
 }

src/engine/renderer/tr_init.cpp

@@ -224,6 +224,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 	cvar_t      *r_showLightGrid;
 	cvar_t      *r_showLightTiles;
 	cvar_t      *r_showBatches;
+	Cvar::Cvar<bool> r_showVertexColors("r_showVertexColors", "show vertex colors used for vertex lighting", Cvar::CHEAT, false);
 	cvar_t      *r_showLightMaps;
 	cvar_t      *r_showDeluxeMaps;
 	cvar_t      *r_showNormalMaps;
@@ -1330,6 +1331,7 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 		r_showLightGrid = Cvar_Get( "r_showLightGrid", "0", CVAR_CHEAT );
 		r_showLightTiles = Cvar_Get("r_showLightTiles", "0", CVAR_CHEAT | CVAR_LATCH );
 		r_showBatches = Cvar_Get( "r_showBatches", "0", CVAR_CHEAT );
+		Cvar::Latch( r_showVertexColors );
 		r_showLightMaps = Cvar_Get( "r_showLightMaps", "0", CVAR_CHEAT | CVAR_LATCH );
 		r_showDeluxeMaps = Cvar_Get( "r_showDeluxeMaps", "0", CVAR_CHEAT | CVAR_LATCH );
 		r_showNormalMaps = Cvar_Get( "r_showNormalMaps", "0", CVAR_CHEAT | CVAR_LATCH );
@@ -1431,6 +1433,21 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 			}
 		}
 
+		if ( r_showNormalMaps->integer
+			|| r_showMaterialMaps->integer )
+		{
+			tr.lightMode = lightMode_t::MAP;
+		}
+		else if ( r_showLightMaps->integer
+			|| r_showDeluxeMaps->integer )
+		{
+			tr.lightMode = lightMode_t::MAP;
+		}
+		else if ( r_showVertexColors.Get() )
+		{
+			tr.lightMode = lightMode_t::VERTEX;
+		}
+
 		backEndData[ 0 ] = ( backEndData_t * ) ri.Hunk_Alloc( sizeof( *backEndData[ 0 ] ), ha_pref::h_low );
 		backEndData[ 0 ]->polys = ( srfPoly_t * ) ri.Hunk_Alloc( r_maxPolys->integer * sizeof( srfPoly_t ), ha_pref::h_low );
 		backEndData[ 0 ]->polyVerts = ( polyVert_t * ) ri.Hunk_Alloc( r_maxPolyVerts->integer * sizeof( polyVert_t ), ha_pref::h_low );

src/engine/renderer/tr_local.h

@@ -1177,7 +1177,8 @@ enum class deluxeMode_t { NONE, GRID, MAP };
 	  ST_HEATHAZEMAP, // heatHaze post process effect
 	  ST_LIQUIDMAP,
 	  ST_LIGHTMAP,
-	  ST_LIGHTSTYLEMAP,
+	  ST_STYLELIGHTMAP,
+	  ST_STYLECOLORMAP,
 	  ST_COLLAPSE_lighting_PBR,   // map|diffusemap + opt:normalmap + opt:glowmap + opt:physicalmap
 	  ST_COLLAPSE_lighting_PHONG, // map|diffusemap + opt:normalmap + opt:glowmap + specularmap
 	  ST_COLLAPSE_reflection_CB,  // color cubemap + normalmap
@@ -3098,6 +3099,7 @@ enum class deluxeMode_t { NONE, GRID, MAP };
 	extern cvar_t *r_showLightGrid;
 	extern cvar_t *r_showLightTiles;
 	extern cvar_t *r_showBatches;
+	extern Cvar::Cvar<bool> r_showVertexColors;
 	extern cvar_t *r_showLightMaps; // render lightmaps only
 	extern cvar_t *r_showDeluxeMaps;
 	extern cvar_t *r_showNormalMaps;

src/engine/renderer/tr_shade.cpp

@@ -785,7 +785,7 @@ static void Render_generic( shaderStage_t *pStage )
 	// bind u_ColorMap
 	GL_SelectTexture( 0 );
 
-	if ( pStage->type == stageType_t::ST_LIGHTSTYLEMAP )
+	if ( pStage->type == stageType_t::ST_STYLELIGHTMAP )
 	{
 		GL_Bind( GetLightMap() );
 	}
@@ -914,7 +914,7 @@ static void Render_lightMapping( shaderStage_t *pStage )
 		  {
 		    map textures/texture_d
 		    heightMap textures/texture_h
-	}
+		  }
 
 		This is doable for some complex multi-stage materials. */
 	}
@@ -954,13 +954,6 @@ static void Render_lightMapping( shaderStage_t *pStage )
 			tess.svars.color.SetRed( 0.0f );
 			tess.svars.color.SetGreen( 0.0f );
 			tess.svars.color.SetBlue( 0.0f );
-
-			if ( stateBits & GLS_ATEST_BITS )
-			{
-				// Do not rewrite pStage->alphaGen.
-				alphaGen = alphaGen_t::AGEN_VERTEX;
-				tess.svars.color.SetAlpha( 1.0f );
-	}
 			break;
 
 		case lightMode_t::GRID:
@@ -973,7 +966,7 @@ static void Render_lightMapping( shaderStage_t *pStage )
 			lightmap = GetLightMap();
 
 			if ( r_showLightMaps->integer )
-	{
+			{
 				stateBits &= ~( GLS_SRCBLEND_BITS | GLS_DSTBLEND_BITS | GLS_ATEST_BITS );
 			}
 			break;
@@ -2800,7 +2793,8 @@ void Tess_StageIteratorGeneric()
 
 				break;
 
-			case stageType_t::ST_LIGHTSTYLEMAP:
+			case stageType_t::ST_STYLELIGHTMAP:
+			case stageType_t::ST_STYLECOLORMAP:
 				Render_generic( pStage );
 				break;
 

src/engine/renderer/tr_shader.cpp

@@ -3848,14 +3848,25 @@ static bool ParseShader( const char *_text )
 			/* Examples of shaders with light styles, those
 			shaders are generated by the q3map2 map compiler.
 
-			All stages with “tcGen lightmap” are light style
-			stages, even if using the “$lightmap” keyword for
-			the texture path, or if using a direct path to
-			the lightmap image like “maps/<nam>/lm_<num>[ext]”.
+			All color stages with “tcGen lightmap” are light style
+			stages, they use direct path to the light map image
+			like “maps/<nam>/lm_<num>[ext]”.
+
+			Light map stages using the “$lightmap” image don't
+			need “tcGen lightmap” and then such stage with such
+			keyword should be light style stages, but some legacy
+			materials may needlessly set “tcGen lightmap” on the
+			light map stage so we need more data to not wrongly
+			detect them as light style stages.
+
+			The textures/conveyor/p_01_s material in cruz map
+			is a good example of such light map stage with
+			useless “tcGen lightmap” keywords.
 
 			It looks like all light style stages use
 			the “blendFunc GL_SRC_ALPHA GL_ONE” blend operation,
-			so we may also detect them by looking for that.
+			so we use that to know if a light map stage is a
+			light map one or a light style one.
 
 			gloom2/673A7C895A24ECE49A80324DF57B0CC1
 			{ // Q3Map2 defaulted
@@ -3954,10 +3965,20 @@ static bool ParseShader( const char *_text )
 			Light style map stages using the direct image path
 			are already rendered by Render_lightmap as they have
 			a ST_COLORMAP type so we don't have to change them. */
+
+			bool lightStyleBlend = ( stage->stateBits & GLS_SRCBLEND_BITS ) == GLS_SRCBLEND_SRC_ALPHA
+				&& ( stage->stateBits & GLS_DSTBLEND_BITS ) == GLS_DSTBLEND_ONE;
+
 			if ( stage->type == stageType_t::ST_LIGHTMAP
+				&& stage->tcGen_Lightmap
+				&& lightStyleBlend )
+			{
+				stage->type = stageType_t::ST_STYLELIGHTMAP;
+			}
+			else if ( stage->type != stageType_t::ST_LIGHTMAP
 				&& stage->tcGen_Lightmap )
 			{
-				stage->type = stageType_t::ST_LIGHTSTYLEMAP;
+				stage->type = stageType_t::ST_STYLECOLORMAP;
 			}
 
 			/* Light styles are only compatible with light mapping
@@ -3972,12 +3993,15 @@ static bool ParseShader( const char *_text )
 			is disabled. */
 			if ( tr.lightMode != lightMode_t::MAP || r_lightStyles->integer == 0 )
 			{
-				/* All light styles, either as ST_LIGHTSTYLEMAP
-				or as ST_COLORMAP have tcGen_Lightmap so we can
-				detect both at once. */
-				if ( stage->tcGen_Lightmap )
+				switch ( stage->type )
 				{
-					stage->active = false;
+					case stageType_t::ST_STYLELIGHTMAP:
+					case stageType_t::ST_STYLECOLORMAP:
+						stage->active = false;
+						break;
+
+					default:
+						break;
 				}
 			}
 
@@ -5397,7 +5421,7 @@ static shader_t *FinishShader()
 		{
 			case stageType_t::ST_LIQUIDMAP:
 			case stageType_t::ST_LIGHTMAP:
-			case stageType_t::ST_LIGHTSTYLEMAP:
+			case stageType_t::ST_STYLELIGHTMAP:
 				// skip
 				break;
 
@@ -6265,7 +6289,8 @@ void R_ShaderList_f()
 	stageTypeStrings[stageType_t::ST_HEATHAZEMAP] = "HEATHAZEMAP";
 	stageTypeStrings[stageType_t::ST_LIQUIDMAP] = "LIQUIDMAP";
 	stageTypeStrings[stageType_t::ST_LIGHTMAP] = "LIGHTMAP";
-	stageTypeStrings[stageType_t::ST_LIGHTSTYLEMAP] = "LIGHTSTYLEMAP";
+	stageTypeStrings[stageType_t::ST_STYLELIGHTMAP] = "STYLELIGHTMAP";
+	stageTypeStrings[stageType_t::ST_STYLECOLORMAP] = "STYLECOLORMAP";
 	stageTypeStrings[stageType_t::ST_COLLAPSE_lighting_PBR] = "LIGHTING_PBR";
 	stageTypeStrings[stageType_t::ST_COLLAPSE_lighting_PHONG] = "LIGHTING_PHONG";
 	stageTypeStrings[stageType_t::ST_COLLAPSE_reflection_CB] = "REFLECTION_CB";