Merge pull request #2369 from Armada651/depth-percision

VideoBackends: Use proper floating point depth precision.

Source/Core/VideoBackends/D3D/FramebufferManager.cpp

@@ -93,10 +93,10 @@ FramebufferManager::FramebufferManager()
 	D3D::SetDebugObjectName((ID3D11DeviceChild*)m_efb.color_staging_buf, "EFB color staging texture (used for Renderer::AccessEFB)");
 
 	// EFB depth buffer - primary depth buffer
-	texdesc = CD3D11_TEXTURE2D_DESC(DXGI_FORMAT_R24G8_TYPELESS, m_target_width, m_target_height, m_efb.slices, 1, D3D11_BIND_DEPTH_STENCIL | D3D11_BIND_SHADER_RESOURCE, D3D11_USAGE_DEFAULT, 0, sample_desc.Count, sample_desc.Quality);
+	texdesc = CD3D11_TEXTURE2D_DESC(DXGI_FORMAT_R32_TYPELESS, m_target_width, m_target_height, m_efb.slices, 1, D3D11_BIND_DEPTH_STENCIL | D3D11_BIND_SHADER_RESOURCE, D3D11_USAGE_DEFAULT, 0, sample_desc.Count, sample_desc.Quality);
 	hr = D3D::device->CreateTexture2D(&texdesc, nullptr, &buf);
 	CHECK(hr==S_OK, "create EFB depth texture (size: %dx%d; hr=%#x)", m_target_width, m_target_height, hr);
-	m_efb.depth_tex = new D3DTexture2D(buf, (D3D11_BIND_FLAG)(D3D11_BIND_DEPTH_STENCIL|D3D11_BIND_SHADER_RESOURCE), DXGI_FORMAT_R24_UNORM_X8_TYPELESS, DXGI_FORMAT_D24_UNORM_S8_UINT, DXGI_FORMAT_UNKNOWN, (sample_desc.Count > 1));
+	m_efb.depth_tex = new D3DTexture2D(buf, (D3D11_BIND_FLAG)(D3D11_BIND_DEPTH_STENCIL|D3D11_BIND_SHADER_RESOURCE), DXGI_FORMAT_R32_FLOAT, DXGI_FORMAT_D32_FLOAT, DXGI_FORMAT_UNKNOWN, (sample_desc.Count > 1));
 	SAFE_RELEASE(buf);
 	D3D::SetDebugObjectName((ID3D11DeviceChild*)m_efb.depth_tex->GetTex(), "EFB depth texture");
 	D3D::SetDebugObjectName((ID3D11DeviceChild*)m_efb.depth_tex->GetDSV(), "EFB depth texture depth stencil view");

Source/Core/VideoBackends/D3D/PixelShaderCache.cpp

@@ -146,7 +146,7 @@ const char depth_matrix_program[] = {
 	" in float4 pos : SV_Position,\n"
 	" in float3 uv0 : TEXCOORD0){\n"
 	"	float4 texcol = Tex0.Sample(samp0,uv0);\n"
-	"	int depth = int(round(texcol.x * float(0xFFFFFF)));\n"
+	"	int depth = clamp(int(texcol.x * 16777216.0), 0, 0xFFFFFF);\n"
 
 	// Convert to Z24 format
 	"	int4 workspace;\n"
@@ -180,7 +180,7 @@ const char depth_matrix_program_msaa[] = {
 	"	for(int i = 0; i < SAMPLES; ++i)\n"
 	"		texcol += Tex0.Load(int3(uv0.x*(width), uv0.y*(height), uv0.z), i);\n"
 	"	texcol /= SAMPLES;\n"
-	"	int depth = int(round(texcol.x * float(0xFFFFFF)));\n"
+	"	int depth = clamp(int(texcol.x * 16777216.0), 0, 0xFFFFFF);\n"
 
 	// Convert to Z24 format
 	"	int4 workspace;\n"

Source/Core/VideoBackends/D3D/Render.cpp

@@ -8,6 +8,7 @@
 #include <strsafe.h>
 #include <unordered_map>
 
+#include "Common/MathUtil.h"
 #include "Common/Timer.h"
 
 #include "Core/ConfigManager.h"
@@ -419,11 +420,11 @@ u32 Renderer::AccessEFB(EFBAccessType type, u32 x, u32 y, u32 poke_data)
 		if (bpmem.zcontrol.pixel_format == PEControl::RGB565_Z16)
 		{
 			// if Z is in 16 bit format you must return a 16 bit integer
-			ret = ((u32)(val * 0xffff));
+			ret = MathUtil::Clamp<u32>((u32)(val * 65536.0f), 0, 0xFFFF);
 		}
 		else
 		{
-			ret = ((u32)(val * 0xffffff));
+			ret = MathUtil::Clamp<u32>((u32)(val * 16777216.0f), 0, 0xFFFFFF);
 		}
 		D3D::context->Unmap(read_tex, 0);
 
@@ -548,7 +549,7 @@ void Renderer::ClearScreen(const EFBRectangle& rc, bool colorEnable, bool alphaE
 
 	// Color is passed in bgra mode so we need to convert it to rgba
 	u32 rgbaColor = (color & 0xFF00FF00) | ((color >> 16) & 0xFF) | ((color << 16) & 0xFF0000);
-	D3D::drawClearQuad(rgbaColor, (z & 0xFFFFFF) / float(0xFFFFFF));
+	D3D::drawClearQuad(rgbaColor, (z & 0xFFFFFF) / 16777216.0f);
 
 	D3D::stateman->PopDepthState();
 	D3D::stateman->PopBlendState();

Source/Core/VideoBackends/OGL/FramebufferManager.cpp

@@ -89,7 +89,7 @@ FramebufferManager::FramebufferManager(int targetWidth, int targetHeight, int ms
 		glTexParameteri(m_textureType, GL_TEXTURE_MAX_LEVEL, 0);
 		glTexParameteri(m_textureType, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 		glTexParameteri(m_textureType, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-		glTexImage3D(m_textureType, 0, GL_DEPTH_COMPONENT24, m_targetWidth, m_targetHeight, m_EFBLayers, 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, nullptr);
+		glTexImage3D(m_textureType, 0, GL_DEPTH_COMPONENT32F, m_targetWidth, m_targetHeight, m_EFBLayers, 0, GL_DEPTH_COMPONENT, GL_FLOAT, nullptr);
 
 		glBindTexture(m_textureType, m_efbColorSwap);
 		glTexParameteri(m_textureType, GL_TEXTURE_MAX_LEVEL, 0);
@@ -111,7 +111,7 @@ FramebufferManager::FramebufferManager(int targetWidth, int targetHeight, int ms
 			glTexImage3DMultisample(m_textureType, m_msaaSamples, GL_RGBA, m_targetWidth, m_targetHeight, m_EFBLayers, false);
 
 			glBindTexture(m_textureType, m_efbDepth);
-			glTexImage3DMultisample(m_textureType, m_msaaSamples, GL_DEPTH_COMPONENT24, m_targetWidth, m_targetHeight, m_EFBLayers, false);
+			glTexImage3DMultisample(m_textureType, m_msaaSamples, GL_DEPTH_COMPONENT32F, m_targetWidth, m_targetHeight, m_EFBLayers, false);
 
 			glBindTexture(m_textureType, m_efbColorSwap);
 			glTexImage3DMultisample(m_textureType, m_msaaSamples, GL_RGBA, m_targetWidth, m_targetHeight, m_EFBLayers, false);
@@ -125,7 +125,7 @@ FramebufferManager::FramebufferManager(int targetWidth, int targetHeight, int ms
 			glTexImage2DMultisample(m_textureType, m_msaaSamples, GL_RGBA, m_targetWidth, m_targetHeight, false);
 
 			glBindTexture(m_textureType, m_efbDepth);
-			glTexImage2DMultisample(m_textureType, m_msaaSamples, GL_DEPTH_COMPONENT24, m_targetWidth, m_targetHeight, false);
+			glTexImage2DMultisample(m_textureType, m_msaaSamples, GL_DEPTH_COMPONENT32F, m_targetWidth, m_targetHeight, false);
 
 			glBindTexture(m_textureType, m_efbColorSwap);
 			glTexImage2DMultisample(m_textureType, m_msaaSamples, GL_RGBA, m_targetWidth, m_targetHeight, false);
@@ -150,7 +150,7 @@ FramebufferManager::FramebufferManager(int targetWidth, int targetHeight, int ms
 		glTexParameteri(resolvedType, GL_TEXTURE_MAX_LEVEL, 0);
 		glTexParameteri(resolvedType, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 		glTexParameteri(resolvedType, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-		glTexImage3D(resolvedType, 0, GL_DEPTH_COMPONENT24, m_targetWidth, m_targetHeight, m_EFBLayers, 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, nullptr);
+		glTexImage3D(resolvedType, 0, GL_DEPTH_COMPONENT32F, m_targetWidth, m_targetHeight, m_EFBLayers, 0, GL_DEPTH_COMPONENT, GL_FLOAT, nullptr);
 
 		// Bind resolved textures to resolved framebuffer.
 		glGenFramebuffers(m_EFBLayers, m_resolvedFramebuffer);

Source/Core/VideoBackends/OGL/Render.cpp

@@ -12,6 +12,7 @@
 #include "Common/Atomic.h"
 #include "Common/CommonPaths.h"
 #include "Common/FileUtil.h"
+#include "Common/MathUtil.h"
 #include "Common/StringUtil.h"
 #include "Common/Thread.h"
 #include "Common/Timer.h"
@@ -914,7 +915,7 @@ void ClearEFBCache()
 	}
 }
 
-void Renderer::UpdateEFBCache(EFBAccessType type, u32 cacheRectIdx, const EFBRectangle& efbPixelRc, const TargetRectangle& targetPixelRc, const u32* data)
+void Renderer::UpdateEFBCache(EFBAccessType type, u32 cacheRectIdx, const EFBRectangle& efbPixelRc, const TargetRectangle& targetPixelRc, const void* data)
 {
 	u32 cacheType = (type == PEEK_Z ? 0 : 1);
 
@@ -936,7 +937,18 @@ void Renderer::UpdateEFBCache(EFBAccessType type, u32 cacheRectIdx, const EFBRec
 			u32 xEFB = efbPixelRc.left + xCache;
 			u32 xPixel = (EFBToScaledX(xEFB) + EFBToScaledX(xEFB + 1)) / 2;
 			u32 xData = xPixel - targetPixelRc.left;
-			s_efbCache[cacheType][cacheRectIdx][yCache * EFB_CACHE_RECT_SIZE + xCache] = data[yData * targetPixelRcWidth + xData];
+			u32 value;
+			if (type == PEEK_Z)
+			{
+				float* ptr = (float*)data;
+				value = MathUtil::Clamp<u32>((u32)(ptr[yData * targetPixelRcWidth + xData] * 16777216.0f), 0, 0xFFFFFF);
+			}
+			else
+			{
+				u32* ptr = (u32*)data;
+				value = ptr[yData * targetPixelRcWidth + xData];
+			}
+			s_efbCache[cacheType][cacheRectIdx][yCache * EFB_CACHE_RECT_SIZE + xCache] = value;
 		}
 	}
 
@@ -1005,10 +1017,10 @@ u32 Renderer::AccessEFB(EFBAccessType type, u32 x, u32 y, u32 poke_data)
 					g_renderer->RestoreAPIState();
 				}
 
-				u32* depthMap = new u32[targetPixelRcWidth * targetPixelRcHeight];
+				float* depthMap = new float[targetPixelRcWidth * targetPixelRcHeight];
 
 				glReadPixels(targetPixelRc.left, targetPixelRc.bottom, targetPixelRcWidth, targetPixelRcHeight,
-				             GL_DEPTH_COMPONENT, GL_UNSIGNED_INT, depthMap);
+				             GL_DEPTH_COMPONENT, GL_FLOAT, depthMap);
 
 				UpdateEFBCache(type, cacheRectIdx, efbPixelRc, targetPixelRc, depthMap);
 
@@ -1019,18 +1031,10 @@ u32 Renderer::AccessEFB(EFBAccessType type, u32 x, u32 y, u32 poke_data)
 			u32 yRect = y % EFB_CACHE_RECT_SIZE;
 			z = s_efbCache[0][cacheRectIdx][yRect * EFB_CACHE_RECT_SIZE + xRect];
 
-			// Scale the 32-bit value returned by glReadPixels to a 24-bit
-			// value (GC uses a 24-bit Z-buffer).
-			// TODO: in RE0 this value is often off by one, which causes lighting to disappear
+			// if Z is in 16 bit format you must return a 16 bit integer
 			if (bpmem.zcontrol.pixel_format == PEControl::RGB565_Z16)
-			{
-				// if Z is in 16 bit format you must return a 16 bit integer
-				z = z >> 16;
-			}
-			else
-			{
 				z = z >> 8;
-			}
+
 			return z;
 		}
 
@@ -1135,7 +1139,7 @@ u32 Renderer::AccessEFB(EFBAccessType type, u32 x, u32 y, u32 poke_data)
 		ResetAPIState();
 
 		glDepthMask(GL_TRUE);
-		glClearDepthf(float(poke_data & 0xFFFFFF) / float(0xFFFFFF));
+		glClearDepthf(float(poke_data & 0xFFFFFF) / 16777216.0f);
 
 		glEnable(GL_SCISSOR_TEST);
 		glScissor(targetPixelRc.left, targetPixelRc.bottom, targetPixelRc.GetWidth(), targetPixelRc.GetHeight());
@@ -1269,7 +1273,7 @@ void Renderer::ClearScreen(const EFBRectangle& rc, bool colorEnable, bool alphaE
 	// depth
 	glDepthMask(zEnable ? GL_TRUE : GL_FALSE);
 
-	glClearDepthf(float(z & 0xFFFFFF) / float(0xFFFFFF));
+	glClearDepthf(float(z & 0xFFFFFF) / 16777216.0f);
 
 	// Update rect for clearing the picture
 	glEnable(GL_SCISSOR_TEST);

Source/Core/VideoBackends/OGL/Render.h

@@ -91,7 +91,7 @@ class Renderer : public ::Renderer
 	int GetMaxTextureSize() override;
 
 private:
-	void UpdateEFBCache(EFBAccessType type, u32 cacheRectIdx, const EFBRectangle& efbPixelRc, const TargetRectangle& targetPixelRc, const u32* data);
+	void UpdateEFBCache(EFBAccessType type, u32 cacheRectIdx, const EFBRectangle& efbPixelRc, const TargetRectangle& targetPixelRc, const void* data);
 
 	void BlitScreen(TargetRectangle src, TargetRectangle dst, GLuint src_texture, int src_width, int src_height);
 };

Source/Core/VideoBackends/OGL/TextureCache.cpp

@@ -294,7 +294,7 @@ void TextureCache::CompileShaders()
 		"\n"
 		"void main(){\n"
 		"	vec4 texcol = texture(samp9, vec3(f_uv0.xy, %s));\n"
-		"	int depth = int(round(texcol.x * float(0xFFFFFF)));\n"
+		"	int depth = clamp(int(texcol.x * 16777216.0), 0, 0xFFFFFF);\n"
 
 		// Convert to Z24 format
 		"	ivec4 workspace;\n"

Source/Core/VideoCommon/PixelShaderGen.cpp

@@ -563,13 +563,14 @@ static inline void GeneratePixelShader(T& out, DSTALPHA_MODE dstAlphaMode, API_T
 	// The performance impact of this additional calculation doesn't matter, but it prevents
 	// the host GPU driver from performing any early depth test optimizations.
 	if (g_ActiveConfig.bFastDepthCalc)
-		out.Write("\tint zCoord = iround(rawpos.z * float(0xFFFFFF));\n");
+		out.Write("\tint zCoord = int(rawpos.z * 16777216.0);\n");
 	else
 	{
 		out.SetConstantsUsed(C_ZBIAS+1, C_ZBIAS+1);
 		// the screen space depth value = far z + (clip z / clip w) * z range
-		out.Write("\tint zCoord = " I_ZBIAS"[1].x + iround((clipPos.z / clipPos.w) * float(" I_ZBIAS"[1].y));\n");
+		out.Write("\tint zCoord = " I_ZBIAS"[1].x + int((clipPos.z / clipPos.w) * float(" I_ZBIAS"[1].y));\n");
 	}
+	out.Write("\tzCoord = clamp(zCoord, " I_ZBIAS"[1].x - " I_ZBIAS"[1].y, " I_ZBIAS"[1].x);\n");
 
 	// depth texture can safely be ignored if the result won't be written to the depth buffer (early_ztest) and isn't used for fog either
 	const bool skip_ztexture = !per_pixel_depth && !bpmem.fog.c_proj_fsel.fsel;
@@ -1169,13 +1170,13 @@ static inline void WriteFog(T& out, pixel_shader_uid_data* uid_data)
 		// TODO: Verify that we want to drop lower bits here! (currently taken over from software renderer)
 		//       Maybe we want to use "ze = (A << B_SHF)/((B << B_SHF) - Zs)" instead?
 		//       That's equivalent, but keeps the lower bits of Zs.
-		out.Write("\tfloat ze = (" I_FOGF"[1].x * 16777215.0) / float(" I_FOGI".y - (zCoord >> " I_FOGI".w));\n");
+		out.Write("\tfloat ze = (" I_FOGF"[1].x * 16777216.0) / float(" I_FOGI".y - (zCoord >> " I_FOGI".w));\n");
 	}
 	else
 	{
 		// orthographic
 		// ze = a*Zs    (here, no B_SHF)
-		out.Write("\tfloat ze = " I_FOGF"[1].x * float(zCoord) / 16777215.0;\n");
+		out.Write("\tfloat ze = " I_FOGF"[1].x * float(zCoord) / 16777216.0;\n");
 	}
 
 	// x_adjust = sqrt((x-center)^2 + k^2)/k
@@ -1224,11 +1225,11 @@ static inline void WritePerPixelDepth(T& out, pixel_shader_uid_data* uid_data, A
 		if (ApiType == API_OPENGL)
 			out.Write("\tscreenpos.y = %i - screenpos.y;\n", EFB_HEIGHT);
 
-		out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * screenpos.x + " I_ZSLOPE".y * screenpos.y) / float(0xFFFFFF);\n");
+		out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * screenpos.x + " I_ZSLOPE".y * screenpos.y) / 16777216.0;\n");
 	}
 	else
 	{
-		out.Write("\tdepth = float(zCoord) / float(0xFFFFFF);\n");
+		out.Write("\tdepth = float(zCoord) / 16777216.0;\n");
 	}
 }
 

Source/Core/VideoCommon/TextureConversionShader.cpp

@@ -487,7 +487,7 @@ static void WriteZ16Encoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 0, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -497,7 +497,7 @@ static void WriteZ16Encoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 1, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -519,7 +519,7 @@ static void WriteZ16LEncoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 0, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -531,7 +531,7 @@ static void WriteZ16LEncoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 1, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -558,7 +558,7 @@ static void WriteZ24Encoder(char*& p, API_TYPE ApiType)
 
 	for (int i = 0; i < 2; i++)
 	{
-		WRITE(p, "  depth%i *= 16777215.0;\n", i);
+		WRITE(p, "  depth%i = clamp(depth%i * 16777216.0, 0.0, float(0xFFFFFF));\n", i, i);
 
 		WRITE(p, "  expanded%i.r = floor(depth%i / (256.0 * 256.0));\n", i, i);
 		WRITE(p, "  depth%i -= expanded%i.r * 256.0 * 256.0;\n", i, i);