Vulkan: Only apply the depth workaround on Qualcomm devices (adreno)

We've seen issues, and the validation layer still doesn't like them very much...

GPU/Vulkan/FragmentShaderGeneratorVulkan.cpp

@@ -43,7 +43,7 @@ static const char *vulkan_glsl_preamble =
 #define WRITE p+=sprintf
 
 // Missing: Z depth range
-bool GenerateVulkanGLSLFragmentShader(const FShaderID &id, char *buffer) {
+bool GenerateVulkanGLSLFragmentShader(const FShaderID &id, char *buffer, uint32_t vulkanVendorId) {
 	char *p = buffer;
 
 	const char *lastFragData = nullptr;
@@ -85,9 +85,11 @@ bool GenerateVulkanGLSLFragmentShader(const FShaderID &id, char *buffer) {
 	const char *shading = doFlatShading ? "flat" : "";
 	bool earlyFragmentTests = ((!enableAlphaTest && !enableColorTest) || testForceToZero) && !gstate_c.Supports(GPU_ROUND_FRAGMENT_DEPTH_TO_16BIT);
 
+	bool isAdreno = vulkanVendorId == VULKAN_VENDOR_QUALCOMM;
+
 	if (earlyFragmentTests) {
 		WRITE(p, "layout (early_fragment_tests) in;\n");
-	} else {
+	} else if (isAdreno) {
 		WRITE(p, "layout (depth_unchanged) out float gl_FragDepth;\n");
 	}
 
@@ -583,7 +585,7 @@ bool GenerateVulkanGLSLFragmentShader(const FShaderID &id, char *buffer) {
 			WRITE(p, "  z = (1.0/65535.0) * floor(z * 65535.0);\n");
 		}
 		WRITE(p, "  gl_FragDepth = z;\n");
-	} else if (!earlyFragmentTests) {
+	} else if (!earlyFragmentTests && isAdreno) {
 		// Adreno (and possibly MESA/others) apply early frag tests even with discard in the shader.
 		// Writing depth prevents the bug, even with depth_unchanged specified.
 		WRITE(p, "  gl_FragDepth = gl_FragCoord.z;\n");

GPU/Vulkan/FragmentShaderGeneratorVulkan.h

@@ -20,4 +20,4 @@
 
 struct FShaderID;
 
-bool GenerateVulkanGLSLFragmentShader(const FShaderID &id, char *buffer);
+bool GenerateVulkanGLSLFragmentShader(const FShaderID &id, char *buffer, uint32_t vulkanVendorId);

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -268,8 +268,9 @@ void ShaderManagerVulkan::GetShaders(int prim, u32 vertType, VulkanVertexShader
 
 	VulkanFragmentShader *fs = fsCache_.Get(FSID);
 	if (!fs) {
+		uint32_t vendorID = vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).vendorID;
 		// Fragment shader not in cache. Let's compile it.
-		GenerateVulkanGLSLFragmentShader(FSID, codeBuffer_);
+		GenerateVulkanGLSLFragmentShader(FSID, codeBuffer_, vendorID);
 		fs = new VulkanFragmentShader(vulkan_, FSID, codeBuffer_);
 		fsCache_.Insert(FSID, fs);
 	}
@@ -389,13 +390,14 @@ bool ShaderManagerVulkan::LoadCache(FILE *f) {
 		VulkanVertexShader *vs = new VulkanVertexShader(vulkan_, id, codeBuffer_, useHWTransform);
 		vsCache_.Insert(id, vs);
 	}
+	uint32_t vendorID = vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).vendorID;
 	for (int i = 0; i < header.numFragmentShaders; i++) {
 		FShaderID id;
 		if (fread(&id, sizeof(id), 1, f) != 1) {
 			ERROR_LOG(G3D, "Vulkan shader cache truncated");
 			break;
 		}
-		GenerateVulkanGLSLFragmentShader(id, codeBuffer_);
+		GenerateVulkanGLSLFragmentShader(id, codeBuffer_, vendorID);
 		VulkanFragmentShader *fs = new VulkanFragmentShader(vulkan_, id, codeBuffer_);
 		fsCache_.Insert(id, fs);
 	}

GPU/Vulkan/StencilBufferVulkan.cpp

@@ -29,7 +29,7 @@
 
 // This shader references gl_FragDepth to prevent early fragment tests.
 // They shouldn't happen since it uses discard, but Adreno detects that incorrectly - see #10634.
-static const char *stencil_fs = R"(#version 450
+static const char *stencil_fs_adreno = R"(#version 450
 #extension GL_ARB_separate_shader_objects : enable
 #extension GL_ARB_shading_language_420pack : enable
 #extension GL_ARB_conservative_depth : enable
@@ -56,6 +56,30 @@ void main() {
 }
 )";
 
+static const char *stencil_fs = R"(#version 450
+#extension GL_ARB_separate_shader_objects : enable
+#extension GL_ARB_shading_language_420pack : enable
+layout (binding = 0) uniform sampler2D tex;
+layout (push_constant) uniform params {
+	int u_stencilValue;
+};
+layout (location = 0) in vec2 v_texcoord0;
+layout (location = 0) out vec4 fragColor0;
+
+void main() {
+	if (u_stencilValue == 0) {
+		fragColor0 = vec4(0.0);
+	} else {
+		vec4 index = texture(tex, v_texcoord0);
+		int indexBits = int(floor(index.a * 255.99)) & 0xFF;
+		if ((indexBits & u_stencilValue) == 0)
+			discard;
+		fragColor0 = index.aaaa;
+	}
+}
+)";
+
+
 static const char stencil_vs[] = R"(#version 450
 #extension GL_ARB_separate_shader_objects : enable
 #extension GL_ARB_shading_language_420pack : enable
@@ -119,8 +143,13 @@ bool FramebufferManagerVulkan::NotifyStencilUpload(u32 addr, int size, bool skip
 
 	std::string error;
 	if (!stencilVs_) {
+		const char *stencil_fs_source = stencil_fs;
+		// See comment above the stencil_fs_adreno definition.
+		if (vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).vendorID == VULKAN_VENDOR_QUALCOMM)
+			stencil_fs_source = stencil_fs_adreno;
+
 		stencilVs_ = CompileShaderModule(vulkan_, VK_SHADER_STAGE_VERTEX_BIT, stencil_vs, &error);
-		stencilFs_ = CompileShaderModule(vulkan_, VK_SHADER_STAGE_FRAGMENT_BIT, stencil_fs, &error);
+		stencilFs_ = CompileShaderModule(vulkan_, VK_SHADER_STAGE_FRAGMENT_BIT, stencil_fs_source, &error);
 	}
 	VkRenderPass rp = (VkRenderPass)draw_->GetNativeObject(Draw::NativeObject::FRAMEBUFFER_RENDERPASS);