gsdx-ogl: accurate date regression. Bad interaction between depth & DATE

In the DATE42 algo, first pass must find the primitive that write the
bad alpha value. If depth test is fail, alpha value won't be written therefore
you mustn't keep the primitive id.

In theory to ensure 100% correctness, depth would need to be fully executed
(currently depth write is disabled). However it requires to copy the depth buffer.
It is likely bad for the perf.

Issue reported on DBZInfWorld

plugins/GSdx/GSDeviceOGL.cpp

@@ -613,6 +613,10 @@ void GSDeviceOGL::InitPrimDateTexture(GSTexture* rt)
 	ClearRenderTarget_i(m_date.t, 0x7FFFFFFF);
 
 	gl_BindImageTexture(2, m_date.t->GetID(), 0, false, 0, GL_READ_WRITE, GL_R32I);
+#ifdef ENABLE_OGL_DEBUG
+	// Help to see the texture in apitrace
+	PSSetShaderResource(2, m_date.t);
+#endif
 }
 
 void GSDeviceOGL::RecycleDateTexture()

plugins/GSdx/res/glsl/tfx_fs.glsl

@@ -48,10 +48,23 @@ layout(binding = 3) uniform sampler2D RtSampler; // note 2 already use by the im
 #ifndef DISABLE_GL42_image
 #if PS_DATE > 0
 // FIXME how to declare memory access
-layout(r32i, binding = 2) coherent uniform iimage2D img_prim_min;
-// Don't enable it. Discard fragment can still write in the depth buffer
-// it breaks shadow in Shin Megami Tensei Nocturne
-//layout(early_fragment_tests) in;
+layout(r32i, binding = 2) uniform iimage2D img_prim_min;
+// WARNING:
+// You can't enable it if you discard the fragment. The depth is still
+// updated (shadow in Shin Megami Tensei Nocturne)
+//
+// early_fragment_tests must still be enabled in the first pass of the 2 passes algo
+// First pass search the first primitive that will write the bad alpha value. Value
+// won't be written if the fragment fails the depth test.
+//
+// In theory the best solution will be do
+// 1/ copy the depth buffer
+// 2/ do the full depth (current depth writes are disabled)
+// 3/ restore the depth buffer for 2nd pass
+// Of course, it is likely too costly.
+#if PS_DATE == 1 || PS_DATE == 2
+layout(early_fragment_tests) in;
+#endif
 
 // I don't remember why I set this parameter but it is surely useless
 //layout(pixel_center_integer) in vec4 gl_FragCoord;
@@ -564,15 +577,15 @@ void ps_main()
     // Pixel with alpha equal to 1 will failed (128-255)
     if (C.a > 127.5f) {
         imageAtomicMin(img_prim_min, ivec2(gl_FragCoord.xy), gl_PrimitiveID);
-        return;
     }
+    return;
 #elif PS_DATE == 2 && !defined(DISABLE_GL42_image)
     // DATM == 1
     // Pixel with alpha equal to 0 will failed (0-127)
     if (C.a < 127.5f) {
         imageAtomicMin(img_prim_min, ivec2(gl_FragCoord.xy), gl_PrimitiveID);
-        return;
     }
+    return;
 #endif
 
     ps_blend(C, alpha_blend);

plugins/GSdx/res/glsl_source.h

@@ -952,10 +952,23 @@ static const char* tfx_fs_all_glsl =
 	"#ifndef DISABLE_GL42_image\n"
 	"#if PS_DATE > 0\n"
 	"// FIXME how to declare memory access\n"
-	"layout(r32i, binding = 2) coherent uniform iimage2D img_prim_min;\n"
-	"// Don't enable it. Discard fragment can still write in the depth buffer\n"
-	"// it breaks shadow in Shin Megami Tensei Nocturne\n"
-	"//layout(early_fragment_tests) in;\n"
+	"layout(r32i, binding = 2) uniform iimage2D img_prim_min;\n"
+	"// WARNING:\n"
+	"// You can't enable it if you discard the fragment. The depth is still\n"
+	"// updated (shadow in Shin Megami Tensei Nocturne)\n"
+	"//\n"
+	"// early_fragment_tests must still be enabled in the first pass of the 2 passes algo\n"
+	"// First pass search the first primitive that will write the bad alpha value. Value\n"
+	"// won't be written if the fragment fails the depth test.\n"
+	"// \n"
+	"// In theory the best solution will be do \n"
+	"// 1/ copy the depth buffer\n"
+	"// 2/ do the full depth (current depth writes are disabled)\n"
+	"// 3/ restore the depth buffer for 2nd pass\n"
+	"// Of course, it is likely too costly.\n"
+	"#if PS_DATE == 1 || PS_DATE == 2\n"
+	"layout(early_fragment_tests) in;\n"
+	"#endif\n"
 	"\n"
 	"// I don't remember why I set this parameter but it is surely useless\n"
 	"//layout(pixel_center_integer) in vec4 gl_FragCoord;\n"
@@ -1468,15 +1481,15 @@ static const char* tfx_fs_all_glsl =
 	"    // Pixel with alpha equal to 1 will failed (128-255)\n"
 	"    if (C.a > 127.5f) {\n"
 	"        imageAtomicMin(img_prim_min, ivec2(gl_FragCoord.xy), gl_PrimitiveID);\n"
-	"        return;\n"
 	"    }\n"
+	"    return;\n"
 	"#elif PS_DATE == 2 && !defined(DISABLE_GL42_image)\n"
 	"    // DATM == 1\n"
 	"    // Pixel with alpha equal to 0 will failed (0-127)\n"
 	"    if (C.a < 127.5f) {\n"
 	"        imageAtomicMin(img_prim_min, ivec2(gl_FragCoord.xy), gl_PrimitiveID);\n"
-	"        return;\n"
 	"    }\n"
+	"    return;\n"
 	"#endif\n"
 	"\n"
 	"    ps_blend(C, alpha_blend);\n"