gsdx-hw: Clamp zbuffer to depthmask instead of wrapping.

The behavior was verified on Dobie to be correct.
The code needs to be ported to SW renderer too to
improve rendering on SW side.

Current PR will fix plenty of games on HW renderer
that had/have zbuffer issues before.

v2. Set DepthMask to the maximum the current depth format allows.
Will properly clamp for 16bit and 24bit formats.

v3. gl: Fix uniform buffer upload/cache for VSConstantBuffer.

plugins/GSdx/Renderers/DX11/GSRendererDX11.cpp

@@ -176,7 +176,7 @@ void GSRendererDX11::EmulateZbuffer()
 	// The real GS appears to do no masking based on the Z buffer format and writing larger Z values
 	// than the buffer supports seems to be an error condition on the real GS, causing it to crash.
 	// We are probably receiving bad coordinates from VU1 in these cases.
-	vs_cb.DepthMask = GSVector2i(0xFFFFFFFF, 0xFFFFFFFF);
+	vs_cb.DepthMask = GSVector2i(max_z, max_z);
 
 	if (m_om_dssel.ztst >= ZTST_ALWAYS && m_om_dssel.zwe && (m_context->ZBUF.PSM != PSM_PSMZ32))
 	{
@@ -189,7 +189,6 @@ void GSRendererDX11::EmulateZbuffer()
 #ifdef _DEBUG
 				fprintf(stdout, "%d: Bad Z size on %s buffers\n", s_n, psm_str(m_context->ZBUF.PSM));
 #endif
-				vs_cb.DepthMask = GSVector2i(max_z, max_z);
 				m_om_dssel.ztst = ZTST_ALWAYS;
 			}
 		}

plugins/GSdx/Renderers/OpenGL/GSDeviceOGL.h

@@ -137,6 +137,7 @@ class GSDeviceOGL final : public GSDevice
 			{
 				a[0] = b[0];
 				a[1] = b[1];
+				a[2] = b[2];
 
 				return true;
 			}

plugins/GSdx/Renderers/OpenGL/GSRendererOGL.cpp

@@ -183,15 +183,14 @@ void GSRendererOGL::EmulateZbuffer()
 	// The real GS appears to do no masking based on the Z buffer format and writing larger Z values
 	// than the buffer supports seems to be an error condition on the real GS, causing it to crash.
 	// We are probably receiving bad coordinates from VU1 in these cases.
-	vs_cb.DepthMask = GSVector2i(0xFFFFFFFF, 0xFFFFFFFF);
+	vs_cb.DepthMask = GSVector2i(max_z, max_z);
 
 	if (m_om_dssel.ztst >= ZTST_ALWAYS && m_om_dssel.zwe && (m_context->ZBUF.PSM != PSM_PSMZ32)) {
 		if (m_vt.m_max.p.z > max_z) {
 			ASSERT(m_vt.m_min.p.z > max_z); // sfex capcom logo
 			// Fixme :Following conditional fixes some dialog frame in Wild Arms 3, but may not be what was intended.
 			if (m_vt.m_min.p.z > max_z) {
 				GL_DBG("Bad Z size (%f %f) on %s buffers", m_vt.m_min.p.z, m_vt.m_max.p.z, psm_str(m_context->ZBUF.PSM));
-				vs_cb.DepthMask = GSVector2i(max_z, max_z);
 				m_om_dssel.ztst = ZTST_ALWAYS;
 			}
 		}

plugins/GSdx/res/glsl/tfx_vgs.glsl

@@ -44,7 +44,8 @@ void texture_coord()
 
 void vs_main()
 {
-    highp uint z = i_z & DepthMask;
+    // Clamp to depth mask, gs doesn't wrap
+    highp uint z = min(i_z, DepthMask);
 
     // pos -= 0.05 (1/320 pixel) helps avoiding rounding problems (integral part of pos is usually 5 digits, 0.05 is about as low as we can go)
     // example: ceil(afterseveralvertextransformations(y = 133)) => 134 => line 133 stays empty

plugins/GSdx/res/tfx.fx

@@ -787,7 +787,8 @@ PS_OUTPUT ps_main(PS_INPUT input)
 
 VS_OUTPUT vs_main(VS_INPUT input)
 {
-	input.z &= DepthMask;
+	// Clamp to depth mask, gs doesn't wrap
+	input.z = min(input.z, DepthMask);
 
 	VS_OUTPUT output;