Basic dithering for Linux OpenGL

Add dithering to yuv2rgb shader. This is not optimal since scaling and
possibly other filtering is done later. Dithering should be the final
step.

PBO resolution needs to be made higher to avoid banding from RGB range
expansion in colorspace conversion when two pass rendering is used.

The dither value scaling is a little ugly, but works well. Even with
quantization to 16 levels (4bpc) picture is reasonable on a computer
screen.

system/shaders/yuv2rgb_basic.glsl

@@ -37,6 +37,10 @@ uniform mat4      m_yuvmat;
 
 uniform float     m_stretch;
 
+uniform sampler2D m_dither;
+uniform float     m_ditherquant;
+uniform vec2      m_dithersize;
+
 vec2 stretch(vec2 pos)
 {
 #if (XBMC_STRETCH)
@@ -66,8 +70,6 @@ void main()
                  , 1.0 );
 
   rgb   = m_yuvmat * yuv;
-  rgb.a = gl_Color.a;
-  gl_FragColor = rgb;
 
 #elif defined(XBMC_VDPAU_NV12)
 
@@ -78,8 +80,6 @@ void main()
                  , 1.0 );
 
   rgb   = m_yuvmat * yuv;
-  rgb.a = gl_Color.a;
-  gl_FragColor = rgb;
 
 #elif defined(XBMC_YUY2) || defined(XBMC_UYVY)
 
@@ -118,8 +118,11 @@ void main()
   vec4  yuv     = vec4(outY, outUV, 1.0);
   vec4  rgb     = m_yuvmat * yuv;
 
-  gl_FragColor   = rgb;
-  gl_FragColor.a = gl_Color.a;
-
 #endif
+
+  vec2 ditherpos = gl_FragCoord.xy / m_dithersize;
+  float ditherval = texture2D(m_dither, ditherpos).r;
+  ditherval = ditherval * 8.0;
+  gl_FragColor = floor(rgb * m_ditherquant + ditherval) / m_ditherquant;
+  gl_FragColor.a = gl_Color.a;
 }

xbmc/cores/VideoRenderers/VideoShaders/YUV2RGBShader.cpp

@@ -22,6 +22,7 @@
 #include "system.h"
 #include "../RenderFlags.h"
 #include "YUV2RGBShader.h"
+#include "dither.h"
 #include "settings/AdvancedSettings.h"
 #include "guilib/TransformMatrix.h"
 #include "windowing/WindowingFactory.h"
@@ -315,8 +316,71 @@ YUV2RGBProgressiveShader::YUV2RGBProgressiveShader(bool rect, unsigned flags, ER
 #elif HAS_GLES == 2
   PixelShader()->LoadSource("yuv2rgb_basic_gles.glsl", m_defines);
 #endif
+  m_tDitherTex = 0;
 }
 
+void YUV2RGBProgressiveShader::OnCompiledAndLinked()
+{
+  if (m_tDitherTex)
+  {
+    glDeleteTextures(1, &m_tDitherTex);
+    m_tDitherTex = 0;
+  }
+
+  glActiveTexture(GL_TEXTURE3);
+  glGenTextures(1, &m_tDitherTex);
+  if ( m_tDitherTex <= 0 )
+  {
+    CLog::Log(LOGERROR, "GL: YUV2RGBProgressiveShader: Error creating dither texture");
+    return;
+  }
+  glBindTexture(GL_TEXTURE_2D, m_tDitherTex);
+  glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+  glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+  glTexImage2D(GL_TEXTURE_2D, 0, GL_R16, dither_size, dither_size, 0, GL_RED,
+GL_SHORT, dither_matrix);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
+  glActiveTexture(GL_TEXTURE0);
+  VerifyGLState();
+
+  m_hDither = glGetUniformLocation(ProgramHandle(), "m_dither");
+  m_hDitherQuant = glGetUniformLocation(ProgramHandle(), "m_ditherquant");
+  m_hDitherSize = glGetUniformLocation(ProgramHandle(), "m_dithersize");
+  VerifyGLState();
+
+  BaseYUV2RGBGLSLShader::OnCompiledAndLinked();
+}
+
+bool YUV2RGBProgressiveShader::OnEnabled()
+{
+  if (!BaseYUV2RGBGLSLShader::OnEnabled())
+    return false;
+
+  glActiveTexture(GL_TEXTURE3);
+  glBindTexture(GL_TEXTURE_2D, m_tDitherTex);
+  glActiveTexture(GL_TEXTURE0);
+  VerifyGLState();
+  glUniform1i(m_hDither, 3);
+  VerifyGLState();
+  glUniform1f(m_hDitherQuant, 255.0); // (1<<depth)-1
+  VerifyGLState();
+  glUniform2f(m_hDitherSize, dither_size, dither_size);
+  VerifyGLState();
+  return true;
+}
+
+void YUV2RGBProgressiveShader::Free()
+{
+  if (m_tDitherTex)
+  {
+    glDeleteTextures(1, &m_tDitherTex);
+    m_tDitherTex = 0;
+  }
+  BaseYUV2RGBGLSLShader::Free();
+}
 
 //////////////////////////////////////////////////////////////////////
 // YUV2RGBBobShader - YUV2RGB with Bob deinterlacing

xbmc/cores/VideoRenderers/VideoShaders/YUV2RGBShader.h

@@ -171,6 +171,14 @@ namespace Shaders {
   {
   public:
     YUV2RGBProgressiveShader(bool rect=false, unsigned flags=0, ERenderFormat format=RENDER_FMT_NONE, bool stretch = false);
+    void OnCompiledAndLinked();
+    bool OnEnabled();
+    void Free();
+  protected:
+    GLuint  m_tDitherTex;
+    GLint   m_hDither;
+    GLint   m_hDitherQuant;
+    GLint   m_hDitherSize;
   };
 
   class YUV2RGBBobShader : public BaseYUV2RGBGLSLShader