OpenG: Enable direct rendering of 10/12/16bit frame formats

- OpenGLVideo can now directly render YV12, NV12, YUY2, YUV420P10, YUV420P12 and YUV420P16 without any conversion - higher depth formats are never pre-processed in OpenGLVideo (unlike YV12 - which still respects the video profile) - supported direct render formats are negotiated between the decoder and the videooutput classes - with the default remaining YV12 only for anything other the OpenGL. - this will not yet guarantee full 10bit output on supported displays. While the main video textures are 16bit, if there is any scaling the intermediate framebuffer will still be 8bit by default; with a resulting loss in precision - there may be issues with some of the OpenGL settings on e.g. Android/ GLES - 12/16bit support is untested TODO - add support for 16bit float framebuffers to ensure no precision loss - check for higher depth framebuffer support and use 8 or 16 bit precision throughout accordingly (i.e. don't waste memory for 16 bit textures on an 8bit display/framebuffer) - add temporal dithering as an option for 10/12/16 to 8 bit downsampling - select and use optimum frame formats for hardware, decode only frame transfers (i.e. some decoders operate on NV12 data internally - which has better performance in the OpenGL layer - and requesting the native format avoids an internal conversion)
MythTV · Mar 19, 2019 · 007b063 · 007b063
1 parent d6317ce
commit 007b063
Show file tree

Hide file tree

Showing 13 changed files with 193 additions and 46 deletions.
diff --git a/mythtv/libs/libmythtv/avformatdecoder.cpp b/mythtv/libs/libmythtv/avformatdecoder.cpp
@@ -1580,19 +1580,6 @@ static enum AVPixelFormat get_format_mediacodec(struct AVCodecContext */*avctx*/
 }
 #endif
 
-
-static bool IS_DR1_PIX_FMT(const enum AVPixelFormat fmt)
-{
-    switch (fmt)
-    {
-        case AV_PIX_FMT_YUV420P:
-        case AV_PIX_FMT_YUVJ420P:
-            return true;
-        default:
-            return false;
-    }
-}
-
 void AvFormatDecoder::InitVideoCodec(AVStream *stream, AVCodecContext *enc,
                                      bool selectedStream)
 {
@@ -3028,20 +3015,36 @@ void AvFormatDecoder::RemoveAudioStreams()
 
 int get_avf_buffer(struct AVCodecContext *c, AVFrame *pic, int flags)
 {
-    AvFormatDecoder *nd = (AvFormatDecoder *)(c->opaque);
+    AvFormatDecoder *decoder = (AvFormatDecoder *)(c->opaque);
 
-    if (!IS_DR1_PIX_FMT(c->pix_fmt))
+    VideoFrameType type = PixelFormatToFrameType(c->pix_fmt);
+    VideoFrameType* supported = decoder->GetPlayer()->DirectRenderFormats();
+    bool found = false;
+    while (*supported != FMT_NONE)
     {
-        nd->m_directrendering = false;
-        return avcodec_default_get_buffer2(c, pic, flags);
+        if (*supported == type)
+        {
+            found = true;
+            break;
+        }
+        supported++;
     }
-    nd->m_directrendering = true;
 
-    VideoFrame *frame = nd->GetPlayer()->GetNextVideoFrame();
+    if (!found)
+    {
+        decoder->m_directrendering = false;
+        return avcodec_default_get_buffer2(c, pic, flags);
+    }
 
+    decoder->m_directrendering = true;
+    VideoFrame *frame = decoder->GetPlayer()->GetNextVideoFrame();
     if (!frame)
         return -1;
 
+    if (frame->codec != type)
+        if (!decoder->GetPlayer()->ReAllocateFrame(frame, type))
+            return -1;
+
     for (int i = 0; i < 3; i++)
     {
         pic->data[i]     = frame->buf + frame->offsets[i];
@@ -3052,8 +3055,7 @@ int get_avf_buffer(struct AVCodecContext *c, AVFrame *pic, int flags)
     pic->reordered_opaque = c->reordered_opaque;
 
     // Set release method
-    AVBufferRef *buffer =
-        av_buffer_create((uint8_t*)frame, 0, release_avf_buffer, nd, 0);
+    AVBufferRef *buffer = av_buffer_create((uint8_t*)frame, 0, release_avf_buffer, decoder, 0);
     pic->buf[0] = buffer;
 
     return 0;

diff --git a/mythtv/libs/libmythtv/mythframe.h b/mythtv/libs/libmythtv/mythframe.h
@@ -187,6 +187,12 @@ static inline void init(VideoFrame *vf, VideoFrameType _codec,
             vf->pitches[0] = width_aligned;
             vf->pitches[1] = vf->pitches[2] = (width_aligned+1) >> 1;
         }
+        else if (FMT_YUV420P10 == _codec || FMT_YUV420P12 == _codec ||
+                 FMT_YUV420P16 == _codec)
+        {
+            vf->pitches[0] = width_aligned << 1;
+            vf->pitches[1] = vf->pitches[2] = width_aligned;
+        }
         else if (FMT_NV12 == _codec)
         {
             vf->pitches[0] = width_aligned;
@@ -213,6 +219,13 @@ static inline void init(VideoFrame *vf, VideoFrameType _codec,
             vf->offsets[2] =
                 vf->offsets[1] + ((width_aligned+1) >> 1) * ((_height+1) >> 1);
         }
+        else if (FMT_YUV420P10 == _codec || FMT_YUV420P12 == _codec ||
+                 FMT_YUV420P16 == _codec)
+        {
+            vf->offsets[0] = 0;
+            vf->offsets[1] = (width_aligned << 1) * _height;
+            vf->offsets[2] = vf->offsets[1] + (width_aligned * (_height >> 1));
+        }
         else if (FMT_YUV422P == _codec)
         {
             vf->offsets[0] = 0;

diff --git a/mythtv/libs/libmythtv/mythplayer.cpp b/mythtv/libs/libmythtv/mythplayer.cpp
@@ -1148,6 +1148,23 @@ int MythPlayer::GetFreeVideoFrames(void) const
     return 0;
 }
 
+/// \brief Return a list of frame types that can be rendered directly.
+VideoFrameType* MythPlayer::DirectRenderFormats(void)
+{
+    static VideoFrameType defaultformats[] = { FMT_YV12, FMT_NONE };
+    if (videoOutput)
+        return videoOutput->DirectRenderFormats();
+    return &defaultformats[0];
+}
+
+/// \brief Reallocate the given from as Type
+bool MythPlayer::ReAllocateFrame(VideoFrame *Frame, VideoFrameType Type)
+{
+    if (videoOutput)
+        return videoOutput->ReAllocateFrame(Frame, Type);
+    return false;
+}
+
 /**
  *  \brief Removes a frame from the available queue for decoding onto.
  *

diff --git a/mythtv/libs/libmythtv/mythplayer.h b/mythtv/libs/libmythtv/mythplayer.h
@@ -263,6 +263,8 @@ class MTV_PUBLIC MythPlayer
     void SetCutList(const frm_dir_map_t &newCutList);
 
     // Decoder stuff..
+    VideoFrameType* DirectRenderFormats(void);
+    bool ReAllocateFrame(VideoFrame *Frame, VideoFrameType Type);
     VideoFrame *GetNextVideoFrame(void);
     VideoFrame *GetRawVideoFrame(long long frameNumber = -1);
     VideoFrame *GetCurrentFrame(int &w, int &h);

diff --git a/mythtv/libs/libmythtv/mythvideotexture.cpp b/mythtv/libs/libmythtv/mythvideotexture.cpp
@@ -159,7 +159,17 @@ vector<MythVideoTexture*> MythVideoTexture::CreateSoftwareTextures(MythRenderOpe
                 if (plane > 0)
                     size = QSize(size.width() >> 1, size.height() >> 1);
                 texture = CreateTexture(Context, size, Target,
-                                        QOpenGLTexture::UInt8,  QOpenGLTexture::Red);
+                                        QOpenGLTexture::UInt8, QOpenGLTexture::Red);
+                break;
+            case FMT_YUV420P10:
+            case FMT_YUV420P12:
+            case FMT_YUV420P16:
+                if (plane > 0)
+                    size = QSize(size.width() >> 1, size.height() >> 1);
+                texture = CreateTexture(Context, size, Target,
+                                        QOpenGLTexture::UInt8, QOpenGLTexture::RG,
+                                        QOpenGLTexture::Linear, QOpenGLTexture::ClampToEdge,
+                                        QOpenGLTexture::RG16_UNorm);
                 break;
             case FMT_YUY2:
                 size.setWidth(size.width() >> 1);
@@ -172,7 +182,7 @@ vector<MythVideoTexture*> MythVideoTexture::CreateSoftwareTextures(MythRenderOpe
                 if (plane == 0)
                 {
                     texture = CreateTexture(Context, size, Target,
-                                            QOpenGLTexture::UInt8,  QOpenGLTexture::Red);
+                                            QOpenGLTexture::UInt8, QOpenGLTexture::Red);
                 }
                 else
                 {
@@ -256,6 +266,17 @@ void MythVideoTexture::UpdateTextures(MythRenderOpenGL *Context,
                 }
                 break;
             }
+            case FMT_YUV420P10:
+            case FMT_YUV420P12:
+            case FMT_YUV420P16:
+                switch (texture->m_frameFormat)
+                {
+                    case FMT_YUV420P10:
+                    case FMT_YUV420P12:
+                    case FMT_YUV420P16: YV12ToYV12(Context, Frame, texture, i); break;
+                    default: break;
+                }
+                break;
             case FMT_NV12:
             {
                 switch (texture->m_frameFormat)
@@ -331,7 +352,10 @@ MythVideoTexture* MythVideoTexture::CreateTexture(MythRenderOpenGL *Context,
 inline void MythVideoTexture::YV12ToYV12(MythRenderOpenGL *Context, const VideoFrame *Frame,
                                          MythVideoTexture *Texture, int Plane)
 {
-    Context->glPixelStorei(GL_UNPACK_ROW_LENGTH, Frame->pitches[Plane]);
+    int pitch = Frame->pitches[Plane];
+    if (Frame->codec != FMT_YV12)
+        pitch = pitch >> 1;
+    Context->glPixelStorei(GL_UNPACK_ROW_LENGTH, pitch);
     Texture->m_texture->setData(Texture->m_pixelFormat, Texture->m_pixelType,
                                 static_cast<uint8_t*>(Frame->buf) + Frame->offsets[Plane]);
     Texture->m_valid = true;

diff --git a/mythtv/libs/libmythtv/openglvideo.cpp b/mythtv/libs/libmythtv/openglvideo.cpp
@@ -116,12 +116,15 @@ void OpenGLVideo::UpdateShaderParameters(void)
                          maxheight - lineheight,                          /* maxheight  */
                          m_inputTextureSize.height() / 2.0f               /* fieldsize  */);
 
+    int range = (1 << ColorDepth(m_inputType)) - 1;
+    QVector2D scaler = QVector2D(1.0, 256.0) *255.0 / (float)range;
     for (int i = Progressive; i < ShaderCount; ++i)
     {
         if (m_shaders[i])
         {
             m_render->EnableShaderProgram(m_shaders[i]);
             m_shaders[i]->setUniformValue("m_frameData", parameters);
+            m_shaders[i]->setUniformValue("m_scaler", scaler);
         }
     }
 }
@@ -273,8 +276,11 @@ bool OpenGLVideo::CreateVideoShader(VideoShaderType Type, QString Deinterlacer)
     {
         switch (m_outputType)
         {
-            case FMT_NV12:  fragment = NV12FragmentShader;    cost = 2; break;
+            case FMT_YUV420P10:
+            case FMT_YUV420P12:
+            case FMT_YUV420P16:
             case FMT_YV12:  fragment = YV12RGBFragmentShader; cost = 3; break;
+            case FMT_NV12:  fragment = NV12FragmentShader;    cost = 2; break;
             case FMT_YUY2:
             case FMT_YUYVHQ:
             default:        fragment = YUV2RGBFragmentShader; cost = 1; break;
@@ -283,7 +289,8 @@ bool OpenGLVideo::CreateVideoShader(VideoShaderType Type, QString Deinterlacer)
     else
     {
         uint bottom = (InterlacedBot == Type);
-        if (FMT_YV12 == m_outputType)
+        if (FMT_YV12 == m_outputType || FMT_YUV420P10 == m_outputType ||
+            FMT_YUV420P12 == m_outputType || FMT_YUV420P16 == m_outputType)
         {
             if (Deinterlacer == "openglonefield" || Deinterlacer == "openglbobdeint")
             {
@@ -355,6 +362,10 @@ bool OpenGLVideo::CreateVideoShader(VideoShaderType Type, QString Deinterlacer)
         }
     }
 
+    // Set correct YV12 sampler
+    int depth = ColorDepth(m_inputType);
+    fragment.replace("%YV12SAMPLER%", depth > 8 ? SampleYV12HDR : SampleYV12);
+
     fragment.replace("SELECT_COLUMN", (FMT_YUY2 == m_outputType) ? SelectColumn : "");
     // update packing code so this can be removed
     fragment.replace("%SWIZZLE%", (FMT_YUY2 == m_outputType) ? "arb" : "abr");
@@ -488,8 +499,7 @@ void OpenGLVideo::ProcessFrame(const VideoFrame *Frame)
 
     // Can we render this frame format - ideally this should be a check
     // against format_is_yuv.
-    if ((Frame->codec != FMT_YV12) && (Frame->codec != FMT_YUY2) &&
-        (Frame->codec != FMT_NV12))
+    if (!format_is_yuv(Frame->codec) || Frame->codec == FMT_YUV422P)
     {
         LOG(VB_GENERAL, LOG_ERR, LOC + "Frame format is not supported");
         return;

diff --git a/mythtv/libs/libmythtv/openglvideoshaders.h b/mythtv/libs/libmythtv/openglvideoshaders.h
@@ -284,15 +284,26 @@ static const QString DefaultFragmentShader =
 "    gl_FragColor = vec4(color.rgb, 1.0);\n"
 "}\n";
 
-#define SAMPLEYVU "\
-highp vec3 sampleYVU(in sampler2D texture1, in sampler2D texture2, in sampler2D texture3, highp vec2 texcoord)\n\
-{\n\
-    highp vec3 yvu;\n\
-    yvu.r = texture2D(texture1, texcoord).r;\n\
-    yvu.g = texture2D(texture2, texcoord).r;\n\
-    yvu.b = texture2D(texture3, texcoord).r;\n\
-    return yvu;\n\
-}\n"
+static const QString SampleYV12 =
+"highp vec3 sampleYVU(in sampler2D texture1, in sampler2D texture2, in sampler2D texture3, highp vec2 texcoord)\n"
+"{\n"
+"    highp vec3 yvu;\n"
+"    yvu.r = texture2D(texture1, texcoord).r;\n"
+"    yvu.g = texture2D(texture2, texcoord).r;\n"
+"    yvu.b = texture2D(texture3, texcoord).r;\n"
+"    return yvu;\n"
+"}\n";
+
+static const QString SampleYV12HDR =
+"uniform highp vec2 m_scaler;\n"
+"highp vec3 sampleYVU(in sampler2D texture1, in sampler2D texture2, in sampler2D texture3, highp vec2 texcoord)\n"
+"{\n"
+"    highp vec3 yvu;\n"
+"    yvu.r = dot(texture2D(texture1, texcoord).rg, m_scaler);\n"
+"    yvu.g = dot(texture2D(texture2, texcoord).rg, m_scaler);\n"
+"    yvu.b = dot(texture2D(texture3, texcoord).rg, m_scaler);\n"
+"    return yvu;\n"
+"}\n";
 
 static const QString YV12RGBFragmentShader =
 "//YV12RGBFragmentShader\n"
@@ -302,7 +313,7 @@ static const QString YV12RGBFragmentShader =
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 "void main(void)\n"
 "{\n"
 "    highp vec3 yvu = sampleYVU(s_texture0, s_texture1, s_texture2, v_texcoord0);\n"
@@ -317,7 +328,7 @@ static const QString YV12RGBOneFieldFragmentShader[2] = {
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 "void main(void)\n"
 "{\n"
 "    highp float field  = min(v_texcoord0.y + (step(0.5, fract(v_texcoord0.y * m_frameData.w))) * m_frameData.x, m_frameData.z);\n"
@@ -332,7 +343,7 @@ SAMPLEYVU
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 "void main(void)\n"
 "{\n"
 "    highp float field  = max(v_texcoord0.y + (step(0.5, 1.0 - fract(v_texcoord0.y * m_frameData.w))) * m_frameData.x, 0.0);\n"
@@ -349,7 +360,7 @@ static const QString YV12RGBLinearBlendFragmentShader[2] = {
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 "void main(void)\n"
 "{\n"
 "    highp vec3 current = sampleYVU(s_texture0, s_texture1, s_texture2, v_texcoord0);\n"
@@ -369,7 +380,7 @@ SAMPLEYVU
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 "void main(void)\n"
 "{\n"
 "    highp vec3 current = sampleYVU(s_texture0, s_texture1, s_texture2, v_texcoord0);\n"
@@ -419,7 +430,7 @@ static const QString YV12RGBKernelShader[2] = {
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 KERNELYVU
 "void main(void)\n"
 "{\n"
@@ -439,7 +450,7 @@ KERNELYVU
 "uniform highp mat4 m_colourMatrix;\n"
 "uniform highp vec4 m_frameData;\n"
 "varying highp vec2 v_texcoord0;\n"
-SAMPLEYVU
+"%YV12SAMPLER%"
 KERNELYVU
 "void main(void)\n"
 "{\n"