NVDEC: Fix handling of stream changes

- the NVDEC decoder would fail if the stream resolution increased
- as a workaroud, allocate our 'own' frames context - which for some
reason the decoder is happy to release at the correct times and the
decoder continues without issue
- as a side effect, we now need to treat nvdec like VAAPI and VDPAU
(which also use an allocated frames context) and release the pause frame
when seeking/flushing.

mythtv/libs/libmythtv/decoders/avformatdecoder.cpp

@@ -3170,6 +3170,10 @@ void AvFormatDecoder::MpegPreProcessPkt(AVStream *stream, AVPacket *pkt)
                                          forceaspectchange, 2,
                                          decoderdeint ? kScan_Progressive : kScan_Ignore);
 
+                if (context->hw_frames_ctx)
+                    if (context->internal)
+                        avcodec_flush_buffers(context);
+
                 m_currentWidth   = width;
                 m_currentHeight  = height;
                 m_fps            = seqFPS;
@@ -3285,7 +3289,8 @@ int AvFormatDecoder::H264PreProcessPkt(AVStream *stream, AVPacket *pkt)
             // TODO check what is needed here when a device context is used
             // TODO check whether any codecs need to be flushed for a frame rate change (e.g. mediacodec?)
             if (context->hw_frames_ctx && (forcechange || res_changed))
-                avcodec_flush_buffers(context);
+                if (context->internal)
+                    avcodec_flush_buffers(context);
 
             m_currentWidth  = width;
             m_currentHeight = height;

mythtv/libs/libmythtv/decoders/mythnvdeccontext.cpp

@@ -338,6 +338,11 @@ void MythNVDECContext::PostProcessFrame(AVCodecContext* /*Context*/, VideoFrame
     }
 }
 
+bool MythNVDECContext::DecoderWillResetOnFlush(void)
+{
+    return codec_is_nvdec(m_codecID);
+}
+
 bool MythNVDECContext::IsDeinterlacing(bool &DoubleRate, bool /*unused*/)
 {
     if (m_deinterlacer != DEINT_NONE)
@@ -349,12 +354,39 @@ bool MythNVDECContext::IsDeinterlacing(bool &DoubleRate, bool /*unused*/)
     return false;
 }
 
-enum AVPixelFormat MythNVDECContext::GetFormat(AVCodecContext* /*Contextconst*/, const AVPixelFormat *PixFmt)
+enum AVPixelFormat MythNVDECContext::GetFormat(AVCodecContext* Context, const AVPixelFormat *PixFmt)
 {
     while (*PixFmt != AV_PIX_FMT_NONE)
     {
         if (*PixFmt == AV_PIX_FMT_CUDA)
+        {
+            // the mpeg2 decoder in particular seems to call GetFormat spontaneously
+            // i.e. not during an apparent stream change. In this case we need
+            // to release the video buffers otherwise they try and release deceased
+            // decoder context
+            AvFormatDecoder* decoder = reinterpret_cast<AvFormatDecoder*>(Context->opaque);
+            if (decoder && decoder->GetPlayer() && codec_is_nvdec(decoder->GetVideoCodecID()))
+                decoder->GetPlayer()->DiscardVideoFrames(true, true);
+
+            AVBufferRef* framesref = av_hwframe_ctx_alloc(Context->hw_device_ctx);
+            AVHWFramesContext *frames = reinterpret_cast<AVHWFramesContext*>(framesref->data);
+            frames->free = MythCodecContext::FramesContextFinished;
+            frames->user_opaque = nullptr;
+            frames->sw_format = Context->sw_pix_fmt;
+            frames->format    = AV_PIX_FMT_CUDA;
+            frames->width     = Context->coded_width;
+            frames->height    = Context->coded_height;
+            if (av_hwframe_ctx_init(framesref) < 0)
+            {
+                av_buffer_unref(&framesref);
+            }
+            else
+            {
+                Context->hw_frames_ctx = framesref;
+                NewHardwareFramesContext();
+            }
             return AV_PIX_FMT_CUDA;
+        }
         PixFmt++;
     }
     return AV_PIX_FMT_NONE;
@@ -602,3 +634,4 @@ const std::vector<MythNVDECContext::MythNVDECCaps> &MythNVDECContext::GetProfile
 
     return s_profiles;
 }
+

mythtv/libs/libmythtv/decoders/mythnvdeccontext.h

@@ -32,12 +32,13 @@ class MythNVDECContext : public MythCodecContext
                                           VideoDisplayProfile *Profile, bool DoubleRate) override;
     void PostProcessFrame                (AVCodecContext *Context, VideoFrame *Frame) override;
     bool IsDeinterlacing                 (bool &DoubleRate, bool StreamChange = false) override;
+    bool DecoderWillResetOnFlush         (void) override;
     static MythCodecID GetSupportedCodec (AVCodecContext **CodecContext,
                                           AVCodec       **Codec,
                                           const QString  &Decoder,
                                           AVStream       *Stream,
                                           uint            StreamType);
-    static enum AVPixelFormat GetFormat  (AVCodecContext *Contextconst, const AVPixelFormat *PixFmt);
+    static enum AVPixelFormat GetFormat  (AVCodecContext *Context, const AVPixelFormat *PixFmt);
     static bool GetBuffer                (AVCodecContext *Context, VideoFrame *Frame,
                                           AVFrame *AvFrame, int Flags);
     static int  InitialiseDecoder        (AVCodecContext *Context);

mythtv/libs/libmythtv/mythframe.h

@@ -79,7 +79,7 @@ static inline int format_is_hw(VideoFrameType Type)
 
 static inline int format_is_hwframes(VideoFrameType Type)
 {
-    return (Type == FMT_VDPAU) || (Type == FMT_VAAPI);
+    return (Type == FMT_VDPAU) || (Type == FMT_VAAPI) || (Type == FMT_NVDEC);
 }
 
 static inline int format_is_420(VideoFrameType Type)

mythtv/libs/libmythtv/opengl/mythopenglvideo.cpp

@@ -606,7 +606,7 @@ void MythOpenGLVideo::PrepareFrame(VideoFrame *Frame, bool TopFieldFirst, FrameS
     bool hwframes = false;
     bool useframebufferimage = false;
 
-    // We lose the pause frame when seeking and using VDPAU or VAAPI direct rendering.
+    // We lose the pause frame when seeking and using VDPAU/VAAPI/NVDEC direct rendering.
     // As a workaround, we always use the resize stage so the last displayed frame
     // should be retained in the Framebuffer used for resizing. If there is
     // nothing to display, then fallback to this framebuffer.

mythtv/libs/libmythtv/videobuffers.cpp

@@ -284,7 +284,7 @@ void VideoBuffers::ReleaseDecoderResources(VideoFrame *Frame)
             av_buffer_unref(&ref);
         Frame->buf = Frame->priv[0] = nullptr;
 
-        if (format_is_hwframes(Frame->codec) || (Frame->codec == FMT_NVDEC))
+        if (format_is_hwframes(Frame->codec))
         {
             ref = reinterpret_cast<AVBufferRef*>(Frame->priv[1]);
             if (ref != nullptr)