Skip to content

Commit

Permalink
AvFormatDecoder: Refactor retrieval of hardware frames
Browse files Browse the repository at this point in the history 
- move retrieval into MythCodecContext and its subclasses
- mark certain codecs as not direct rendering (strictly speaking they
aren't) but ensure they are not release early by VideoBuffers (as we
still need to retain the buffers)
  • Loading branch information
@mark-kendall
mark-kendall committed Jul 8, 2019
1 parent db4d63c commit da2bcc8
Show file tree
Hide file tree
Showing 16 changed files with 318 additions and 271 deletions.
297 changes: 102 additions & 195 deletions mythtv/libs/libmythtv/decoders/avformatdecoder.cpp
View file

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions mythtv/libs/libmythtv/decoders/avformatdecoder.h
View file
Expand Up @@ -230,8 +230,7 @@ class AvFormatDecoder : public DecoderBase
int H264PreProcessPkt(AVStream *stream, AVPacket *pkt);
bool PreProcessVideoPacket(AVStream *stream, AVPacket *pkt);
virtual bool ProcessVideoPacket(AVStream *stream, AVPacket *pkt, bool &Retry);
virtual bool ProcessVideoFrame(AVStream *stream, AVFrame *mpa_pic);
AVPixelFormat GetBestVideoFormat(AVPixelFormat* Formats);
virtual bool ProcessVideoFrame(AVStream *Stream, AVFrame *AvFrame);
bool ProcessAudioPacket(AVStream *stream, AVPacket *pkt,
DecodeType decodetype);
bool ProcessSubtitlePacket(AVStream *stream, AVPacket *pkt);
Expand Down
18 changes: 17 additions & 1 deletion mythtv/libs/libmythtv/decoders/decoderbase.cpp
View file
Expand Up @@ -23,7 +23,7 @@ DecoderBase::DecoderBase(MythPlayer *parent, const ProgramInfo &pginfo)
// language preference
m_languagePreference(iso639_get_language_key_list()),
// this will be deleted and recreated once decoder is set up
m_mythcodecctx(new MythCodecContext(kCodec_NONE))
m_mythcodecctx(new MythCodecContext(this, kCodec_NONE))
{
ResetTracks();
m_tracks[kTrackTypeAudio].push_back(StreamInfo(0, 0, 0, 0, 0));
Expand Down Expand Up @@ -1388,5 +1388,21 @@ DecoderBase::TranslatePositionRelToAbs(const frm_dir_map_t &deleteMap,
return relPosition + addition;
}

/*! \brief Find a suitable frame format that is mutually acceptable to the decoder and render device.
* \note Most hardware decoders will only provide one software frame format.
* \note Currently only the OpenGL renderer supports anything other than FMT_YV12/AV_PIX_FMT_YUV420P
*/
AVPixelFormat DecoderBase::GetBestVideoFormat(AVPixelFormat* Formats)
{
if (m_parent)
{
VideoFrameType* mythfmts = m_parent->DirectRenderFormats();
for (AVPixelFormat *format = Formats; *format != AV_PIX_FMT_NONE; format++)
for (VideoFrameType* mythfmt = mythfmts; *mythfmt != FMT_NONE; mythfmt++)
if (FrameTypeToPixelFormat(*mythfmt) == *format)
return *format;
}
return AV_PIX_FMT_NONE;
}

/* vim: set expandtab tabstop=4 shiftwidth=4: */
1 change: 1 addition & 0 deletions mythtv/libs/libmythtv/decoders/decoderbase.h
View file
Expand Up @@ -267,6 +267,7 @@ class DecoderBase
int GetfpsMultiplier(void) { return m_fpsMultiplier; }
MythCodecContext *GetMythCodecContext(void) { return m_mythcodecctx; }
VideoDisplayProfile * GetVideoDisplayProfile(void) { return &m_videoDisplayProfile; }
AVPixelFormat GetBestVideoFormat(AVPixelFormat* Formats);

protected:
virtual int AutoSelectTrack(uint type);
Expand Down
110 changes: 83 additions & 27 deletions mythtv/libs/libmythtv/decoders/mythcodeccontext.cpp
View file
Expand Up @@ -47,39 +47,40 @@

#define LOC QString("MythCodecContext: ")

MythCodecContext::MythCodecContext(MythCodecID CodecID)
: m_codecID(CodecID)
MythCodecContext::MythCodecContext(DecoderBase *Parent, MythCodecID CodecID)
: m_parent(Parent),
m_codecID(CodecID)
{
}

// static
MythCodecContext *MythCodecContext::CreateContext(MythCodecID Codec)
MythCodecContext *MythCodecContext::CreateContext(DecoderBase *Parent, MythCodecID Codec)
{
MythCodecContext *mctx = nullptr;
#ifdef USING_VAAPI
if (codec_is_vaapi(Codec) || codec_is_vaapi_dec(Codec))
mctx = new MythVAAPIContext(Codec);
mctx = new MythVAAPIContext(Parent, Codec);
#endif
#ifdef USING_VDPAU
if (codec_is_vdpau_hw(Codec) || codec_is_vdpau_hw(Codec))
mctx = new MythVDPAUContext(Codec);
mctx = new MythVDPAUContext(Parent, Codec);
#endif
#ifdef USING_NVDEC
if (codec_is_nvdec_dec(Codec) || codec_is_nvdec(Codec))
mctx = new MythNVDECContext(Codec);
mctx = new MythNVDECContext(Parent, Codec);
#endif
#ifdef USING_VTB
if (codec_is_vtb_dec(Codec) || codec_is_vtb(Codec))
mctx = new MythVTBContext(Codec);
mctx = new MythVTBContext(Parent, Codec);
#endif
#ifdef USING_MEDIACODEC
if (codec_is_mediacodec(Codec) || codec_is_mediacodec_dec(Codec))
mctx = new MythMediaCodecContext(Codec);
mctx = new MythMediaCodecContext(Parent, Codec);
#endif
Q_UNUSED(Codec);

if (!mctx)
mctx = new MythCodecContext(Codec);
mctx = new MythCodecContext(Parent, Codec);
return mctx;
}

Expand Down Expand Up @@ -135,44 +136,44 @@ int MythCodecContext::GetBuffer(struct AVCodecContext *Context, AVFrame *Frame,


/// \brief A generic hardware buffer initialisation method when AVHWFramesContext is NOT used.
int MythCodecContext::GetBuffer2(struct AVCodecContext *Context, AVFrame *Frame, int)
bool MythCodecContext::GetBuffer2(struct AVCodecContext *Context, VideoFrame* Frame,
AVFrame *AvFrame, int)
{
if (!Frame || !Context)
return -1;
if (!AvFrame || !Context || !Frame)
return false;

AvFormatDecoder *avfd = static_cast<AvFormatDecoder*>(Context->opaque);
VideoFrame *videoframe = avfd->GetPlayer()->GetNextVideoFrame();

Frame->opaque = videoframe;
videoframe->pix_fmt = Context->pix_fmt;
Frame->reordered_opaque = Context->reordered_opaque;
Frame->pix_fmt = Context->pix_fmt;
Frame->directrendering = 1;
Frame->colorshifted = 1;

// N.B. this is untested (VideoToolBox and MediaCodec)
videoframe->colorshifted = 1;
AvFrame->reordered_opaque = Context->reordered_opaque;
AvFrame->opaque = Frame;

// retrieve the software format
if (Frame->hw_frames_ctx)
if (AvFrame->hw_frames_ctx)
{
AVHWFramesContext *context = reinterpret_cast<AVHWFramesContext*>(Frame->hw_frames_ctx->data);
AVHWFramesContext *context = reinterpret_cast<AVHWFramesContext*>(AvFrame->hw_frames_ctx->data);
if (context)
videoframe->sw_pix_fmt = context->sw_format;
Frame->sw_pix_fmt = context->sw_format;
}

// the hardware surface is stored in Frame->data[3]
videoframe->buf = Frame->data[3];
Frame->buf = AvFrame->data[3];

// Frame->buf[0] contains the release method. Take another reference to
// ensure the frame is not released before it is displayed.
videoframe->priv[0] = reinterpret_cast<unsigned char*>(av_buffer_ref(Frame->buf[0]));
Frame->priv[0] = reinterpret_cast<unsigned char*>(av_buffer_ref(AvFrame->buf[0]));

// Retrieve and set the interop class
AVHWDeviceContext *devicectx = reinterpret_cast<AVHWDeviceContext*>(Context->hw_device_ctx->data);
videoframe->priv[1] = reinterpret_cast<unsigned char*>(devicectx->user_opaque);
Frame->priv[1] = reinterpret_cast<unsigned char*>(devicectx->user_opaque);

// Set release method
Frame->buf[1] = av_buffer_create(reinterpret_cast<uint8_t*>(videoframe), 0,
MythCodecContext::ReleaseBuffer, avfd, 0);
return 0;
AvFrame->buf[1] = av_buffer_create(reinterpret_cast<uint8_t*>(Frame), 0,
MythCodecContext::ReleaseBuffer, avfd, 0);
return true;
}

void MythCodecContext::ReleaseBuffer(void *Opaque, uint8_t *Data)
Expand Down Expand Up @@ -289,3 +290,58 @@ int MythCodecContext::FilteredReceiveFrame(AVCodecContext *Context, AVFrame *Fra
{
return avcodec_receive_frame(Context, Frame);
}

bool MythCodecContext::RetrieveHWFrame(VideoFrame *Frame, AVFrame *AvFrame)
{
if (!Frame || !AvFrame)
return false;

AVFrame *temp = av_frame_alloc();
if (!temp)
return false;

AVPixelFormat *pixelformats = nullptr;
int ret = av_hwframe_transfer_get_formats(AvFrame->hw_frames_ctx,
AV_HWFRAME_TRANSFER_DIRECTION_FROM,
&pixelformats, 0);
if (ret == 0)
{
AVPixelFormat best = m_parent->GetBestVideoFormat(pixelformats);
if (best != AV_PIX_FMT_NONE)
{
VideoFrameType type = PixelFormatToFrameType(best);
bool valid = Frame->codec == type;
if (!valid || (Frame->width != AvFrame->width) || (Frame->height != AvFrame->height))
valid = VideoBuffers::ReinitBuffer(Frame, type, m_parent->GetVideoCodecID(),
AvFrame->width, AvFrame->height);

if (valid)
{
// Retrieve the picture directly into the VideoFrame Buffer
temp->format = best;
uint max = planes(Frame->codec);
for (uint i = 0; i < 3; i++)
{
temp->data[i] = (i < max) ? (Frame->buf + Frame->offsets[i]) : nullptr;
temp->linesize[i] = Frame->pitches[i];
}

// Dummy release method - we do not want to free the buffer
temp->buf[0] = av_buffer_create(reinterpret_cast<uint8_t*>(Frame), 0,
[](void*, uint8_t*){}, this, 0);
temp->width = AvFrame->width;
temp->height = AvFrame->height;
}
}
}
av_freep(&pixelformats);

// retrieve data from GPU to CPU
if (ret >= 0)
if ((ret = av_hwframe_transfer_data(temp, AvFrame, 0)) < 0)
LOG(VB_GENERAL, LOG_ERR, LOC + QString("Error %1 transferring the data to system memory").arg(ret));

Frame->colorshifted = 1;
av_frame_free(&temp);
return ret >= 0;
}
16 changes: 11 additions & 5 deletions mythtv/libs/libmythtv/decoders/mythcodeccontext.h
View file
Expand Up @@ -34,6 +34,7 @@
#include "mythtvexp.h"
#include "mythcodecid.h"
#include "mythframe.h"
#include "decoderbase.h"

typedef int (*CreateHWDecoder)(AVCodecContext *Context);

Expand All @@ -43,12 +44,13 @@ class VideoDisplayProfile;
class MTV_PUBLIC MythCodecContext
{
public:
explicit MythCodecContext(MythCodecID CodecID);
explicit MythCodecContext(DecoderBase *Parent, MythCodecID CodecID);
virtual ~MythCodecContext() = default;

static MythCodecContext* CreateContext (MythCodecID Codec);
static MythCodecContext* CreateContext (DecoderBase *Parent, MythCodecID Codec);
static int GetBuffer (struct AVCodecContext *Context, AVFrame *Frame, int Flags);
static int GetBuffer2 (struct AVCodecContext *Context, AVFrame *Frame, int Flags);
static bool GetBuffer2 (struct AVCodecContext *Context, VideoFrame *Frame,
AVFrame *AvFrame, int Flags);
static int InitialiseDecoder (AVCodecContext *Context, CreateHWDecoder Callback, const QString &Debug);
static int InitialiseDecoder2 (AVCodecContext *Context, CreateHWDecoder Callback, const QString &Debug);
static void ReleaseBuffer (void *Opaque, uint8_t *Data);
Expand All @@ -59,14 +61,18 @@ class MTV_PUBLIC MythCodecContext
static AVBufferRef* CreateDevice (AVHWDeviceType Type, const QString &Device = QString());

virtual int HwDecoderInit (AVCodecContext*) { return 0; }
virtual bool RetrieveFrame (AVCodecContext*, VideoFrame*, AVFrame*) { return false; }
virtual int FilteredReceiveFrame (AVCodecContext *Context, AVFrame *Frame);
virtual void SetDeinterlacing (AVCodecContext*, VideoDisplayProfile*, bool) {}
virtual void PostProcessFrame (AVCodecContext*, VideoFrame*) {}
virtual bool IsDeinterlacing (bool &, bool = false) { return false; }

protected:
static void DestroyInterop (MythOpenGLInterop *Interop);
MythCodecID m_codecID;
virtual bool RetrieveHWFrame (VideoFrame* Frame, AVFrame* AvFrame);
static void DestroyInterop (MythOpenGLInterop *Interop);

DecoderBase* m_parent;
MythCodecID m_codecID;
};

#endif // MYTHCODECCONTEXT_H
13 changes: 11 additions & 2 deletions mythtv/libs/libmythtv/mythmediacodeccontext.cpp
View file
Expand Up @@ -86,8 +86,8 @@ MythCodecID MythMediaCodecContext::GetBestSupportedCodec(AVCodecContext*,
return failure;
}

MythMediaCodecContext::MythMediaCodecContext(MythCodecID CodecID)
: MythCodecContext(CodecID)
MythMediaCodecContext::MythMediaCodecContext(DecoderBase *Parent, MythCodecID CodecID)
: MythCodecContext(Parent, CodecID)
{
}

Expand All @@ -100,6 +100,15 @@ int MythMediaCodecContext::HwDecoderInit(AVCodecContext *Context)
return -1;
}

bool MythMediaCodecContext::RetrieveFrame(AVCodecContext *Context, VideoFrame *Frame, AVFrame *AvFrame)
{
if (AvFrame->format != AV_PIX_FMT_MEDIACODEC)
return false;
if (codec_is_mediacodec(m_codecID))
return GetBuffer2(Context, Frame, AvFrame, 0);
return false;
}

AVPixelFormat MythMediaCodecContext::GetFormat(AVCodecContext*, const AVPixelFormat *PixFmt)
{
while (*PixFmt != AV_PIX_FMT_NONE)
Expand Down
3 changes: 2 additions & 1 deletion mythtv/libs/libmythtv/mythmediacodeccontext.h
View file
Expand Up @@ -15,8 +15,9 @@ class MythMediaCodecContext : public MythCodecContext
{
public:
// MythCodecContext
MythMediaCodecContext(MythCodecID CodecID);
MythMediaCodecContext(DecoderBase *Parent, MythCodecID CodecID);
int HwDecoderInit(AVCodecContext *Context) override;
bool RetrieveFrame(AVCodecContext *Context, VideoFrame *Frame, AVFrame *AvFrame) override;

static MythCodecID GetBestSupportedCodec(AVCodecContext*,
AVCodec **Codec,
Expand Down

0 comments on commit da2bcc8

Please sign in to comment.