Implement SEI parsing for QSV decoders #31
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
/submit |
|
Submitted as pull.31.ffstaging.FFmpeg.1653552529.ffmpegagent@gmail.com To fetch this version into To fetch this version to local tag |
![ffmpeg-codebot[bot]](https://avatars.githubusercontent.com/in/158557?s=60&v=4){kind=small}
|
User |
|
User |
|
/submit |
|
Submitted as pull.31.v2.ffstaging.FFmpeg.1654074366.ffmpegagent@gmail.com To fetch this version into To fetch this version to local tag |
|
/submit |
|
Submitted as pull.31.v3.ffstaging.FFmpeg.1654106506.ffmpegagent@gmail.com To fetch this version into To fetch this version to local tag |
|
On the FFmpeg mailing list, Kieran Kunhya wrote (reply to this): |
|
User |
|
On the FFmpeg mailing list, Soft Works wrote (reply to this): |
|
On the FFmpeg mailing list, Kieran Kunhya wrote (reply to this): |
|
On the FFmpeg mailing list, Soft Works wrote (reply to this): |
softworkz
force-pushed
the
submit_qsv_sei
branch
from
27c3dde
to
3f25885
Compare
|
/submit |
|
Submitted as pull.31.v4.ffstaging.FFmpeg.1656286888.ffmpegagent@gmail.com To fetch this version into To fetch this version to local tag |
|
On the FFmpeg mailing list, "Xiang, Haihao" wrote (reply to this): |
| @@ -49,6 +49,12 @@ | |||
| #include "hwconfig.h" | |||
There was a problem hiding this comment.
On the FFmpeg mailing list, Andreas Rheinhardt wrote (reply to this):
softworkz:
> From: softworkz <softworkz@hotmail.com>
>
> Signed-off-by: softworkz <softworkz@hotmail.com>
> ---
> libavcodec/qsvdec.c | 234 ++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 234 insertions(+)
>
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 5fc5bed4c8..e854f363ec 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -49,6 +49,12 @@
> #include "hwconfig.h"
> #include "qsv.h"
> #include "qsv_internal.h"
> +#include "h264dec.h"
> +#include "h264_sei.h"
> +#include "hevcdec.h"
> +#include "hevc_ps.h"
> +#include "hevc_sei.h"
> +#include "mpeg12.h"
>
> static const AVRational mfx_tb = { 1, 90000 };
>
> @@ -60,6 +66,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> AV_NOPTS_VALUE : pts_tb.num ? \
> av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
>
> +#define PAYLOAD_BUFFER_SIZE 65535
> +
> typedef struct QSVAsyncFrame {
> mfxSyncPoint *sync;
> QSVFrame *frame;
> @@ -101,6 +109,9 @@ typedef struct QSVContext {
>
> mfxExtBuffer **ext_buffers;
> int nb_ext_buffers;
> +
> + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> + Mpeg1Context mpeg_ctx;
> } QSVContext;
>
> static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -599,6 +610,210 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
> return 0;
> }
> #endif
> +static int find_start_offset(mfxU8 data[4])
> +{
> + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> + return 3;
> +
> + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
> + return 4;
> +
> + return 0;
> +}
> +
> +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> + H264SEIContext sei = { 0 };
> + GetBitContext gb = { 0 };
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
> + mfxU64 ts;
> + int ret;
> +
> + while (1) {
> + int start;
> + memset(payload.Data, 0, payload.BufSize);
> +
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + start = find_start_offset(payload.Data);
> +
> + switch (payload.Type) {
> + case SEI_TYPE_BUFFERING_PERIOD:
> + case SEI_TYPE_PIC_TIMING:
> + continue;
> + }
> +
> + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else {
> + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit);
> + }
> + }
> +
> + if (out)
> + return ff_h264_export_frame_props(avctx, &sei, NULL, out);
> +
> + return 0;
> +}
> +
> +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out)
> +{
> + HEVCSEI sei = { 0 };
> + HEVCParamSets ps = { 0 };
> + GetBitContext gb = { 0 };
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
> + mfxFrameSurface1 *surface = &out->surface;
> + mfxU64 ts;
> + int ret, has_logged = 0;
> +
> + while (1) {
> + int start;
> + memset(payload.Data, 0, payload.BufSize);
> +
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + if (!has_logged) {
> + has_logged = 1;
> + av_log(avctx, AV_LOG_VERBOSE, "-----------------------------------------\n");
> + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
> + }
> +
> + if (ts != surface->Data.TimeStamp) {
> + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
> + }
> +
> + start = find_start_offset(payload.Data);
> +
> + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits %3d Start: %d\n", payload.Type, payload.NumBit, start);
> +
> + switch (payload.Type) {
> + case SEI_TYPE_BUFFERING_PERIOD:
> + case SEI_TYPE_PIC_TIMING:
> + continue;
> + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> + // There seems to be a bug in MSDK
> + payload.NumBit -= 8;
> +
> + break;
> + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> + // There seems to be a bug in MSDK
> + payload.NumBit = 48;
> +
> + break;
> + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> + // There seems to be a bug in MSDK
> + if (payload.NumBit == 552)
> + payload.NumBit = 528;
> + break;
> + }
> +
> + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else {
> + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX);
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit);
> + }
> + }
> +
> + if (has_logged) {
> + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> + }
> +
> + if (out && out->frame)
> + return ff_hevc_set_side_data(avctx, &sei, NULL, out->frame);
> +
> + return 0;
> +}
> +
> +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> + Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) };
> + mfxU64 ts;
> + int ret;
> +
> + while (1) {
> + int start;
> +
> + memset(payload.Data, 0, payload.BufSize);
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + start = find_start_offset(payload.Data);
> +
> + start++;
> +
> + ff_mpeg_decode_user_data(avctx, mpeg_ctx, &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
> +
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&payload.Data[start]));
> + }
> +
> + if (!out)
> + return 0;
> +
> + if (mpeg_ctx->a53_buf_ref) {
> +
> + AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
> + if (!sd)
> + av_buffer_unref(&mpeg_ctx->a53_buf_ref);
> + mpeg_ctx->a53_buf_ref = NULL;
> + }
> +
> + if (mpeg_ctx->has_stereo3d) {
> + AVStereo3D *stereo = av_stereo3d_create_side_data(out);
> + if (!stereo)
> + return AVERROR(ENOMEM);
> +
> + *stereo = mpeg_ctx->stereo3d;
> + mpeg_ctx->has_stereo3d = 0;
> + }
> +
> + if (mpeg_ctx->has_afd) {
> + AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD, 1);
> + if (!sd)
> + return AVERROR(ENOMEM);
> +
> + *sd->data = mpeg_ctx->afd;
> + mpeg_ctx->has_afd = 0;
> + }
> +
> + return 0;
> +}
>
> static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame, int *got_frame,
> @@ -636,6 +851,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> insurf, &outsurf, sync);
> if (ret == MFX_WRN_DEVICE_BUSY)
> av_usleep(500);
> + else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> + parse_sei_mpeg12(avctx, q, NULL);
>
> } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
>
> @@ -677,6 +894,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> return AVERROR_BUG;
> }
>
> + switch (avctx->codec_id) {
> + case AV_CODEC_ID_MPEG2VIDEO:
> + ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> + break;
> + case AV_CODEC_ID_H264:
> + ret = parse_sei_h264(avctx, q, out_frame->frame);
> + break;
> + case AV_CODEC_ID_HEVC:
> + ret = parse_sei_hevc(avctx, q, out_frame);
> + break;
> + default:
> + ret = 0;
> + }
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
> +
> out_frame->queued += 1;
>
> aframe = (QSVAsyncFrame){ sync, out_frame };
You completely forgot necessary changes to configure/the Makefile. The
way you are doing it here means that you basically have the qsv decoders
to rely on the H.264/HEVC/MPEG-1/2 decoders which is way too much.
- Andreas
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
There was a problem hiding this comment.
On the FFmpeg mailing list, Soft Works wrote (reply to this):
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Andreas Rheinhardt
> Sent: Tuesday, June 28, 2022 6:17 AM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH v4 6/6] avcodec/qsvdec: Implement
> SEI parsing for QSV decoders
>
> softworkz:
> > From: softworkz <softworkz@hotmail.com>
> >
> > Signed-off-by: softworkz <softworkz@hotmail.com>
> > ---
> > libavcodec/qsvdec.c | 234
> ++++++++++++++++++++++++++++++++++++++++++++
> > 1 file changed, 234 insertions(+)
> >
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 5fc5bed4c8..e854f363ec 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -49,6 +49,12 @@
> > #include "hwconfig.h"
> > #include "qsv.h"
> > #include "qsv_internal.h"
> > +#include "h264dec.h"
> > +#include "h264_sei.h"
> > +#include "hevcdec.h"
> > +#include "hevc_ps.h"
> > +#include "hevc_sei.h"
> > +#include "mpeg12.h"
> >
> > static const AVRational mfx_tb = { 1, 90000 };
> >
> > @@ -60,6 +66,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> > AV_NOPTS_VALUE : pts_tb.num ? \
> > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> >
> > +#define PAYLOAD_BUFFER_SIZE 65535
> > +
> > typedef struct QSVAsyncFrame {
> > mfxSyncPoint *sync;
> > QSVFrame *frame;
> > @@ -101,6 +109,9 @@ typedef struct QSVContext {
> >
> > mfxExtBuffer **ext_buffers;
> > int nb_ext_buffers;
> > +
> > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > + Mpeg1Context mpeg_ctx;
> > } QSVContext;
> >
> > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -599,6 +610,210 @@ static int
> qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
> > return 0;
> > }
> > #endif
> > +static int find_start_offset(mfxU8 data[4])
> > +{
> > + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > + return 3;
> > +
> > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> 1)
> > + return 4;
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > + H264SEIContext sei = { 0 };
> > + GetBitContext gb = { 0 };
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize = sizeof(q->payload_buffer) };
> > + mfxU64 ts;
> > + int ret;
> > +
> > + while (1) {
> > + int start;
> > + memset(payload.Data, 0, payload.BufSize);
> > +
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer), payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + switch (payload.Type) {
> > + case SEI_TYPE_BUFFERING_PERIOD:
> > + case SEI_TYPE_PIC_TIMING:
> > + continue;
> > + }
> > +
> > + if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start * 8) < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > + else {
> > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit,
> ret);
> > + else
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits %d\n", payload.Type, payload.NumBit);
> > + }
> > + }
> > +
> > + if (out)
> > + return ff_h264_export_frame_props(avctx, &sei, NULL, out);
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> QSVFrame* out)
> > +{
> > + HEVCSEI sei = { 0 };
> > + HEVCParamSets ps = { 0 };
> > + GetBitContext gb = { 0 };
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize = sizeof(q->payload_buffer) };
> > + mfxFrameSurface1 *surface = &out->surface;
> > + mfxU64 ts;
> > + int ret, has_logged = 0;
> > +
> > + while (1) {
> > + int start;
> > + memset(payload.Data, 0, payload.BufSize);
> > +
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer), payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + if (!has_logged) {
> > + has_logged = 1;
> > + av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> ------------------\n");
> > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> payload timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> >Data.TimeStamp);
> > + }
> > +
> > + if (ts != surface->Data.TimeStamp) {
> > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> (%llu) does not match surface timestamp: (%llu)\n", ts, surface-
> >Data.TimeStamp);
> > + }
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> Numbits %3d Start: %d\n", payload.Type, payload.NumBit, start);
> > +
> > + switch (payload.Type) {
> > + case SEI_TYPE_BUFFERING_PERIOD:
> > + case SEI_TYPE_PIC_TIMING:
> > + continue;
> > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > + // There seems to be a bug in MSDK
> > + payload.NumBit -= 8;
> > +
> > + break;
> > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > + // There seems to be a bug in MSDK
> > + payload.NumBit = 48;
> > +
> > + break;
> > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > + // There seems to be a bug in MSDK
> > + if (payload.NumBit == 552)
> > + payload.NumBit = 528;
> > + break;
> > + }
> > +
> > + if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start * 8) < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > + else {
> > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> HEVC_NAL_SEI_PREFIX);
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit,
> ret);
> > + else
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits %d\n", payload.Type, payload.NumBit);
> > + }
> > + }
> > +
> > + if (has_logged) {
> > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > + }
> > +
> > + if (out && out->frame)
> > + return ff_hevc_set_side_data(avctx, &sei, NULL, out-
> >frame);
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > + Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize = sizeof(q->payload_buffer) };
> > + mfxU64 ts;
> > + int ret;
> > +
> > + while (1) {
> > + int start;
> > +
> > + memset(payload.Data, 0, payload.BufSize);
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer), payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + start++;
> > +
> > + ff_mpeg_decode_user_data(avctx, mpeg_ctx,
> &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
> > +
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits
> %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char
> *)(&payload.Data[start]));
> > + }
> > +
> > + if (!out)
> > + return 0;
> > +
> > + if (mpeg_ctx->a53_buf_ref) {
> > +
> > + AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
> AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
> > + if (!sd)
> > + av_buffer_unref(&mpeg_ctx->a53_buf_ref);
> > + mpeg_ctx->a53_buf_ref = NULL;
> > + }
> > +
> > + if (mpeg_ctx->has_stereo3d) {
> > + AVStereo3D *stereo = av_stereo3d_create_side_data(out);
> > + if (!stereo)
> > + return AVERROR(ENOMEM);
> > +
> > + *stereo = mpeg_ctx->stereo3d;
> > + mpeg_ctx->has_stereo3d = 0;
> > + }
> > +
> > + if (mpeg_ctx->has_afd) {
> > + AVFrameSideData *sd = av_frame_new_side_data(out,
> AV_FRAME_DATA_AFD, 1);
> > + if (!sd)
> > + return AVERROR(ENOMEM);
> > +
> > + *sd->data = mpeg_ctx->afd;
> > + mpeg_ctx->has_afd = 0;
> > + }
> > +
> > + return 0;
> > +}
> >
> > static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> > AVFrame *frame, int *got_frame,
> > @@ -636,6 +851,8 @@ static int qsv_decode(AVCodecContext *avctx,
> QSVContext *q,
> > insurf, &outsurf,
> sync);
> > if (ret == MFX_WRN_DEVICE_BUSY)
> > av_usleep(500);
> > + else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> > + parse_sei_mpeg12(avctx, q, NULL);
> >
> > } while (ret == MFX_WRN_DEVICE_BUSY || ret ==
> MFX_ERR_MORE_SURFACE);
> >
> > @@ -677,6 +894,23 @@ static int qsv_decode(AVCodecContext *avctx,
> QSVContext *q,
> > return AVERROR_BUG;
> > }
> >
> > + switch (avctx->codec_id) {
> > + case AV_CODEC_ID_MPEG2VIDEO:
> > + ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> > + break;
> > + case AV_CODEC_ID_H264:
> > + ret = parse_sei_h264(avctx, q, out_frame->frame);
> > + break;
> > + case AV_CODEC_ID_HEVC:
> > + ret = parse_sei_hevc(avctx, q, out_frame);
> > + break;
> > + default:
> > + ret = 0;
> > + }
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data:
> %d\n", ret);
> > +
> > out_frame->queued += 1;
> >
> > aframe = (QSVAsyncFrame){ sync, out_frame };
>
> You completely forgot necessary changes to configure/the Makefile.
> The
> way you are doing it here means that you basically have the qsv
> decoders
> to rely on the H.264/HEVC/MPEG-1/2 decoders which is way too much.
You are referring to the hypothetical case where one would disable
one of the sw decoders while having a qsv decoder enabled, right?
The SEI parsing code is not trivial and tied to those decoders
(means using these contexts). It would be not a straightforward
task to extract/separate those parts, that's why I preferred to
just make that functionality accessible.
I wouldn't mind when the QSV decoders would be dependent on those
decoders being included in compilation, even more when considering
that so many other hwaccel decoders have the same dependencies;
DXVA2, D3D11VA, NVDEC, VAAPI.
The question would be whether to not build the qsv decoders when
the sw decoders are deselected or whether to build the sw decoder
code even these are disabled. AFAIU, both would be possible?
Or would you have a better idea?
Thanks,
sw
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
|
User |
|
/submit |
|
Submitted as pull.31.v5.ffstaging.FFmpeg.1656708534.ffmpegagent@gmail.com To fetch this version into To fetch this version to local tag |
|
On the FFmpeg mailing list, "Xiang, Haihao" wrote (reply to this): |
|
On the FFmpeg mailing list, Soft Works wrote (reply to this): |
|
On the FFmpeg mailing list, Andreas Rheinhardt wrote (reply to this): |
|
On the FFmpeg mailing list, Soft Works wrote (reply to this): |
Signed-off-by: softworkz <softworkz@hotmail.com>
Signed-off-by: softworkz <softworkz@hotmail.com>
softworkz
force-pushed
the
submit_qsv_sei
branch
from
19bc00b
to
adb6f6e
Compare
Signed-off-by: softworkz <softworkz@hotmail.com>
softworkz
force-pushed
the
submit_qsv_sei
branch
from
adb6f6e
to
61626eb
Compare
|
/submit |
|
Submitted as pull.31.v6.ffstaging.FFmpeg.1666670616.ffmpegagent@gmail.com To fetch this version into To fetch this version to local tag |
| @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o | |||
| OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o | |||
There was a problem hiding this comment.
On the FFmpeg mailing list, "Xiang, Haihao" wrote (reply to this):
On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> From: softworkz <softworkz@hotmail.com>
>
> Signed-off-by: softworkz <softworkz@hotmail.com>
> ---
> libavcodec/Makefile | 2 +-
> libavcodec/qsvdec.c | 321 ++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 322 insertions(+), 1 deletion(-)
>
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 90c7f113a3..cbddbb0ace 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o
> OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
> OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
> OBJS-$(CONFIG_QSV) += qsv.o
> -OBJS-$(CONFIG_QSVDEC) += qsvdec.o
> +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o
> OBJS-$(CONFIG_QSVENC) += qsvenc.o
> OBJS-$(CONFIG_RANGECODER) += rangecoder.o
> OBJS-$(CONFIG_RDFT) += rdft.o
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 73405b5747..467a248224 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,6 +41,7 @@
> #include "libavutil/time.h"
> #include "libavutil/imgutils.h"
> #include "libavutil/film_grain_params.h"
> +#include <libavutil/reverse.h>
>
> #include "avcodec.h"
> #include "codec_internal.h"
> @@ -49,6 +50,9 @@
> #include "hwconfig.h"
> #include "qsv.h"
> #include "qsv_internal.h"
> +#include "h264_sei.h"
> +#include "hevc_ps.h"
> +#include "hevc_sei.h"
>
> #if QSV_ONEVPL
> #include <mfxdispatcher.h>
> @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> AV_NOPTS_VALUE : pts_tb.num ? \
> av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
>
> +#define PAYLOAD_BUFFER_SIZE 65535
> +
> typedef struct QSVAsyncFrame {
> mfxSyncPoint *sync;
> QSVFrame *frame;
> @@ -107,6 +113,9 @@ typedef struct QSVContext {
>
> mfxExtBuffer **ext_buffers;
> int nb_ext_buffers;
> +
> + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> + AVBufferRef *a53_buf_ref;
> } QSVContext;
>
> static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx,
> mfxExtAV1FilmGrainParam
> }
> #endif
>
> +static int find_start_offset(mfxU8 data[4])
> +{
> + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> + return 3;
> +
> + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
> + return 4;
> +
> + return 0;
> +}
> +
> +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> + H264SEIContext sei = { 0 };
> + GetBitContext gb = { 0 };
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> + mfxU64 ts;
> + int ret;
> +
> + while (1) {
> + int start;
> + memset(payload.Data, 0, payload.BufSize);
> +
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + start = find_start_offset(payload.Data);
> +
> + switch (payload.Type) {
> + case SEI_TYPE_BUFFERING_PERIOD:
> + case SEI_TYPE_PIC_TIMING:
> + continue;
> + }
> +
> + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start *
> 8) < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader
> SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else {
> + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type:
> %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits
> %d\n", payload.Type, payload.NumBit);
> + }
> + }
> +
> + if (out)
> + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0);
> +
> + return 0;
> +}
> +
> +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame*
> out)
> +{
> + HEVCSEI sei = { 0 };
> + HEVCParamSets ps = { 0 };
> + GetBitContext gb = { 0 };
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> + mfxFrameSurface1 *surface = &out->surface;
> + mfxU64 ts;
> + int ret, has_logged = 0;
> +
> + while (1) {
> + int start;
> + memset(payload.Data, 0, payload.BufSize);
> +
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + if (!has_logged) {
> + has_logged = 1;
> + av_log(avctx, AV_LOG_VERBOSE, "--------------------------------
> ---------\n");
> + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload
> timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
> + }
> +
> + if (ts != surface->Data.TimeStamp) {
> + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does
> not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
> + }
> +
> + start = find_start_offset(payload.Data);
> +
> + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits
> %3d Start: %d\n", payload.Type, payload.NumBit, start);
> +
> + switch (payload.Type) {
> + case SEI_TYPE_BUFFERING_PERIOD:
> + case SEI_TYPE_PIC_TIMING:
> + continue;
> + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> + // There seems to be a bug in MSDK
> + payload.NumBit -= 8;
> +
> + break;
> + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> + // There seems to be a bug in MSDK
> + payload.NumBit = 48;
> +
> + break;
> + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> + // There seems to be a bug in MSDK
> + if (payload.NumBit == 552)
> + payload.NumBit = 528;
> + break;
> + }
> +
> + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start *
> 8) < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader
> SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else {
> + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> HEVC_NAL_SEI_PREFIX);
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type:
> %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits
> %d\n", payload.Type, payload.NumBit);
> + }
> + }
> +
> + if (has_logged) {
> + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> + }
> +
> + if (out && out->frame)
> + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx-
> >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
I got segfault when trying your patchset,
Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
777 return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
(gdb) bt
#0 0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
#1 0x00007ffff67c1afe in qsv_decode (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, frame=frame@entry=0x5555556df740, got_frame=got_frame@entry=0x7fffffffd6bc,
avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020
BTW the SDK provides support for hevc HDR metadata, we needn't parse SEI payload
in qsvdec and may get the corresponding info from the SDK, see
https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
Thanks
Haihao
> +
> + return 0;
> +}
> +
> +#define A53_MAX_CC_COUNT 2000
> +
> +static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s,
> + const uint8_t *p, int buf_size)
> +{
> + if (buf_size >= 6 &&
> + p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' &&
> + p[4] == 3 && (p[5] & 0x40)) {
> + /* extract A53 Part 4 CC data */
> + unsigned cc_count = p[5] & 0x1f;
> + if (cc_count > 0 && buf_size >= 7 + cc_count * 3) {
> + const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size :
> 0;
> + const uint64_t new_size = (old_size + cc_count
> + * UINT64_C(3));
> + int ret;
> +
> + if (new_size > 3*A53_MAX_CC_COUNT)
> + return AVERROR(EINVAL);
> +
> + ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> + if (ret >= 0)
> + memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count *
> UINT64_C(3));
> +
> + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> + }
> + return 1;
> + } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) {
> + /* extract SCTE-20 CC data */
> + GetBitContext gb;
> + unsigned cc_count = 0;
> + int ret;
> +
> + init_get_bits8(&gb, p + 2, buf_size - 2);
> + cc_count = get_bits(&gb, 5);
> + if (cc_count > 0) {
> + uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
> + uint64_t new_size = (old_size + cc_count * UINT64_C(3));
> + if (new_size > 3 * A53_MAX_CC_COUNT)
> + return AVERROR(EINVAL);
> +
> + ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> + if (ret >= 0) {
> + uint8_t field, cc1, cc2;
> + uint8_t *cap = s->a53_buf_ref->data;
> +
> + memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3);
> + for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >=
> 26; i++) {
> + skip_bits(&gb, 2); // priority
> + field = get_bits(&gb, 2);
> + skip_bits(&gb, 5); // line_offset
> + cc1 = get_bits(&gb, 8);
> + cc2 = get_bits(&gb, 8);
> + skip_bits(&gb, 1); // marker
> +
> + if (!field) { // forbidden
> + cap[0] = cap[1] = cap[2] = 0x00;
> + } else {
> + field = (field == 2 ? 1 : 0);
> + ////if (!s1->mpeg_enc_ctx.top_field_first) field =
> !field;
> + cap[0] = 0x04 | field;
> + cap[1] = ff_reverse[cc1];
> + cap[2] = ff_reverse[cc2];
> + }
> + cap += 3;
> + }
> + }
> + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> + }
> + return 1;
> + } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01
> && p[3] == 0xf8) {
> + int cc_count = 0;
> + int i, ret;
> + // There is a caption count field in the data, but it is often
> + // incorrect. So count the number of captions present.
> + for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6)
> + cc_count++;
> + // Transform the DVD format into A53 Part 4 format
> + if (cc_count > 0) {
> + int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
> + uint64_t new_size = (old_size + cc_count
> + * UINT64_C(6));
> + if (new_size > 3*A53_MAX_CC_COUNT)
> + return AVERROR(EINVAL);
> +
> + ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> + if (ret >= 0) {
> + uint8_t field1 = !!(p[4] & 0x80);
> + uint8_t *cap = s->a53_buf_ref->data;
> + p += 5;
> + for (i = 0; i < cc_count; i++) {
> + cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd;
> + cap[1] = p[1];
> + cap[2] = p[2];
> + cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd;
> + cap[4] = p[4];
> + cap[5] = p[5];
> + cap += 6;
> + p += 6;
> + }
> + }
> + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> + }
> + return 1;
> + }
> + return 0;
> +}
> +
> +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame*
> out)
> +{
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> + mfxU64 ts;
> + int ret;
> +
> + while (1) {
> + int start;
> +
> + memset(payload.Data, 0, payload.BufSize);
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + start = find_start_offset(payload.Data);
> +
> + start++;
> +
> + mpeg_decode_a53_cc(avctx, q, &payload.Data[start],
> (int)((payload.NumBit + 7) / 8) - start);
> +
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d
> -> %.s\n", payload.Type, payload.NumBit, start, (char
> *)(&payload.Data[start]));
> + }
> +
> + if (!out)
> + return 0;
> +
> + if (q->a53_buf_ref) {
> +
> + AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
> AV_FRAME_DATA_A53_CC, q->a53_buf_ref);
> + if (!sd)
> + av_buffer_unref(&q->a53_buf_ref);
> + q->a53_buf_ref = NULL;
> + }
> +
> + return 0;
> +}
> +
> static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame, int *got_frame,
> const AVPacket *avpkt)
> @@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
> insurf, &outsurf, sync);
> if (ret == MFX_WRN_DEVICE_BUSY)
> av_usleep(500);
> + else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> + parse_sei_mpeg12(avctx, q, NULL);
>
> } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
>
> @@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
> return AVERROR_BUG;
> }
>
> + switch (avctx->codec_id) {
> + case AV_CODEC_ID_MPEG2VIDEO:
> + ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> + break;
> + case AV_CODEC_ID_H264:
> + ret = parse_sei_h264(avctx, q, out_frame->frame);
> + break;
> + case AV_CODEC_ID_HEVC:
> + ret = parse_sei_hevc(avctx, q, out_frame);
> + break;
> + default:
> + ret = 0;
> + }
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
> +
> out_frame->queued += 1;
>
> aframe = (QSVAsyncFrame){ sync, out_frame };
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
There was a problem hiding this comment.
On the FFmpeg mailing list, Soft Works wrote (reply to this):
> -----Original Message-----
> From: Xiang, Haihao <haihao.xiang@intel.com>
> Sent: Monday, November 21, 2022 3:45 AM
> To: ffmpeg-devel@ffmpeg.org
> Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at-
> intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com
> Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> SEI parsing for QSV decoders
>
> On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > From: softworkz <softworkz@hotmail.com>
> >
> > Signed-off-by: softworkz <softworkz@hotmail.com>
> > ---
> > libavcodec/Makefile | 2 +-
> > libavcodec/qsvdec.c | 321
> ++++++++++++++++++++++++++++++++++++++++++++
> > 2 files changed, 322 insertions(+), 1 deletion(-)
> >
> > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > index 90c7f113a3..cbddbb0ace 100644
> > --- a/libavcodec/Makefile
> > +++ b/libavcodec/Makefile
> > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) +=
> mss34dsp.o
> > OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
> > OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
> > OBJS-$(CONFIG_QSV) += qsv.o
> > -OBJS-$(CONFIG_QSVDEC) += qsvdec.o
> > +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o
> hevc_sei.o
> > OBJS-$(CONFIG_QSVENC) += qsvenc.o
> > OBJS-$(CONFIG_RANGECODER) += rangecoder.o
> > OBJS-$(CONFIG_RDFT) += rdft.o
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 73405b5747..467a248224 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -41,6 +41,7 @@
> > #include "libavutil/time.h"
> > #include "libavutil/imgutils.h"
> > #include "libavutil/film_grain_params.h"
> > +#include <libavutil/reverse.h>
> >
> > #include "avcodec.h"
> > #include "codec_internal.h"
> > @@ -49,6 +50,9 @@
> > #include "hwconfig.h"
> > #include "qsv.h"
> > #include "qsv_internal.h"
> > +#include "h264_sei.h"
> > +#include "hevc_ps.h"
> > +#include "hevc_sei.h"
> >
> > #if QSV_ONEVPL
> > #include <mfxdispatcher.h>
> > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> > AV_NOPTS_VALUE : pts_tb.num ? \
> > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> >
> > +#define PAYLOAD_BUFFER_SIZE 65535
> > +
> > typedef struct QSVAsyncFrame {
> > mfxSyncPoint *sync;
> > QSVFrame *frame;
> > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> >
> > mfxExtBuffer **ext_buffers;
> > int nb_ext_buffers;
> > +
> > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > + AVBufferRef *a53_buf_ref;
> > } QSVContext;
> >
> > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -628,6 +637,299 @@ static int
> qsv_export_film_grain(AVCodecContext *avctx,
> > mfxExtAV1FilmGrainParam
> > }
> > #endif
> >
> > +static int find_start_offset(mfxU8 data[4])
> > +{
> > + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > + return 3;
> > +
> > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> 1)
> > + return 4;
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > + H264SEIContext sei = { 0 };
> > + GetBitContext gb = { 0 };
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > + mfxU64 ts;
> > + int ret;
> > +
> > + while (1) {
> > + int start;
> > + memset(payload.Data, 0, payload.BufSize);
> > +
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + switch (payload.Type) {
> > + case SEI_TYPE_BUFFERING_PERIOD:
> > + case SEI_TYPE_PIC_TIMING:
> > + continue;
> > + }
> > +
> > + if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > + else {
> > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > + else
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > + }
> > + }
> > +
> > + if (out)
> > + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> 0);
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> QSVFrame*
> > out)
> > +{
> > + HEVCSEI sei = { 0 };
> > + HEVCParamSets ps = { 0 };
> > + GetBitContext gb = { 0 };
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > + mfxFrameSurface1 *surface = &out->surface;
> > + mfxU64 ts;
> > + int ret, has_logged = 0;
> > +
> > + while (1) {
> > + int start;
> > + memset(payload.Data, 0, payload.BufSize);
> > +
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + if (!has_logged) {
> > + has_logged = 1;
> > + av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> ---------
> > ---------\n");
> > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> payload
> > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> >Data.TimeStamp);
> > + }
> > +
> > + if (ts != surface->Data.TimeStamp) {
> > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> (%llu) does
> > not match surface timestamp: (%llu)\n", ts, surface-
> >Data.TimeStamp);
> > + }
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> Numbits
> > %3d Start: %d\n", payload.Type, payload.NumBit, start);
> > +
> > + switch (payload.Type) {
> > + case SEI_TYPE_BUFFERING_PERIOD:
> > + case SEI_TYPE_PIC_TIMING:
> > + continue;
> > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > + // There seems to be a bug in MSDK
> > + payload.NumBit -= 8;
> > +
> > + break;
> > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > + // There seems to be a bug in MSDK
> > + payload.NumBit = 48;
> > +
> > + break;
> > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > + // There seems to be a bug in MSDK
> > + if (payload.NumBit == 552)
> > + payload.NumBit = 528;
> > + break;
> > + }
> > +
> > + if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > + else {
> > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > HEVC_NAL_SEI_PREFIX);
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > + else
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > + }
> > + }
> > +
> > + if (has_logged) {
> > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > + }
> > +
> > + if (out && out->frame)
> > + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> avctx-
> > >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
>
> I got segfault when trying your patchset,
>
> Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> 0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> >frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
> (gdb) bt
> #0 0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> #1 0x00007ffff67c1afe in qsv_decode
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> frame=frame@entry=0x5555556df740,
> got_frame=got_frame@entry=0x7fffffffd6bc,
> avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020
> BTW the SDK provides support for hevc HDR metadata, we needn't parse
> SEI payload
> in qsvdec and may get the corresponding info from the SDK, see
> https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
I know. I was the one who had requested this to be added to MSDK :-)
But it's just one small part of SEI information, it's limited to
the latest MSDK versions and I'm not sure whether it's working
as reliably as this implementation. This would need to be tested.
You should still have access to the repo with the test files for
demoing the offset problems (which my patchset is working around).
But it's also about dynamic HDR data, dovi data, etc. - which
MSDK doesn't provide, so this single bit of SEI data doesn't
help much.
Best regards,
softworkz
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
There was a problem hiding this comment.
On the FFmpeg mailing list, "Xiang, Haihao" wrote (reply to this):
On Mon, 2022-11-21 at 15:58 +0000, Soft Works wrote:
> > -----Original Message-----
> > From: Xiang, Haihao <haihao.xiang@intel.com>
> > Sent: Monday, November 21, 2022 3:45 AM
> > To: ffmpeg-devel@ffmpeg.org
> > Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at-
> > intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com
> > Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> > SEI parsing for QSV decoders
> >
> > On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > > From: softworkz <softworkz@hotmail.com>
> > >
> > > Signed-off-by: softworkz <softworkz@hotmail.com>
> > > ---
> > > libavcodec/Makefile | 2 +-
> > > libavcodec/qsvdec.c | 321
> > ++++++++++++++++++++++++++++++++++++++++++++
> > > 2 files changed, 322 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > > index 90c7f113a3..cbddbb0ace 100644
> > > --- a/libavcodec/Makefile
> > > +++ b/libavcodec/Makefile
> > > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) +=
> > mss34dsp.o
> > > OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
> > > OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
> > > OBJS-$(CONFIG_QSV) += qsv.o
> > > -OBJS-$(CONFIG_QSVDEC) += qsvdec.o
> > > +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o
> > hevc_sei.o
> > > OBJS-$(CONFIG_QSVENC) += qsvenc.o
> > > OBJS-$(CONFIG_RANGECODER) += rangecoder.o
> > > OBJS-$(CONFIG_RDFT) += rdft.o
> > > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > > index 73405b5747..467a248224 100644
> > > --- a/libavcodec/qsvdec.c
> > > +++ b/libavcodec/qsvdec.c
> > > @@ -41,6 +41,7 @@
> > > #include "libavutil/time.h"
> > > #include "libavutil/imgutils.h"
> > > #include "libavutil/film_grain_params.h"
> > > +#include <libavutil/reverse.h>
> > >
> > > #include "avcodec.h"
> > > #include "codec_internal.h"
> > > @@ -49,6 +50,9 @@
> > > #include "hwconfig.h"
> > > #include "qsv.h"
> > > #include "qsv_internal.h"
> > > +#include "h264_sei.h"
> > > +#include "hevc_ps.h"
> > > +#include "hevc_sei.h"
> > >
> > > #if QSV_ONEVPL
> > > #include <mfxdispatcher.h>
> > > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> > > AV_NOPTS_VALUE : pts_tb.num ? \
> > > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> > >
> > > +#define PAYLOAD_BUFFER_SIZE 65535
> > > +
> > > typedef struct QSVAsyncFrame {
> > > mfxSyncPoint *sync;
> > > QSVFrame *frame;
> > > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> > >
> > > mfxExtBuffer **ext_buffers;
> > > int nb_ext_buffers;
> > > +
> > > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > > + AVBufferRef *a53_buf_ref;
> > > } QSVContext;
> > >
> > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > > @@ -628,6 +637,299 @@ static int
> > qsv_export_film_grain(AVCodecContext *avctx,
> > > mfxExtAV1FilmGrainParam
> > > }
> > > #endif
> > >
> > > +static int find_start_offset(mfxU8 data[4])
> > > +{
> > > + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > > + return 3;
> > > +
> > > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> > 1)
> > > + return 4;
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> > AVFrame* out)
> > > +{
> > > + H264SEIContext sei = { 0 };
> > > + GetBitContext gb = { 0 };
> > > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> > .BufSize =
> > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > > + mfxU64 ts;
> > > + int ret;
> > > +
> > > + while (1) {
> > > + int start;
> > > + memset(payload.Data, 0, payload.BufSize);
> > > +
> > > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> > &payload);
> > > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> > buffer on
> > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> > > payload_buffer),
> > > payload.BufSize);
> > > + return 0;
> > > + }
> > > + if (ret != MFX_ERR_NONE)
> > > + return ret;
> > > +
> > > + if (payload.NumBit == 0 || payload.NumBit >=
> > payload.BufSize * 8)
> > > + break;
> > > +
> > > + start = find_start_offset(payload.Data);
> > > +
> > > + switch (payload.Type) {
> > > + case SEI_TYPE_BUFFERING_PERIOD:
> > > + case SEI_TYPE_PIC_TIMING:
> > > + continue;
> > > + }
> > > +
> > > + if (init_get_bits(&gb, &payload.Data[start],
> > payload.NumBit - start *
> > > 8) < 0)
> > > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> > bitstream reader
> > > SEI type: %d Numbits %d error: %d\n", payload.Type,
> > payload.NumBit, ret);
> > > + else {
> > > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > > +
> > > + if (ret < 0)
> > > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> > type:
> > > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > > + else
> > > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> > Numbits
> > > %d\n", payload.Type, payload.NumBit);
> > > + }
> > > + }
> > > +
> > > + if (out)
> > > + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> > 0);
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> > QSVFrame*
> > > out)
> > > +{
> > > + HEVCSEI sei = { 0 };
> > > + HEVCParamSets ps = { 0 };
> > > + GetBitContext gb = { 0 };
> > > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> > .BufSize =
> > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > > + mfxFrameSurface1 *surface = &out->surface;
> > > + mfxU64 ts;
> > > + int ret, has_logged = 0;
> > > +
> > > + while (1) {
> > > + int start;
> > > + memset(payload.Data, 0, payload.BufSize);
> > > +
> > > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> > &payload);
> > > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> > buffer on
> > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> > > payload_buffer),
> > > payload.BufSize);
> > > + return 0;
> > > + }
> > > + if (ret != MFX_ERR_NONE)
> > > + return ret;
> > > +
> > > + if (payload.NumBit == 0 || payload.NumBit >=
> > payload.BufSize * 8)
> > > + break;
> > > +
> > > + if (!has_logged) {
> > > + has_logged = 1;
> > > + av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> > ---------
> > > ---------\n");
> > > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> > payload
> > > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> > > Data.TimeStamp);
> > > + }
> > > +
> > > + if (ts != surface->Data.TimeStamp) {
> > > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> > (%llu) does
> > > not match surface timestamp: (%llu)\n", ts, surface-
> > > Data.TimeStamp);
> > > + }
> > > +
> > > + start = find_start_offset(payload.Data);
> > > +
> > > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> > Numbits
> > > %3d Start: %d\n", payload.Type, payload.NumBit, start);
> > > +
> > > + switch (payload.Type) {
> > > + case SEI_TYPE_BUFFERING_PERIOD:
> > > + case SEI_TYPE_PIC_TIMING:
> > > + continue;
> > > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > > + // There seems to be a bug in MSDK
> > > + payload.NumBit -= 8;
> > > +
> > > + break;
> > > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > > + // There seems to be a bug in MSDK
> > > + payload.NumBit = 48;
> > > +
> > > + break;
> > > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > > + // There seems to be a bug in MSDK
> > > + if (payload.NumBit == 552)
> > > + payload.NumBit = 528;
> > > + break;
> > > + }
> > > +
> > > + if (init_get_bits(&gb, &payload.Data[start],
> > payload.NumBit - start *
> > > 8) < 0)
> > > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> > bitstream reader
> > > SEI type: %d Numbits %d error: %d\n", payload.Type,
> > payload.NumBit, ret);
> > > + else {
> > > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > > HEVC_NAL_SEI_PREFIX);
> > > +
> > > + if (ret < 0)
> > > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> > type:
> > > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > > + else
> > > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> > Numbits
> > > %d\n", payload.Type, payload.NumBit);
> > > + }
> > > + }
> > > +
> > > + if (has_logged) {
> > > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > > + }
> > > +
> > > + if (out && out->frame)
> > > + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> > avctx-
> > > > framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> > > bit_depth_chroma);
> >
> > I got segfault when trying your patchset,
> >
> > Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> > 0x00007ffff67c0497 in parse_sei_hevc
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> > 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> > > frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> > > bit_depth_chroma);
> > (gdb) bt
> > #0 0x00007ffff67c0497 in parse_sei_hevc
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> > #1 0x00007ffff67c1afe in qsv_decode
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > frame=frame@entry=0x5555556df740,
> > got_frame=got_frame@entry=0x7fffffffd6bc,
> > avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020
> > BTW the SDK provides support for hevc HDR metadata, we needn't parse
> > SEI payload
> > in qsvdec and may get the corresponding info from the SDK, see
> > https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
>
> I know. I was the one who had requested this to be added to MSDK :-)
>
> But it's just one small part of SEI information, it's limited to
> the latest MSDK versions and I'm not sure whether it's working
> as reliably as this implementation. This would need to be tested.
>
> You should still have access to the repo with the test files for
> demoing the offset problems (which my patchset is working around).
I worked out a patchset (https://github.com/intel-media-ci/ffmpeg/pull/518),
including https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html an
d others. I may use your command (with some changes) to convert all HDR clips in
your repo to SDR clips except one clip (w/wo my changes, there is a GPU hang
issue with this clip).
We may only do a small update to support AV1 HDR in the future if applying
https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html.
Note the command below doesn't work with all test files in your repo with your
patchset v6.
$ ffmpeg -hwaccel qsv -c:v hevc_qsv -i input.mp4 -f null -
Thanks
Haihao
>
> But it's also about dynamic HDR data, dovi data, etc. - which
> MSDK doesn't provide, so this single bit of SEI data doesn't
> help much.
>
> Best regards,
> softworkz
>
>
>
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Missing SEI information has always been a major drawback when using the QSV decoders. It turned out that there's a hardly known api method that provides access to all SEI (h264/hevc) or user data (mpeg2video).
This allows to get things like closed captions, frame packing, display orientation, HDR data (mastering display, content light level, etc.) without having to rely on those data being provided by the MSDK as extended buffers.
The commit "Implement SEI parsing for QSV decoders" includes some hard-coded workarounds for MSDK bugs which I reported:
Intel-Media-SDK/MediaSDK#2597 (comment)
If someone is interested in the details please contact me directly.
v5
QSV Overlay Filter: Copy side data from input to output frame #44
ff_hxxx_set_sei_to_framefunctions to avoid being dependent on the full decoder contextshttps://patchwork.ffmpeg.org/project/ffmpeg/cover/pull.31.v5.ffstaging.FFmpeg.1656708534.ffmpegagent@gmail.com/
v4
Now, build still works when someone uses
configure --disable-decoder=h264 --disable-decoder=hevc --disable-decoder=mpegvideo --disable-decoder=mpeg1video --disable-decoder=mpeg2video --enable-libmfxv3
v2
v3
cc: Soft Works softworkz@hotmail.com
cc: "Xiang, Haihao" haihao.xiang-at-intel.com@ffmpeg.org
cc: Kieran Kunhya kierank@obe.tv
cc: Andreas Rheinhardt andreas.rheinhardt@outlook.com