From ac110f1b49a15a326f53e6fac2a8b8300380cebc Mon Sep 17 00:00:00 2001 From: Martin Pulec Date: Thu, 5 Oct 2023 09:28:14 +0200 Subject: [PATCH] vaapi dec.: deduce SW format to reported valid Set AVHWFramesContext::sw_format to first of av_hwframe_transfer_get_formats(). This is consistent how MPV does that. Fixes NV12 being transmitted despite AVHWFramesContext::sw_format was set to yuv420p causing chroma channels corruption (because the nv12 data was misinterpreted as the latter one) occuring on AMD cards, steps to reproduce: ``` uv -t testcard -c lavc:enc=libx264:safe -d gl --param use-hw-accel=vaapi ``` See also: --- src/hwaccel_vaapi.c | 94 +++++++++++++++++++++++++++++-- src/libavcodec/lavc_common.c | 22 ++++++++ src/libavcodec/lavc_common.h | 1 + src/video_compress/libavcodec.cpp | 14 ++--- 4 files changed, 115 insertions(+), 16 deletions(-) diff --git a/src/hwaccel_vaapi.c b/src/hwaccel_vaapi.c index 282883992..e69b54965 100644 --- a/src/hwaccel_vaapi.c +++ b/src/hwaccel_vaapi.c @@ -48,18 +48,20 @@ #include "config_win32.h" #endif // defined HAVE_CONFIG_H -#include "hwaccel_vaapi.h" - -#include "debug.h" - -#include "hwaccel_libav_common.h" #include +#include #if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) #include #endif #include +#include "debug.h" +#include "hwaccel_libav_common.h" +#include "hwaccel_vaapi.h" +#include "libavcodec/lavc_common.h" + #define DEFAULT_SURFACES 20 +#define MOD_NAME "[vaapi] " struct vaapi_ctx { AVBufferRef *device_ref; @@ -196,6 +198,84 @@ static int vaapi_create_context(struct vaapi_ctx *ctx, } #endif //LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) +/** + * Returns first SW format from valid_sw_formats. This is usually + * AV_PIX_FMT_YUV420P or AV_PIX_FMT_NV12. + * + * The code borrows heavily from mpv + * + * namely from function try_format_config(). + */ +static enum AVPixelFormat +get_sw_format(VADisplay display, AVBufferRef *device_ref, + enum AVPixelFormat fallback_fmt) +{ + enum AVPixelFormat ret = AV_PIX_FMT_NONE; + AVVAAPIHWConfig *hwconfig = NULL; + VAConfigID config_id = 0; + AVHWFramesConstraints *fc = NULL; + + VAStatus status = vaCreateConfig( + display, VAProfileNone, VAEntrypointVideoProc, NULL, 0, &config_id); + if (status != VA_STATUS_SUCCESS) { + MSG(ERROR, "cannot create config\n"); + goto fail; + } + fc = av_hwdevice_get_hwframe_constraints(device_ref, hwconfig); + if (!fc) { + MSG(ERROR, "failed to retrieve libavutil frame constraints\n"); + goto fail; + } + + /* + * We need a hwframe_ctx to be able to get the valid formats, but to + * initialise it, we need a format, so we get the first format from the + * hwconfig. We don't care about the other formats in the config because + * the transfer formats list will already include them. + */ + AVBufferRef *fref = NULL; + fref = av_hwframe_ctx_alloc(device_ref); + if (!fref) { + MSG(ERROR, "failed to alloc libavutil frame context\n"); + goto fail; + } + AVHWFramesContext *fctx = (void *) fref->data; + enum { + INIT_SIZE = 128, ///< just some valid size + }; + fctx->format = AV_PIX_FMT_VAAPI; + fctx->sw_format = fc->valid_sw_formats[0]; + fctx->width = INIT_SIZE; + fctx->height = INIT_SIZE; + if (av_hwframe_ctx_init(fref) < 0) { + MSG(ERROR, "failed to init libavutil frame context\n"); + goto fail; + } + + enum AVPixelFormat *fmts = NULL; + int rc = av_hwframe_transfer_get_formats( + fref, AV_HWFRAME_TRANSFER_DIRECTION_FROM, &fmts, 0); + if (rc) { + MSG(ERROR, "failed to get libavutil frame context supported " + "formats\n"); + goto fail; + } + MSG(DEBUG, "Available HW layouts: %s\n", get_avpixfmts_names(fmts)); + ret = fmts[0]; + +fail: + av_hwframe_constraints_free(&fc); + av_buffer_unref(&fref); + if (ret == AV_PIX_FMT_NONE) { + MSG(WARNING, "Using fallback HW frames layout: %s\n", + av_get_pix_fmt_name(ret)); + ret = fallback_fmt; + } + MSG(VERBOSE, "Selected HW frames layout: %s\n", + av_get_pix_fmt_name(ret)); + return ret; +} + int vaapi_init(struct AVCodecContext *s, struct hw_accel_state *state, codec_t out_codec) @@ -224,11 +304,13 @@ int vaapi_init(struct AVCodecContext *s, if (s->active_thread_type & FF_THREAD_FRAME) decode_surfaces += s->thread_count; + const enum AVPixelFormat sw_format = get_sw_format( + ctx->device_vaapi_ctx->display, ctx->device_ref, s->sw_pix_fmt); ret = create_hw_frame_ctx(ctx->device_ref, s->coded_width, s->coded_height, AV_PIX_FMT_VAAPI, - s->sw_pix_fmt, + sw_format, decode_surfaces, &ctx->hw_frames_ctx); if(ret < 0) diff --git a/src/libavcodec/lavc_common.c b/src/libavcodec/lavc_common.c index b520d82ca..b78138617 100644 --- a/src/libavcodec/lavc_common.c +++ b/src/libavcodec/lavc_common.c @@ -56,6 +56,7 @@ #include "host.h" #include "libavcodec/lavc_common.h" +#include "utils/macros.h" #include "video.h" #define MOD_NAME "[lavc_common] " @@ -356,4 +357,25 @@ audio_bps_to_av_sample_fmt(int bps, bool planar) abort(); } } + +/** + * Prints space-separated nammes of AVPixelFormats in AV_PIX_FMT_NONE-terminated + * pixfmts list to given buf and returns pointer to given buf. + */ +const char * +get_avpixfmts_names(const enum AVPixelFormat *pixfmts) +{ + _Thread_local static char buf[STR_LEN]; + if (pixfmts == NULL || *pixfmts == AV_PIX_FMT_NONE) { + snprintf(buf, sizeof buf, " (none)"); + return buf; + } + const enum AVPixelFormat *it = pixfmts; + while (*it != AV_PIX_FMT_NONE) { + snprintf(buf + strlen(buf), sizeof buf - strlen(buf), "%s%s", + it != pixfmts ? " " : "", av_get_pix_fmt_name(*it)); + it++; + } + return buf; +} /* vi: set expandtab sw=8: */ diff --git a/src/libavcodec/lavc_common.h b/src/libavcodec/lavc_common.h index 71c2dd7d9..a8d5eebf8 100644 --- a/src/libavcodec/lavc_common.h +++ b/src/libavcodec/lavc_common.h @@ -123,6 +123,7 @@ void lavd_flush(AVCodecContext *codec_ctx); const char *lavc_thread_type_to_str(int thread_type); struct audio_desc audio_desc_from_av_frame(const AVFrame *frm); enum AVSampleFormat audio_bps_to_av_sample_fmt(int bps, bool planar); +const char *get_avpixfmts_names(const enum AVPixelFormat *pixfmts); #ifdef __cplusplus } diff --git a/src/video_compress/libavcodec.cpp b/src/video_compress/libavcodec.cpp index 688a98414..3f64136fd 100644 --- a/src/video_compress/libavcodec.cpp +++ b/src/video_compress/libavcodec.cpp @@ -672,16 +672,10 @@ static int vaapi_init(struct AVCodecContext *s){ #endif void print_codec_supp_pix_fmts(const enum AVPixelFormat *first) { - char out[STR_LEN] = MOD_NAME "Codec supported pixel formats:" TERM_BOLD; - if (first == nullptr) { - snprintf(out + strlen(out), sizeof out - strlen(out), - " (none)"); - } - const enum AVPixelFormat *it = first; - while (it != nullptr && *it != AV_PIX_FMT_NONE) { - snprintf(out + strlen(out), sizeof out - strlen(out), " %s", - av_get_pix_fmt_name(*it++)); - } + char out[STR_LEN]; + snprintf(out, sizeof out, + MOD_NAME "Codec supported pixel formats: " TBOLD("%s"), + get_avpixfmts_names(first)); LOG(LOG_LEVEL_VERBOSE) << wrap_paragraph(out) << TERM_RESET "\n"; }