video: add --hwdec=vdpau-copy mode

At this point, all other hwaccels provide -copy modes, and vdpau is the
exception with not having one. Although there is vf_vdpaurb, it's less
convenient in certain situations, and exposes some issues with the
filter chain code as well.

Author: wm4

DOCS/man/options.rst

@@ -610,6 +610,7 @@ Video
     :yes:       exactly the same as ``auto``
     :auto-copy: enable best hw decoder with copy-back (see below)
     :vdpau:     requires ``--vo=vdpau`` or ``--vo=opengl`` (Linux only)
+    :vdpau-copy: copies video back into system RAM (Linux with some GPUs only)
     :vaapi:     requires ``--vo=opengl`` or ``--vo=vaapi`` (Linux only)
     :vaapi-copy: copies video back into system RAM (Linux with Intel GPUs only)
     :videotoolbox: requires ``--vo=opengl`` (OS X 10.8 and up only)
@@ -651,10 +652,10 @@ Video
     primarily implemented on the CPU. Some exceptions are ``vdpaupp``,
     ``vdpaurb`` and ``vavpp``. See `VIDEO FILTERS`_ for more details.
 
-    The ``vaapi-copy`` and ``dxva2-copy`` modes allow you to use hardware
+    The ``...-copy`` modes (e.g. ``dxva2-copy``) allow you to use hardware
     decoding with any VO, backend or filter. Because these copy the decoded
-    video back to system RAM, they're likely less efficient than the ``vaapi``
-    or ``dxva2`` modes respectively.
+    video back to system RAM, they're likely less efficient than the direct
+    modes (like e.g. ``dxva2``).
 
     .. note::
 

options/options.c

@@ -93,6 +93,7 @@ const struct m_opt_choice_alternatives mp_hwdec_names[] = {
     {"yes" ,        HWDEC_AUTO},
     {"auto-copy",   HWDEC_AUTO_COPY},
     {"vdpau",       HWDEC_VDPAU},
+    {"vdpau-copy",  HWDEC_VDPAU_COPY},
     {"videotoolbox",HWDEC_VIDEOTOOLBOX},
     {"videotoolbox-copy",HWDEC_VIDEOTOOLBOX_COPY},
     {"vaapi",       HWDEC_VAAPI},

video/decode/vd_lavc.c

@@ -125,6 +125,7 @@ const struct m_sub_options vd_lavc_conf = {
 };
 
 extern const struct vd_lavc_hwdec mp_vd_lavc_vdpau;
+extern const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy;
 extern const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox;
 extern const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox_copy;
 extern const struct vd_lavc_hwdec mp_vd_lavc_vaapi;
@@ -177,6 +178,7 @@ static const struct vd_lavc_hwdec *const hwdec_list[] = {
 #endif
 #if HAVE_VDPAU_HWACCEL
     &mp_vd_lavc_vdpau,
+    &mp_vd_lavc_vdpau_copy,
 #endif
 #if HAVE_VIDEOTOOLBOX_HWACCEL
     &mp_vd_lavc_videotoolbox,

video/decode/vdpau.c

@@ -21,13 +21,17 @@
 
 #include "lavc.h"
 #include "common/common.h"
+#include "video/mp_image_pool.h"
 #include "video/vdpau.h"
 #include "video/hwdec.h"
 
 struct priv {
     struct mp_log              *log;
     struct mp_vdpau_ctx        *mpvdp;
     uint64_t                    preemption_counter;
+    // vdpau-copy
+    Display                    *display;
+    struct mp_image_pool       *sw_pool;
 };
 
 static int init_decoder(struct lavc_ctx *ctx, int w, int h)
@@ -76,9 +80,16 @@ static void uninit(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
 
+    if (p->display) {
+        // for copy path: we own this stuff
+        mp_vdpau_destroy(p->mpvdp);
+        XCloseDisplay(p->display);
+    }
+
     talloc_free(p);
 
-    av_freep(&ctx->avctx->hwaccel_context);
+    if (ctx->avctx)
+        av_freep(&ctx->avctx->hwaccel_context);
 }
 
 static int init(struct lavc_ctx *ctx)
@@ -102,6 +113,56 @@ static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
     return 0;
 }
 
+static int init_copy(struct lavc_ctx *ctx)
+{
+    struct priv *p = talloc_ptrtype(NULL, p);
+    *p = (struct priv) {
+        .log = mp_log_new(p, ctx->log, "vdpau"),
+    };
+
+    p->display = XOpenDisplay(NULL);
+    if (!p->display)
+        goto error;
+
+    p->mpvdp = mp_vdpau_create_device_x11(p->log, p->display, true);
+    if (!p->mpvdp)
+        goto error;
+
+    p->sw_pool = talloc_steal(p, mp_image_pool_new(17));
+
+    ctx->hwdec_priv = p;
+
+    mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter);
+    return 0;
+
+error:
+    if (p->display)
+        XCloseDisplay(p->display);
+    talloc_free(p);
+    return -1;
+}
+
+static int probe_copy(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
+                      const char *codec)
+{
+    assert(!ctx->hwdec_priv);
+    int r = init_copy(ctx);
+    if (ctx->hwdec_priv)
+        uninit(ctx);
+    ctx->hwdec_priv = NULL;
+
+    return r < 0 ? HWDEC_ERR_NO_CTX : 0;
+}
+
+static struct mp_image *copy_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    struct priv *p = ctx->hwdec_priv;
+    struct mp_hwdec_ctx *hwctx = &p->mpvdp->hwctx;
+    struct mp_image *out = hwctx->download_image(hwctx, img, p->sw_pool);
+    talloc_free(img);
+    return out;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_vdpau = {
     .type = HWDEC_VDPAU,
     .image_format = IMGFMT_VDPAU,
@@ -112,3 +173,15 @@ const struct vd_lavc_hwdec mp_vd_lavc_vdpau = {
     .allocate_image = allocate_image,
     .process_image = update_format,
 };
+
+const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy = {
+    .type = HWDEC_VDPAU_COPY,
+    .copying = true,
+    .image_format = IMGFMT_VDPAU,
+    .probe = probe_copy,
+    .init = init_copy,
+    .uninit = uninit,
+    .init_decoder = init_decoder,
+    .allocate_image = allocate_image,
+    .process_image = copy_image,
+};

video/hwdec.h

@@ -11,6 +11,7 @@ enum hwdec_type {
     HWDEC_AUTO,
     HWDEC_AUTO_COPY,
     HWDEC_VDPAU,
+    HWDEC_VDPAU_COPY,
     HWDEC_VIDEOTOOLBOX,
     HWDEC_VIDEOTOOLBOX_COPY,
     HWDEC_VAAPI,