From 9d0e73adcf795b90b80aaef4e9791a423af7c297 Mon Sep 17 00:00:00 2001
From: BtbN <btbn@btbn.de>
Date: Wed, 17 Oct 2018 22:03:24 +0200
Subject: [PATCH] vo_gpu: hwdec_cuda: add win32 support for vulkan memory
 mapping

---
 video/out/opengl/hwdec_cuda.c | 47 ++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/video/out/opengl/hwdec_cuda.c b/video/out/opengl/hwdec_cuda.c
index 61ed5119fa2e1..a12cade7a5d16 100644
--- a/video/out/opengl/hwdec_cuda.c
+++ b/video/out/opengl/hwdec_cuda.c
@@ -39,6 +39,10 @@
 #include "video/out/vulkan/ra_vk.h"
 #include "video/out/vulkan/utils.h"
 
+#if HAVE_WIN32_DESKTOP
+#include <versionhelpers.h>
+#endif
+
 struct priv_owner {
     struct mp_hwdec_ctx hwctx;
     CudaFunctions *cu;
@@ -47,7 +51,11 @@ struct priv_owner {
 };
 
 struct ext_buf {
+#if HAVE_WIN32_DESKTOP
+    HANDLE handle;
+#else
     int fd;
+#endif
     CUexternalMemory mem;
     CUdeviceptr buf;
 };
@@ -117,8 +125,11 @@ static int cuda_init(struct ra_hwdec *hw)
 #if HAVE_VULKAN
     if (is_vk) {
     #if HAVE_WIN32_DESKTOP
-        MP_ERR(hw, "CUDA hwdec with Vulkan is currently not supported on Windows\n");
-        return -1;
+        if (!ra_vk_get(hw->ra)->has_ext_external_memory_win32) {
+            MP_ERR(hw, "CUDA hwdec with Vulkan requires the %s extension\n",
+                   VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME);
+            return -1;
+        }
     #else
         if (!ra_vk_get(hw->ra)->has_ext_external_memory_fd) {
             MP_ERR(hw, "CUDA hwdec with Vulkan requires the %s extension\n",
@@ -302,21 +313,33 @@ static struct ra_buf *cuda_buf_pool_get(struct ra_hwdec_mapper *mapper, int n)
 
     if (!ra_vk_buf_get_user_data(buf)) {
         struct ext_buf *ebuf = talloc_zero(NULL, struct ext_buf);
+        struct vk_external_mem mem_info;
 
-        struct vk_external_fd fd_info;
-        bool success = ra_vk_buf_get_external_fd(mapper->ra, buf, &fd_info);
+        bool success = ra_vk_buf_get_external_info(mapper->ra, buf, &mem_info);
         if (!success) {
             ret = -1;
             goto error;
         }
 
-        ebuf->fd = fd_info.mem_fd;
-        MP_DBG(mapper, "vk_external_fd[%d][%d]: %d %lu %lu\n", n, pool->index, ebuf->fd, fd_info.size, fd_info.offset);
+#if HAVE_WIN32_DESKTOP
+        MP_DBG(mapper, "vk_external_info[%d][%d]: %p %zu %zu\n", n, pool->index, ebuf->handle, mem_info.size, mem_info.offset);
+        ebuf->handle = mem_info.mem_handle;
+#else
+        MP_DBG(mapper, "vk_external_info[%d][%d]: %d %zu %zu\n", n, pool->index, ebuf->fd, mem_info.size, mem_info.offset);
+        ebuf->fd = mem_info.mem_fd;
+#endif
 
         CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
+#if HAVE_WIN32_DESKTOP
+            .type = IsWindows8OrGreater()
+                ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
+                : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
+            .handle.win32.handle = ebuf->handle,
+#else
             .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
             .handle.fd = ebuf->fd,
-            .size = fd_info.mem_size,
+#endif
+            .size = mem_info.mem_size,
             .flags = 0,
         };
         ret = CHECK_CU(cu->cuImportExternalMemory(&ebuf->mem, &ext_desc));
@@ -324,8 +347,8 @@ static struct ra_buf *cuda_buf_pool_get(struct ra_hwdec_mapper *mapper, int n)
             goto error;
 
         CUDA_EXTERNAL_MEMORY_BUFFER_DESC buf_desc = {
-            .offset = fd_info.offset,
-            .size = fd_info.size,
+            .offset = mem_info.offset,
+            .size = mem_info.size,
             .flags = 0,
         };
         ret = CHECK_CU(cu->cuExternalMemoryGetMappedBuffer(&ebuf->buf, ebuf->mem, &buf_desc));
@@ -354,9 +377,15 @@ static void cuda_buf_pool_uninit(struct ra_hwdec_mapper *mapper, int n)
         if (ebuf) {
             if (ebuf->mem > 0) {
                 CHECK_CU(cu->cuDestroyExternalMemory(ebuf->mem));
+#if HAVE_WIN32_DESKTOP
+            } else if (ebuf->handle) {
+                CloseHandle(ebuf->handle);
+            }
+#else
             } else if (ebuf->fd > -1) {
                 close(ebuf->fd);
             }
+#endif
         }
         talloc_free(ebuf);
         ra_vk_buf_set_user_data(buf, NULL);