vo_gpu: vulkan: add support for win32 external memory extension

Author: BtbN

video/out/opengl/hwdec_cuda.c

@@ -117,8 +117,11 @@ static int cuda_init(struct ra_hwdec *hw)
 #if HAVE_VULKAN
     if (is_vk) {
     #if HAVE_WIN32_DESKTOP
-        MP_ERR(hw, "CUDA hwdec with Vulkan is currently not supported on Windows\n");
-        return -1;
+        if (!ra_vk_get(hw->ra)->has_ext_external_memory_win32) {
+            MP_ERR(hw, "CUDA hwdec with Vulkan requires the %s extension\n",
+                   VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME);
+            return -1;
+        }
     #else
         if (!ra_vk_get(hw->ra)->has_ext_external_memory_fd) {
             MP_ERR(hw, "CUDA hwdec with Vulkan requires the %s extension\n",

video/out/vulkan/common.h

@@ -25,6 +25,10 @@
 
 #include <vulkan/vulkan.h>
 
+#if HAVE_WIN32_DESKTOP
+#include <vulkan/vulkan_win32.h>
+#endif
+
 // Vulkan allows the optional use of a custom allocator. We don't need one but
 // mark this parameter with a better name in case we ever decide to change this
 // in the future. (And to make the code more readable)
@@ -77,4 +81,5 @@ struct mpvk_ctx {
     // Extension availability
     bool has_ext_external_memory;
     bool has_ext_external_memory_fd;
+    bool has_ext_external_memory_win32;
 };

video/out/vulkan/malloc.c

@@ -2,6 +2,11 @@
 #include "utils.h"
 #include "osdep/timer.h"
 
+#if HAVE_WIN32_DESKTOP
+#include <aclapi.h>
+#include <dxgi1_2.h>
+#endif
+
 // Controls the multiplication factor for new slab allocations. The new slab
 // will always be allocated such that the size of the slab is this factor times
 // the previous slab. Higher values make it grow faster.
@@ -127,9 +132,59 @@ static struct vk_slab *slab_alloc(struct mpvk_ctx *vk, struct vk_heap *heap,
         .end   = slab->size,
     });
 
+#if HAVE_WIN32_DESKTOP
+    PSECURITY_DESCRIPTOR secdesc = (PSECURITY_DESCRIPTOR)calloc(1, SECURITY_DESCRIPTOR_MIN_LENGTH + 2 * sizeof(void*));
+    PSID *sid = (PSID*)((PBYTE)secdesc + SECURITY_DESCRIPTOR_MIN_LENGTH);
+    PACL *acl = (PACL*)((PBYTE)sid + sizeof(PSID*));
+    SID_IDENTIFIER_AUTHORITY sididauth = SECURITY_WORLD_SID_AUTHORITY;
+
+    if (!secdesc)
+        goto error;
+
+    EXPLICIT_ACCESS expaccess = {
+        .grfAccessPermissions = STANDARD_RIGHTS_ALL | SPECIFIC_RIGHTS_ALL,
+        .grfAccessMode = SET_ACCESS,
+        .grfInheritance = INHERIT_ONLY,
+        .Trustee.TrusteeForm = TRUSTEE_IS_SID,
+        .Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP,
+        .Trustee.ptstrName = (LPTSTR)*sid,
+    };
+
+    if (!InitializeSecurityDescriptor(secdesc, SECURITY_DESCRIPTOR_REVISION))
+        goto error;
+
+    if (!AllocateAndInitializeSid(&sididauth, 1, SECURITY_WORLD_RID, 0, 0, 0, 0, 0, 0, 0, sid))
+        goto error;
+
+    if (SetEntriesInAcl(1, &expaccess, NULL, acl) != ERROR_SUCCESS)
+        goto error;
+
+    if (!SetSecurityDescriptorDacl(secdesc, TRUE, *acl, FALSE))
+        goto error;
+
+    SECURITY_ATTRIBUTES secattr = {
+        .nLength = sizeof(SECURITY_ATTRIBUTES),
+        .lpSecurityDescriptor = secdesc,
+        .bInheritHandle = TRUE,
+    };
+
+    VkExportMemoryWin32HandleInfoKHR emw32info = {
+        .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_WIN32_HANDLE_INFO_KHR,
+        .pNext = NULL,
+        .pAttributes = &secattr,
+        .dwAccess = 0x80000001, // DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE
+        .name = NULL,
+    };
+#endif
+
     VkExportMemoryAllocateInfoKHR eminfo = {
         .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR,
+#if HAVE_WIN32_DESKTOP
+        .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT_KHR,
+        .pNext = &emw32info,
+#else
         .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
+#endif
     };
 
     VkMemoryAllocateInfo minfo = {
@@ -189,10 +244,31 @@ static struct vk_slab *slab_alloc(struct mpvk_ctx *vk, struct vk_heap *heap,
     if (slab->buffer)
         VK(vkBindBufferMemory(vk->dev, slab->buffer, slab->mem, 0));
 
+#if HAVE_WIN32_DESKTOP
+    if (secdesc && *sid)
+        FreeSid(*sid);
+
+    if (secdesc && *acl)
+        LocalFree(*acl);
+
+    free(secdesc);
+#endif
+
     return slab;
 
 error:
     slab_free(vk, slab);
+
+#if HAVE_WIN32_DESKTOP
+    if (secdesc && *sid)
+        FreeSid(*sid);
+
+    if (secdesc && *acl)
+        LocalFree(*acl);
+
+    free(secdesc);
+#endif
+
     return NULL;
 }
 
@@ -442,8 +518,11 @@ bool vk_malloc_buffer(struct mpvk_ctx *vk, VkBufferUsageFlags bufFlags,
 {
     if (exportable) {
 #if HAVE_WIN32_DESKTOP
-        MP_ERR(vk, "Exportable memory is not currently supported on Windows\n");
-        return false;
+        if (!vk->has_ext_external_memory_win32) {
+            MP_ERR(vk, "Exportable memory requires the %s extension\n",
+                   VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME);
+            return false;
+        }
 #else
         if (!vk->has_ext_external_memory_fd) {
             MP_ERR(vk, "Exportable memory requires the %s extension\n",

video/out/vulkan/utils.c

@@ -484,6 +484,13 @@ static bool detect_device_extensions(struct mpvk_ctx *vk)
             vk->has_ext_external_memory_fd = true;
             continue;
         }
+#if HAVE_WIN32_DESKTOP
+        if (!strcmp(VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME,
+                    props[i].extensionName)) {
+            vk->has_ext_external_memory_win32 = true;
+            continue;
+        }
+#endif
     }
 
     ret = true;
@@ -559,6 +566,10 @@ bool mpvk_device_init(struct mpvk_ctx *vk, struct mpvk_device_opts opts)
         MP_TARRAY_APPEND(tmp, exts, num_exts, VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME);
     if (vk->has_ext_external_memory_fd)
         MP_TARRAY_APPEND(tmp, exts, num_exts, VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME);
+#if HAVE_WIN32_DESKTOP
+    if (vk->has_ext_external_memory_win32)
+        MP_TARRAY_APPEND(tmp, exts, num_exts, VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME);
+#endif
     if (vk->spirv->required_ext)
         MP_TARRAY_APPEND(tmp, exts, num_exts, vk->spirv->required_ext);