From 2c8d97dbee94cb1e5b9bf4d212b8d72f27981d37 Mon Sep 17 00:00:00 2001
From: David Pinedo <david@lunarg.com>
Date: Wed, 23 Aug 2023 11:21:52 -0600
Subject: [PATCH 1/5] gfxrecon-replay: Use infinite timeout for 3 Vulkan api
 calls

When replaying:
    vkAcquireProfilingLockKHR
    vkWaitForPresentKHR
    vkWaitSemaphores
use an infinite timeout if the expected result is VK_SUCCESS.
---
 .../decode/vulkan_replay_consumer_base.cpp    | 72 +++++++++++++++++++
 .../decode/vulkan_replay_consumer_base.h      | 17 +++++
 .../generated_vulkan_replay_consumer.cpp      | 18 +++--
 .../vulkan_generators/replay_overrides.json   |  5 +-
 4 files changed, 101 insertions(+), 11 deletions(-)

diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp
index 1adbd9948d..0af770bad4 100644
--- a/framework/decode/vulkan_replay_consumer_base.cpp
+++ b/framework/decode/vulkan_replay_consumer_base.cpp
@@ -6641,6 +6641,78 @@ VkResult VulkanReplayConsumerBase::OverrideGetPhysicalDeviceToolProperties(
     }
 }
 
+VkResult
+VulkanReplayConsumerBase::OverrideWaitSemaphores(PFN_vkWaitSemaphores func,
+                                                 VkResult             original_result,
+                                                 const DeviceInfo*    device_info,
+                                                 const StructPointerDecoder<Decoded_VkSemaphoreWaitInfo>* pInfo,
+                                                 uint64_t                                                 timeout)
+{
+    assert((device_info != nullptr) && (pInfo != nullptr) && !pInfo->IsNull() && (pInfo->GetPointer() != nullptr));
+    VkDevice                   device    = device_info->handle;
+    const VkSemaphoreWaitInfo* wait_info = pInfo->GetPointer();
+    VkResult                   result;
+
+    // If expected result is VK_SUCCESS, ensure that vkWaitSemaphores waits until semaphores
+    // are available by using a timeout of UINT64_MAX.
+    // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
+    // If expected result is anything else, use the passed in timeout value.
+    if (original_result == VK_SUCCESS)
+        timeout = std::numeric_limits<uint64_t>::max();
+    else if (original_result == VK_TIMEOUT)
+        timeout = 0;
+    result = func(device, wait_info, timeout);
+    return result;
+}
+
+VkResult VulkanReplayConsumerBase::OverrideAcquireProfilingLockKHR(
+    PFN_vkAcquireProfilingLockKHR                                      func,
+    VkResult                                                           original_result,
+    const DeviceInfo*                                                  device_info,
+    const StructPointerDecoder<Decoded_VkAcquireProfilingLockInfoKHR>* pInfo)
+{
+    assert((device_info != nullptr) && (pInfo != nullptr) && !pInfo->IsNull() && (pInfo->GetPointer() != nullptr));
+    VkDevice                             device       = device_info->handle;
+    const VkAcquireProfilingLockInfoKHR* acquire_info = pInfo->GetPointer();
+    VkResult                             result;
+
+    // If expected result is VK_SUCCESS, ensure that vkAcquireProfilingLockKHR waits for locks
+    // using a timeout of UINT64_MAX.
+    // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
+    // If expected result is anything else, use the passed in timeout value.
+    VkAcquireProfilingLockInfoKHR acquire_info_copy = *acquire_info;
+    if (original_result == VK_SUCCESS)
+        acquire_info_copy.timeout = std::numeric_limits<uint64_t>::max();
+    else if (original_result == VK_TIMEOUT)
+        acquire_info_copy.timeout = 0;
+    result = func(device, &acquire_info_copy);
+    return result;
+}
+
+VkResult VulkanReplayConsumerBase::OverrideWaitForPresentKHR(PFN_vkWaitForPresentKHR func,
+                                                             VkResult                original_result,
+                                                             const DeviceInfo*       device_info,
+                                                             SwapchainKHRInfo*       swapchain_info,
+                                                             uint64_t                presentid,
+                                                             uint64_t                timeout)
+{
+    assert((device_info != nullptr) && (swapchain_info != nullptr));
+    VkDevice       device    = device_info->handle;
+    VkSwapchainKHR swapchain = swapchain_info->handle;
+    VkResult       result;
+
+    // If expected result is VK_SUCCESS, ensure that vkWaitForPresent waits for present by
+    // using a timeout of UINT64_MAX.
+    // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
+    // If expected result is anything else, use the passed in timeout value.
+    if (original_result == VK_SUCCESS)
+        timeout = std::numeric_limits<uint64_t>::max();
+    else if (original_result == VK_TIMEOUT)
+        timeout = 0;
+    result = func(device, swapchain, presentid, timeout);
+    return result;
+}
+
 void VulkanReplayConsumerBase::MapDescriptorUpdateTemplateHandles(
     const DescriptorUpdateTemplateInfo* update_template_info, DescriptorUpdateTemplateDecoder* decoder)
 {
diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h
index ebe26e565f..e5a99099e8 100644
--- a/framework/decode/vulkan_replay_consumer_base.h
+++ b/framework/decode/vulkan_replay_consumer_base.h
@@ -935,6 +935,23 @@ class VulkanReplayConsumerBase : public VulkanConsumer
     void OverrideCmdDebugMarkerInsertEXT(PFN_vkCmdDebugMarkerInsertEXT                             func,
                                          CommandBufferInfo*                                        command_buffer_info,
                                          StructPointerDecoder<Decoded_VkDebugMarkerMarkerInfoEXT>* marker_info_decoder);
+    VkResult OverrideWaitSemaphores(PFN_vkWaitSemaphores                                     func,
+                                    VkResult                                                 original_result,
+                                    const DeviceInfo*                                        device_info,
+                                    const StructPointerDecoder<Decoded_VkSemaphoreWaitInfo>* pInfo,
+                                    uint64_t                                                 timeout);
+
+    VkResult OverrideAcquireProfilingLockKHR(PFN_vkAcquireProfilingLockKHR func,
+                                             VkResult                      original_result,
+                                             const DeviceInfo*             device_info,
+                                             const StructPointerDecoder<Decoded_VkAcquireProfilingLockInfoKHR>* pInfo);
+
+    VkResult OverrideWaitForPresentKHR(PFN_vkWaitForPresentKHR func,
+                                       VkResult                original_result,
+                                       const DeviceInfo*       device_info,
+                                       SwapchainKHRInfo*       swapchain_info,
+                                       uint64_t                presentid,
+                                       uint64_t                timeout);
 
     void OverrideCmdBeginRenderPass(PFN_vkCmdBeginRenderPass                             func,
                                     CommandBufferInfo*                                   command_buffer_info,
diff --git a/framework/generated/generated_vulkan_replay_consumer.cpp b/framework/generated/generated_vulkan_replay_consumer.cpp
index 3bff45ecd0..f6ceab3470 100644
--- a/framework/generated/generated_vulkan_replay_consumer.cpp
+++ b/framework/generated/generated_vulkan_replay_consumer.cpp
@@ -2569,11 +2569,11 @@ void VulkanReplayConsumer::Process_vkWaitSemaphores(
     StructPointerDecoder<Decoded_VkSemaphoreWaitInfo>* pWaitInfo,
     uint64_t                                    timeout)
 {
-    VkDevice in_device = MapHandle<DeviceInfo>(device, &VulkanObjectInfoTable::GetDeviceInfo);
-    const VkSemaphoreWaitInfo* in_pWaitInfo = pWaitInfo->GetPointer();
+    auto in_device = GetObjectInfoTable().GetDeviceInfo(device);
+
     MapStructHandles(pWaitInfo->GetMetaStructPointer(), GetObjectInfoTable());
 
-    VkResult replay_result = GetDeviceTable(in_device)->WaitSemaphores(in_device, in_pWaitInfo, timeout);
+    VkResult replay_result = OverrideWaitSemaphores(GetDeviceTable(in_device->handle)->WaitSemaphores, returnValue, in_device, pWaitInfo, timeout);
     CheckResult("vkWaitSemaphores", returnValue, replay_result);
 }
 
@@ -4435,10 +4435,9 @@ void VulkanReplayConsumer::Process_vkAcquireProfilingLockKHR(
     format::HandleId                            device,
     StructPointerDecoder<Decoded_VkAcquireProfilingLockInfoKHR>* pInfo)
 {
-    VkDevice in_device = MapHandle<DeviceInfo>(device, &VulkanObjectInfoTable::GetDeviceInfo);
-    const VkAcquireProfilingLockInfoKHR* in_pInfo = pInfo->GetPointer();
+    auto in_device = GetObjectInfoTable().GetDeviceInfo(device);
 
-    VkResult replay_result = GetDeviceTable(in_device)->AcquireProfilingLockKHR(in_device, in_pInfo);
+    VkResult replay_result = OverrideAcquireProfilingLockKHR(GetDeviceTable(in_device->handle)->AcquireProfilingLockKHR, returnValue, in_device, pInfo);
     CheckResult("vkAcquireProfilingLockKHR", returnValue, replay_result);
 }
 
@@ -4810,11 +4809,10 @@ void VulkanReplayConsumer::Process_vkWaitForPresentKHR(
     uint64_t                                    presentId,
     uint64_t                                    timeout)
 {
-    VkDevice in_device = MapHandle<DeviceInfo>(device, &VulkanObjectInfoTable::GetDeviceInfo);
-    VkSwapchainKHR in_swapchain = MapHandle<SwapchainKHRInfo>(swapchain, &VulkanObjectInfoTable::GetSwapchainKHRInfo);
-    if (GetObjectInfoTable().GetSurfaceKHRInfo(GetObjectInfoTable().GetSwapchainKHRInfo(swapchain)->surface_id) == nullptr || GetObjectInfoTable().GetSurfaceKHRInfo(GetObjectInfoTable().GetSwapchainKHRInfo(swapchain)->surface_id)->surface_creation_skipped) { return; }
+    auto in_device = GetObjectInfoTable().GetDeviceInfo(device);
+    auto in_swapchain = GetObjectInfoTable().GetSwapchainKHRInfo(swapchain);
 
-    VkResult replay_result = GetDeviceTable(in_device)->WaitForPresentKHR(in_device, in_swapchain, presentId, timeout);
+    VkResult replay_result = OverrideWaitForPresentKHR(GetDeviceTable(in_device->handle)->WaitForPresentKHR, returnValue, in_device, in_swapchain, presentId, timeout);
     CheckResult("vkWaitForPresentKHR", returnValue, replay_result);
 }
 
diff --git a/framework/generated/vulkan_generators/replay_overrides.json b/framework/generated/vulkan_generators/replay_overrides.json
index 2152821035..811203ead3 100644
--- a/framework/generated/vulkan_generators/replay_overrides.json
+++ b/framework/generated/vulkan_generators/replay_overrides.json
@@ -93,6 +93,9 @@
     "vkCmdDebugMarkerInsertEXT": "OverrideCmdDebugMarkerInsertEXT",
     "vkCmdBeginRenderPass": "OverrideCmdBeginRenderPass",
     "vkCreateImageView": "OverrideCreateImageView",
-    "vkCreateFramebuffer": "OverrideCreateFramebuffer"
+    "vkCreateFramebuffer": "OverrideCreateFramebuffer",
+    "vkAcquireProfilingLockKHR": "OverrideAcquireProfilingLockKHR",
+    "vkWaitForPresentKHR": "OverrideWaitForPresentKHR",
+    "vkWaitSemaphores": "OverrideWaitSemaphores"
   }
 }

From 8a8543d0ab8c55630032713c767103dc91e12473 Mon Sep 17 00:00:00 2001
From: David Pinedo <david@lunarg.com>
Date: Wed, 30 Aug 2023 12:02:11 -0600
Subject: [PATCH 2/5] gfxrecon-replay: Add missing braces

Added braces to code changes in 2 prior commits

Renamed a variable in VulkanReplayConsumerBase::OverrideAcquireProfilingLockKHR in
framework/decode/vulkan_replay_consumer_base.cpp
---
 .../decode/vulkan_replay_consumer_base.cpp    | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 framework/decode/vulkan_replay_consumer_base.cpp

diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp
old mode 100644
new mode 100755
index 0af770bad4..7ba7029771
--- a/framework/decode/vulkan_replay_consumer_base.cpp
+++ b/framework/decode/vulkan_replay_consumer_base.cpp
@@ -5294,9 +5294,13 @@ VkResult VulkanReplayConsumerBase::OverrideAcquireNextImageKHR(PFN_vkAcquireNext
             // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
             // If expected result is anything else, use the passed in timeout value.
             if (original_result == VK_SUCCESS)
+            {
                 timeout = std::numeric_limits<uint64_t>::max();
+            }
             else if (original_result == VK_TIMEOUT)
+            {
                 timeout = 0;
+            }
             result = swapchain_->AcquireNextImageKHR(
                 func, device_info, swapchain_info, timeout, semaphore_info, fence_info, captured_index, replay_index);
 
@@ -5412,9 +5416,13 @@ VkResult VulkanReplayConsumerBase::OverrideAcquireNextImage2KHR(
             // If expected result is anything else, use the passed in timeout value.
             VkAcquireNextImageInfoKHR modified_acquire_info = *replay_acquire_info;
             if (original_result == VK_SUCCESS)
+            {
                 modified_acquire_info.timeout = std::numeric_limits<uint64_t>::max();
+            }
             else if (original_result == VK_TIMEOUT)
+            {
                 modified_acquire_info.timeout = 0;
+            }
             result = swapchain_->AcquireNextImage2KHR(
                 func, device_info, swapchain_info, &modified_acquire_info, captured_index, replay_index);
 
@@ -6658,9 +6666,13 @@ VulkanReplayConsumerBase::OverrideWaitSemaphores(PFN_vkWaitSemaphores func,
     // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
     // If expected result is anything else, use the passed in timeout value.
     if (original_result == VK_SUCCESS)
+    {
         timeout = std::numeric_limits<uint64_t>::max();
+    }
     else if (original_result == VK_TIMEOUT)
+    {
         timeout = 0;
+    }
     result = func(device, wait_info, timeout);
     return result;
 }
@@ -6680,12 +6692,16 @@ VkResult VulkanReplayConsumerBase::OverrideAcquireProfilingLockKHR(
     // using a timeout of UINT64_MAX.
     // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
     // If expected result is anything else, use the passed in timeout value.
-    VkAcquireProfilingLockInfoKHR acquire_info_copy = *acquire_info;
+    VkAcquireProfilingLockInfoKHR modified_acquire_info = *acquire_info;
     if (original_result == VK_SUCCESS)
-        acquire_info_copy.timeout = std::numeric_limits<uint64_t>::max();
+    {
+        modified_acquire_info.timeout = std::numeric_limits<uint64_t>::max();
+    }
     else if (original_result == VK_TIMEOUT)
-        acquire_info_copy.timeout = 0;
-    result = func(device, &acquire_info_copy);
+    {
+        modified_acquire_info.timeout = 0;
+    }
+    result = func(device, &modified_acquire_info);
     return result;
 }
 
@@ -6706,9 +6722,13 @@ VkResult VulkanReplayConsumerBase::OverrideWaitForPresentKHR(PFN_vkWaitForPresen
     // If expected result is VK_TIMEOUT, try to get a timeout by using a timeout of 0.
     // If expected result is anything else, use the passed in timeout value.
     if (original_result == VK_SUCCESS)
+    {
         timeout = std::numeric_limits<uint64_t>::max();
+    }
     else if (original_result == VK_TIMEOUT)
+    {
         timeout = 0;
+    }
     result = func(device, swapchain, presentid, timeout);
     return result;
 }

From cfc210640513c33b7ab2b3ef20f1cc9e9165e9d5 Mon Sep 17 00:00:00 2001
From: Mark Young <marky@lunarg.com>
Date: Tue, 27 Jun 2023 20:10:47 -0600
Subject: [PATCH 3/5] Modify virtual swapchain behavior

Modify virtual swapchain to create a command buffer per queue and
and per swapchain image.  Then when a vkQueuePresent occurs,
perform the blits in the same queue as the presented queue to
reduce race conditions.
This resolves the issue where a QueuePresent is presented inline
in a queue, but has no wait semaphore information.  Instead,
the QueuePresent relied upon the previous submit (also in the same
queue) to perform the synchronization.

Previously, the virtual swapchain had it's own Queue that it used
which caused a race condition resulting in old images being used.

Fixes: #1122
---
 .gitignore                                    |   1 -
 .../decode/vulkan_captured_swapchain.cpp      |   1 -
 framework/decode/vulkan_object_info.h         |  23 +-
 .../decode/vulkan_replay_consumer_base.cpp    |  62 ++
 .../decode/vulkan_replay_consumer_base.h      |  20 +-
 framework/decode/vulkan_virtual_swapchain.cpp | 775 +++++++++++-------
 framework/decode/vulkan_virtual_swapchain.h   |  46 +-
 .../generated_vulkan_replay_consumer.cpp      |  19 +-
 .../vulkan_generators/replay_overrides.json   |   2 +
 9 files changed, 619 insertions(+), 330 deletions(-)

diff --git a/.gitignore b/.gitignore
index 550568a71d..818a0a7ffa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,4 +7,3 @@ __pycache__/
 .vscode/
 *.db
 *.pyc
-
diff --git a/framework/decode/vulkan_captured_swapchain.cpp b/framework/decode/vulkan_captured_swapchain.cpp
index c4e176319b..659c120c49 100644
--- a/framework/decode/vulkan_captured_swapchain.cpp
+++ b/framework/decode/vulkan_captured_swapchain.cpp
@@ -279,7 +279,6 @@ void VulkanCapturedSwapchain::ProcessSetSwapchainImageStateCommand(
     const VulkanObjectInfoTable&                        object_info_table,
     SwapchainImageTracker&                              swapchain_image_tracker)
 {
-
     VkDevice       device    = device_info->handle;
     VkSwapchainKHR swapchain = swapchain_info->handle;
 
diff --git a/framework/decode/vulkan_object_info.h b/framework/decode/vulkan_object_info.h
index dd1f552eb7..3945f89d83 100644
--- a/framework/decode/vulkan_object_info.h
+++ b/framework/decode/vulkan_object_info.h
@@ -266,6 +266,7 @@ struct DeviceInfo : public VulkanObjectInfo<VkDevice>
     graphics::VulkanDevicePropertyFeatureInfo property_feature_info;
 
     std::unordered_map<uint32_t, VkDeviceQueueCreateFlags> queue_family_creation_flags;
+    std::vector<bool>                                      queue_family_index_enabled;
 
     std::vector<VkPhysicalDevice> replay_device_group;
 };
@@ -273,6 +274,8 @@ struct DeviceInfo : public VulkanObjectInfo<VkDevice>
 struct QueueInfo : public VulkanObjectInfo<VkQueue>
 {
     std::unordered_map<uint32_t, size_t> array_counts;
+    uint32_t                             family_index;
+    uint32_t                             queue_index;
 };
 
 struct SemaphoreInfo : public VulkanObjectInfo<VkSemaphore>
@@ -401,31 +404,13 @@ struct SwapchainKHRInfo : public VulkanObjectInfo<VkSwapchainKHR>
 
     // When replay is restricted to a specific surface, a dummy swapchain is created for the omitted surfaces, requiring
     // backing images.
+    uint32_t                  replay_image_count{ 0 };
     std::vector<ImageInfo>    image_infos;
     VkSwapchainCreateFlagsKHR image_flags{ 0 };
     VkFormat                  image_format{ VK_FORMAT_UNDEFINED };
     uint32_t                  image_array_layers{ 0 };
     VkImageUsageFlags         image_usage{ 0 };
     VkSharingMode             image_sharing_mode{ VK_SHARING_MODE_EXCLUSIVE };
-
-    // TODO: These values are used by the virtual swapchain.  They should be replaced with an opaque handle, similar to
-    // DeviceMemoryInfo::allocator_data, which is really a pointer to a struct that contains the virtual swapchain's
-    // internal info.  The memory for the struct referenced by the opaque handle would be managed by the virtual
-    // swapchain class, similar to the way that the VulkanRebindAllocator works.
-    struct VirtualImage
-    {
-        VkDeviceMemory                        memory{ VK_NULL_HANDLE };
-        VkImage                               image{ VK_NULL_HANDLE };
-        VulkanResourceAllocator::MemoryData   memory_allocator_data{ 0 };
-        VulkanResourceAllocator::ResourceData resource_allocator_data{ 0 };
-    };
-    uint32_t                     replay_image_count{ 0 };
-    std::vector<VirtualImage>    virtual_images; // Images created by replay, returned in place of the swapchain images.
-    std::vector<VkImage>         swapchain_images; // The real swapchain images.
-    VkQueue                      blit_queue{ VK_NULL_HANDLE };
-    VkCommandPool                blit_command_pool{ VK_NULL_HANDLE };
-    std::vector<VkCommandBuffer> blit_command_buffers;
-    std::vector<VkSemaphore>     blit_semaphores;
 };
 
 struct ValidationCacheEXTInfo : public VulkanObjectInfo<VkValidationCacheEXT>
diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp
index 7ba7029771..612247866d 100755
--- a/framework/decode/vulkan_replay_consumer_base.cpp
+++ b/framework/decode/vulkan_replay_consumer_base.cpp
@@ -41,8 +41,10 @@
 
 #include "generated/generated_vulkan_enum_to_string.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <limits>
+#include <numeric>
 #include <unordered_set>
 #include <future>
 
@@ -2664,6 +2666,19 @@ VulkanReplayConsumerBase::OverrideCreateDevice(VkResult            original_resu
             // Track state of physical device properties and features at device creation
             device_info->property_feature_info = property_feature_info;
 
+            // Keep track of what queue families this device is planning on using.  This information is
+            // very important if we end up using the VulkanVirtualSwapchain path.
+            auto max = [](uint32_t current_max, const VkDeviceQueueCreateInfo& dqci) {
+                return std::max(current_max, dqci.queueFamilyIndex);
+            };
+            uint32_t max_queue_family =
+                std::accumulate(modified_create_info.pQueueCreateInfos,
+                                modified_create_info.pQueueCreateInfos + modified_create_info.queueCreateInfoCount,
+                                0,
+                                max);
+            device_info->queue_family_index_enabled.clear();
+            device_info->queue_family_index_enabled.resize(max_queue_family + 1, false);
+
             for (uint32_t q = 0; q < modified_create_info.queueCreateInfoCount; ++q)
             {
                 const VkDeviceQueueCreateInfo* queue_create_info = &modified_create_info.pQueueCreateInfos[q];
@@ -2671,6 +2686,7 @@ VulkanReplayConsumerBase::OverrideCreateDevice(VkResult            original_resu
                        device_info->queue_family_creation_flags.end());
                 device_info->queue_family_creation_flags[queue_create_info->queueFamilyIndex] =
                     queue_create_info->flags;
+                device_info->queue_family_index_enabled[queue_create_info->queueFamilyIndex] = true;
             }
         }
 
@@ -2907,6 +2923,52 @@ VkResult VulkanReplayConsumerBase::OverrideEnumeratePhysicalDeviceGroups(
     return result;
 }
 
+void VulkanReplayConsumerBase::OverrideGetDeviceQueue(PFN_vkGetDeviceQueue           func,
+                                                      DeviceInfo*                    device_info,
+                                                      uint32_t                       queueFamilyIndex,
+                                                      uint32_t                       queueIndex,
+                                                      HandlePointerDecoder<VkQueue>* pQueue)
+{
+    VkDevice device = device_info->handle;
+    if (!pQueue->IsNull())
+    {
+        pQueue->SetHandleLength(1);
+    }
+    VkQueue* out_pQueue = pQueue->GetHandlePointer();
+
+    func(device, queueFamilyIndex, queueIndex, out_pQueue);
+
+    // Add tracking for which VkQueue objects are associated with what queue family and index.
+    // This is necessary for the virtual swapchain to determine which command buffer to use when
+    // Bliting the images on the Presenting Queue.
+    auto queue_info          = reinterpret_cast<QueueInfo*>(pQueue->GetConsumerData(0));
+    queue_info->family_index = queueFamilyIndex;
+    queue_info->queue_index  = queueIndex;
+}
+
+void VulkanReplayConsumerBase::OverrideGetDeviceQueue2(PFN_vkGetDeviceQueue2                             func,
+                                                       DeviceInfo*                                       device_info,
+                                                       StructPointerDecoder<Decoded_VkDeviceQueueInfo2>* pQueueInfo,
+                                                       HandlePointerDecoder<VkQueue>*                    pQueue)
+{
+    VkDevice                  device        = device_info->handle;
+    const VkDeviceQueueInfo2* in_pQueueInfo = pQueueInfo->GetPointer();
+    if (!pQueue->IsNull())
+    {
+        pQueue->SetHandleLength(1);
+    }
+    VkQueue* out_pQueue = pQueue->GetHandlePointer();
+
+    func(device, in_pQueueInfo, out_pQueue);
+
+    // Add tracking for which VkQueue objects are associated with what queue family and index.
+    // This is necessary for the virtual swapchain to determine which command buffer to use when
+    // Bliting the images on the Presenting Queue.
+    auto queue_info          = reinterpret_cast<QueueInfo*>(pQueue->GetConsumerData(0));
+    queue_info->family_index = in_pQueueInfo->queueFamilyIndex;
+    queue_info->queue_index  = in_pQueueInfo->queueIndex;
+}
+
 void VulkanReplayConsumerBase::OverrideGetPhysicalDeviceProperties(
     PFN_vkGetPhysicalDeviceProperties                         func,
     PhysicalDeviceInfo*                                       physical_device_info,
diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h
index e5a99099e8..fc6fc5e539 100644
--- a/framework/decode/vulkan_replay_consumer_base.h
+++ b/framework/decode/vulkan_replay_consumer_base.h
@@ -483,6 +483,16 @@ class VulkanReplayConsumerBase : public VulkanConsumer
         PointerDecoder<uint32_t>*                                     pToolCount,
         StructPointerDecoder<Decoded_VkPhysicalDeviceToolProperties>* pToolProperties);
 
+    void OverrideGetDeviceQueue(PFN_vkGetDeviceQueue           func,
+                                DeviceInfo*                    device_info,
+                                uint32_t                       queueFamilyIndex,
+                                uint32_t                       queueIndex,
+                                HandlePointerDecoder<VkQueue>* pQueue);
+    void OverrideGetDeviceQueue2(PFN_vkGetDeviceQueue2                             func,
+                                 DeviceInfo*                                       device_info,
+                                 StructPointerDecoder<Decoded_VkDeviceQueueInfo2>* pQueueInfo,
+                                 HandlePointerDecoder<VkQueue>*                    pQueue);
+
     VkResult OverrideWaitForFences(PFN_vkWaitForFences                  func,
                                    VkResult                             original_result,
                                    const DeviceInfo*                    device_info,
@@ -1132,7 +1142,6 @@ class VulkanReplayConsumerBase : public VulkanConsumer
     ActiveWindows                                                    active_windows_;
     const VulkanReplayOptions                                        options_;
     bool                                                             loading_trim_state_;
-    bool                                                             have_imported_semaphores_;
     SwapchainImageTracker                                            swapchain_image_tracker_;
     HardwareBufferMap                                                hardware_buffers_;
     HardwareBufferMemoryMap                                          hardware_buffer_memory_info_;
@@ -1142,7 +1151,14 @@ class VulkanReplayConsumerBase : public VulkanConsumer
     int32_t                                                          create_surface_count_;
     graphics::FpsInfo*                                               fps_info_;
 
-    // Used to track if any shadow sync objects are active to avoid checking if not needed
+    // Imported semaphores are semaphores that are used to track external memory.
+    // During replay, the external memory is not present (we have no Fds or handles to valid
+    // data), so we ignore those semaphores when they are encountered.
+    bool have_imported_semaphores_;
+
+    // Used to track if any shadow sync objects are active to avoid checking if not needed.
+    // SHadowed objects are ignored when they would have been unsignaled (waited on).
+    // [Currently set during a call to AcquireNextImage if the VkSurfaceKHR is VK_NULL_HANDLE.
     std::unordered_set<VkSemaphore> shadow_semaphores_;
     std::unordered_set<VkFence>     shadow_fences_;
 
diff --git a/framework/decode/vulkan_virtual_swapchain.cpp b/framework/decode/vulkan_virtual_swapchain.cpp
index 9eb1a34363..aba86a2dde 100644
--- a/framework/decode/vulkan_virtual_swapchain.cpp
+++ b/framework/decode/vulkan_virtual_swapchain.cpp
@@ -38,7 +38,8 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainKHR(PFN_vkCreateSwapchainKHR
                                                     const encode::InstanceTable*    instance_table,
                                                     const encode::DeviceTable*      device_table)
 {
-    VkDevice device = VK_NULL_HANDLE;
+    VkDevice                 device = VK_NULL_HANDLE;
+    VkSurfaceCapabilitiesKHR surfCapabilities{};
 
     if (device_info != nullptr)
     {
@@ -51,8 +52,7 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainKHR(PFN_vkCreateSwapchainKHR
     modified_create_info.imageUsage =
         modified_create_info.imageUsage | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
 
-    VkSurfaceCapabilitiesKHR surfCapabilities;
-    auto                     result = instance_table_->GetPhysicalDeviceSurfaceCapabilitiesKHR(
+    VkResult result = instance_table_->GetPhysicalDeviceSurfaceCapabilitiesKHR(
         physical_device, create_info->surface, &surfCapabilities);
     GFXRECON_ASSERT(result == VK_SUCCESS);
 
@@ -64,7 +64,19 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainKHR(PFN_vkCreateSwapchainKHR
     {
         modified_create_info.minImageCount = surfCapabilities.maxImageCount;
     }
-    return func(device, &modified_create_info, allocator, swapchain);
+
+    result = func(device, &modified_create_info, allocator, swapchain);
+    if (result == VK_SUCCESS && *swapchain != VK_NULL_HANDLE)
+    {
+        auto data = std::make_unique<SwapchainResourceData>();
+        if (data == nullptr)
+        {
+            GFXRECON_LOG_ERROR("Virtual swapchain failed creating swapchain resource data during vkCreateSwapchainKHR");
+            return VK_ERROR_OUT_OF_HOST_MEMORY;
+        }
+        swapchain_resources_.emplace(*swapchain, std::move(data));
+    }
+    return result;
 }
 
 void VulkanVirtualSwapchain::DestroySwapchainKHR(PFN_vkDestroySwapchainKHR    func,
@@ -89,20 +101,34 @@ void VulkanVirtualSwapchain::DestroySwapchainKHR(PFN_vkDestroySwapchainKHR    fu
             allocator->FreeMemoryDirect(image_info.memory, nullptr, image_info.memory_allocator_data);
         }
 
-        device_table_->FreeCommandBuffers(device,
-                                          swapchain_info->blit_command_pool,
-                                          static_cast<uint32_t>(swapchain_info->blit_command_buffers.size()),
-                                          swapchain_info->blit_command_buffers.data());
-        device_table_->DestroyCommandPool(device, swapchain_info->blit_command_pool, nullptr);
-
-        for (const SwapchainKHRInfo::VirtualImage& image_info : swapchain_info->virtual_images)
-        {
-            allocator->DestroyImageDirect(image_info.image, nullptr, image_info.resource_allocator_data);
-            allocator->FreeMemoryDirect(image_info.memory, nullptr, image_info.memory_allocator_data);
-        }
-        for (const auto semaphore : swapchain_info->blit_semaphores)
+        // Delete the virtual swapchain-specific swapchain resource data
+        if (swapchain_resources_.find(swapchain) != swapchain_resources_.end())
         {
-            device_table_->DestroySemaphore(device, semaphore, nullptr);
+            auto& swapchain_resources = swapchain_resources_[swapchain];
+            for (const VirtualImage& image_info : swapchain_resources->virtual_swapchain_images)
+            {
+                allocator->DestroyImageDirect(image_info.image, nullptr, image_info.resource_allocator_data);
+                allocator->FreeMemoryDirect(image_info.memory, nullptr, image_info.memory_allocator_data);
+            }
+
+            for (auto& copy_cmd_data : swapchain_resources->copy_cmd_data)
+            {
+                if (copy_cmd_data.second.command_pool != VK_NULL_HANDLE)
+                {
+                    device_table_->FreeCommandBuffers(
+                        device,
+                        copy_cmd_data.second.command_pool,
+                        static_cast<uint32_t>(copy_cmd_data.second.command_buffers.size()),
+                        copy_cmd_data.second.command_buffers.data());
+                    device_table_->DestroyCommandPool(device, copy_cmd_data.second.command_pool, nullptr);
+                }
+                for (auto& semaphore : copy_cmd_data.second.semaphores)
+                {
+                    device_table_->DestroySemaphore(device, semaphore, nullptr);
+                }
+            }
+
+            swapchain_resources_.erase(swapchain);
         }
     }
     func(device, swapchain, allocator);
@@ -115,10 +141,11 @@ VkResult VulkanVirtualSwapchain::GetSwapchainImagesKHR(PFN_vkGetSwapchainImagesK
                                                        uint32_t*                   image_count,
                                                        VkImage*                    images)
 {
-    VkDevice             device             = VK_NULL_HANDLE;
-    VkSwapchainKHR       swapchain          = VK_NULL_HANDLE;
-    uint32_t*            replay_image_count = nullptr;
-    std::vector<VkImage> replay_swapchain_images;
+    VkDevice       device             = VK_NULL_HANDLE;
+    VkSwapchainKHR swapchain          = VK_NULL_HANDLE;
+    uint32_t*      replay_image_count = nullptr;
+    VkResult       result;
+    VkImage*       replay_images = images;
 
     if (device_info != nullptr)
     {
@@ -129,201 +156,377 @@ VkResult VulkanVirtualSwapchain::GetSwapchainImagesKHR(PFN_vkGetSwapchainImagesK
     {
         swapchain          = swapchain_info->handle;
         replay_image_count = &swapchain_info->replay_image_count;
+    }
 
-        if (images != nullptr)
-        {
-            replay_swapchain_images.resize(*replay_image_count);
-        }
+    // Get the swapchain resource data so we have access to the virtual swapchain-specific information.
+    if (swapchain == VK_NULL_HANDLE || swapchain_resources_.find(swapchain) == swapchain_resources_.end())
+    {
+        GFXRECON_LOG_ERROR(
+            "Virtual swapchain vkGetSwapchainImagesKHR missing swapchain resource data for swapchain (ID = %" PRIu64
+            ")",
+            swapchain_info->capture_id);
+    }
+    else if (images != nullptr)
+    {
+        auto& swapchain_resources = swapchain_resources_[swapchain];
+        swapchain_resources->replay_swapchain_images.resize(*replay_image_count);
+
+        // Use the resized replay images vector to contain the replay device swapchain images.
+        replay_images = swapchain_resources->replay_swapchain_images.data();
     }
 
     // TODO: Adjust the swapchain image format if the specified format is not supported by the replay device.
 
-    auto result = func(device, swapchain, replay_image_count, replay_swapchain_images.data());
+    result = func(device, swapchain, replay_image_count, replay_images);
 
     if ((result == VK_SUCCESS) && (image_count != nullptr))
     {
-        if (swapchain_info->blit_command_pool == VK_NULL_HANDLE)
+        // Return the capture count.  The virtual swapchain will create a number of virtual images equal to the capture
+        // count.  The virtual images will be returned to the caller in place of the real swapchain images.
+        (*image_count) = capture_image_count;
+
+        if (images == nullptr || device_info == nullptr || swapchain_info == nullptr)
         {
-            device_table_->GetDeviceQueue(
-                device, swapchain_info->queue_family_indices[0], 0, &swapchain_info->blit_queue);
-
-            VkCommandPoolCreateInfo command_pool_create_info = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO };
-            command_pool_create_info.flags =
-                VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
-            command_pool_create_info.queueFamilyIndex = swapchain_info->queue_family_indices[0];
-            VkResult result                           = device_table_->CreateCommandPool(
-                device, &command_pool_create_info, nullptr, &swapchain_info->blit_command_pool);
-            if (result != VK_SUCCESS)
+            return result;
+        }
+
+        bool     found_copy_queue_family           = false;
+        uint32_t copy_queue_family_index           = VK_QUEUE_FAMILY_IGNORED;
+        bool     found_transfer_queue_family_index = false;
+        uint32_t transfer_queue_family_index       = 0;
+
+        // Determine what queue to use for the initial virtual image setup
+        VkQueue                              initial_copy_queue = VK_NULL_HANDLE;
+        uint32_t                             property_count     = 0;
+        std::vector<VkQueueFamilyProperties> props;
+
+        instance_table_->GetPhysicalDeviceQueueFamilyProperties(device_info->parent, &property_count, nullptr);
+        props.resize(property_count);
+        instance_table_->GetPhysicalDeviceQueueFamilyProperties(device_info->parent, &property_count, props.data());
+
+        for (uint32_t queue_family_index = 0; queue_family_index < property_count; ++queue_family_index)
+        {
+            // If we're past the point of enabled queues, then stop looking because we really can't enable
+            // a queue that isn't flagged during device creation.
+            if (queue_family_index >= static_cast<uint32_t>(device_info->queue_family_index_enabled.size()))
             {
-                return result;
+                break;
             }
 
-            for (uint32_t i = 0; i < capture_image_count; ++i)
+            if (!device_info->queue_family_index_enabled[queue_family_index])
             {
-                VkCommandBufferAllocateInfo allocate_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO };
-                allocate_info.pNext                       = nullptr;
-                allocate_info.commandPool                 = swapchain_info->blit_command_pool;
-                allocate_info.level                       = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
-                allocate_info.commandBufferCount          = 1;
-                VkCommandBuffer command_buffer            = VK_NULL_HANDLE;
-                result = device_table_->AllocateCommandBuffers(device, &allocate_info, &command_buffer);
-                if (result != VK_SUCCESS)
-                {
-                    return result;
-                }
-                swapchain_info->blit_command_buffers.emplace_back(command_buffer);
+                continue;
+            }
 
-                VkSemaphoreCreateInfo semaphore_create_info = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO };
-                semaphore_create_info.pNext                 = nullptr;
-                semaphore_create_info.flags                 = 0;
-                VkSemaphore semaphore                       = VK_NULL_HANDLE;
-                result = device_table_->CreateSemaphore(device, &semaphore_create_info, nullptr, &semaphore);
-                if (result != VK_SUCCESS)
-                {
-                    return result;
-                }
-                swapchain_info->blit_semaphores.emplace_back(semaphore);
+            // If we find a graphics queue, we're good, so grab it and bail
+            if (props[queue_family_index].queueFlags & VK_QUEUE_GRAPHICS_BIT)
+            {
+                copy_queue_family_index = queue_family_index;
+                found_copy_queue_family = true;
+                break;
             }
-        }
-        // Return the capture count.  The virtual swapchain will create a number of virtual images equal to the capture
-        // count.  The virtual images will be returned to the caller in place of the real swapchain images.
-        (*image_count) = capture_image_count;
 
-        if ((device_info != nullptr) && (swapchain_info != nullptr) && (images != nullptr))
+            // Find a transfer queue as an alternative, just in case
+            if (!found_transfer_queue_family_index && props[queue_family_index].queueFlags & VK_QUEUE_TRANSFER_BIT)
+            {
+                transfer_queue_family_index       = queue_family_index;
+                found_transfer_queue_family_index = true;
+            }
+        }
+        if (!found_copy_queue_family)
         {
-            // Store the retrieved images and create new images to return to the caller.  The replay call always
-            // retrieves the full swapchain image count, so this only needs to be done once.  It does not need to handle
-            // the VK_INCOMPLETE case that the virtual image creation must handle.
-            if (swapchain_info->swapchain_images.empty())
+            if (!found_transfer_queue_family_index)
             {
-                swapchain_info->swapchain_images = std::vector<VkImage>(
-                    replay_swapchain_images.data(), std::next(replay_swapchain_images.data(), *replay_image_count));
+                GFXRECON_LOG_ERROR("Virtual swapchain failed finding a queue to create initial virtual swapchain "
+                                   "images for swapchain (ID = %" PRIu64 ")",
+                                   swapchain_info->capture_id);
+                return VK_ERROR_INITIALIZATION_FAILED;
             }
+            copy_queue_family_index = transfer_queue_family_index;
+            GFXRECON_LOG_INFO("Virtual swapchain using transfer queue %d to create initial virtual swapchain "
+                              "images for swapchain (ID = %" PRIu64 ")",
+                              transfer_queue_family_index,
+                              swapchain_info->capture_id);
+        }
+        device_table_->GetDeviceQueue(device, copy_queue_family_index, 0, &initial_copy_queue);
+        if (initial_copy_queue == VK_NULL_HANDLE)
+        {
+            GFXRECON_LOG_ERROR("Virtual swapchain failed getting device queue %d to create initial virtual swapchain "
+                               "images for swapchain (ID = %" PRIu64 ")",
+                               copy_queue_family_index,
+                               swapchain_info->capture_id);
+            return VK_ERROR_INITIALIZATION_FAILED;
+        }
+
+        auto& swapchain_resources = swapchain_resources_[swapchain];
 
-            // If the call was made more than once because the first call returned VK_INCOMPLETE, only the new images
-            // returned by the second call will have virtual images created and appended to the end of the virtual image
-            // array.
-            if (capture_image_count != swapchain_info->virtual_images.size())
+        for (uint32_t queue_family_index = 0; queue_family_index < property_count; ++queue_family_index)
+        {
+            if (swapchain_resources->copy_cmd_data.find(queue_family_index) == swapchain_resources->copy_cmd_data.end())
             {
-                uint32_t start_index = static_cast<uint32_t>(swapchain_info->virtual_images.size());
-
-                // TODO: This is the same code used in VulkanReplayConsumerBase::OverrideGetSwapchainImagesKHR, which
-                // should be moved to a shared graphics utility function.
-
-                //  Create an image for the virtual swapchain.  Based on vkspec.html#swapchain-wsi-image-create-info.
-                VkImageCreateInfo image_create_info = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
-                image_create_info.pNext             = nullptr;
-                image_create_info.flags             = 0;
-                image_create_info.imageType         = VK_IMAGE_TYPE_2D;
-                image_create_info.format            = swapchain_info->format;
-                image_create_info.extent            = { swapchain_info->width, swapchain_info->height, 1 };
-                image_create_info.mipLevels         = 1;
-                image_create_info.arrayLayers       = swapchain_info->image_array_layers;
-                image_create_info.samples           = VK_SAMPLE_COUNT_1_BIT;
-                image_create_info.tiling            = VK_IMAGE_TILING_OPTIMAL;
-                image_create_info.usage             = swapchain_info->image_usage | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
-                image_create_info.sharingMode       = swapchain_info->image_sharing_mode;
-                image_create_info.queueFamilyIndexCount =
-                    static_cast<uint32_t>(swapchain_info->queue_family_indices.size());
-                image_create_info.pQueueFamilyIndices = swapchain_info->queue_family_indices.data();
-                image_create_info.initialLayout       = VK_IMAGE_LAYOUT_UNDEFINED;
-
-                if ((swapchain_info->image_flags & VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR) ==
-                    VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR)
+                VkBool32 supported = VK_FALSE;
+
+                // We only want to look at a given queue if it was enabled during device creation time
+                // and if it supports present.  Otherwise, we don't need to create a command pool,
+                // command buffers, and semaphores for performing the swapchain copy.
+                if (device_info->queue_family_index_enabled.size() <= queue_family_index ||
+                    !device_info->queue_family_index_enabled[queue_family_index])
                 {
-                    image_create_info.flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
+                    GFXRECON_LOG_DEBUG("Virtual swapchain skipping creating blit info for queue family %d because it "
+                                       "was not enabled by the device",
+                                       queue_family_index);
+                    continue;
                 }
 
-                VkCommandBufferBeginInfo begin_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
-                begin_info.pNext                    = nullptr;
-                begin_info.flags                    = 0;
-                begin_info.pInheritanceInfo         = nullptr;
-
-                auto command_buffer = swapchain_info->blit_command_buffers[0];
-                result              = device_table_->ResetCommandBuffer(command_buffer, 0);
-                if (result != VK_SUCCESS)
+                result = instance_table_->GetPhysicalDeviceSurfaceSupportKHR(
+                    device_info->parent, queue_family_index, swapchain_info->surface, &supported);
+                if (result != VK_SUCCESS || supported == VK_FALSE)
                 {
-                    return result;
+                    GFXRECON_LOG_DEBUG(
+                        "Virtual swapchain skipping queue family %d since present support is not present "
+                        "for swapchain (ID = %" PRIu64 ")",
+                        queue_family_index,
+                        swapchain_info->capture_id);
+                    continue;
                 }
 
-                result = device_table_->BeginCommandBuffer(command_buffer, &begin_info);
+                // Create one command pool per queue.
+                VkCommandPoolCreateInfo command_pool_create_info = {
+                    VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,                                             // sType
+                    nullptr,                                                                                // pNext
+                    VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, // flags
+                    static_cast<uint32_t>(queue_family_index) // queueFamilyIndex
+                };
+
+                CopyCmdData copy_cmd_data = {};
+                result                    = device_table_->CreateCommandPool(
+                    device, &command_pool_create_info, nullptr, &copy_cmd_data.command_pool);
                 if (result != VK_SUCCESS)
                 {
+                    GFXRECON_LOG_ERROR("Virtual swapchain failed creating command pool %d for swapchain (ID = %" PRIu64
+                                       ")",
+                                       queue_family_index,
+                                       swapchain_info->capture_id);
                     return result;
                 }
+                swapchain_resources->copy_cmd_data.emplace(queue_family_index, std::move(copy_cmd_data));
+            }
 
-                for (uint32_t i = start_index; i < capture_image_count; ++i)
-                {
-                    SwapchainKHRInfo::VirtualImage image;
-
-                    result = CreateSwapchainImage(device_info, image_create_info, image);
+            auto& copy_cmd_data = swapchain_resources->copy_cmd_data[queue_family_index];
 
+            // Make sure we have enough storage for each of our tracked components (Command pools,
+            // Command Buffers, Semaphores, etc) as many queue families that are available.
+            // This is because at any point, the application may get a Device queue from that family and
+            // use it during the present.
+            uint32_t start_size = static_cast<uint32_t>(copy_cmd_data.command_buffers.size());
+            uint32_t new_count  = property_count;
+            if (start_size < new_count)
+            {
+                // Create one command buffer per queue per swapchain image so that we don't reset a command buffer that
+                // may be in active use.
+                uint32_t command_buffer_count = static_cast<uint32_t>(copy_cmd_data.command_buffers.size());
+                if (command_buffer_count < capture_image_count)
+                {
+                    copy_cmd_data.command_buffers.resize(capture_image_count);
+
+                    uint32_t                    new_count     = capture_image_count - command_buffer_count;
+                    VkCommandBufferAllocateInfo allocate_info = {
+                        VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, // sType
+                        nullptr,                                        // pNext
+                        copy_cmd_data.command_pool,                     // commandPool
+                        VK_COMMAND_BUFFER_LEVEL_PRIMARY,                // level
+                        new_count                                       // commandBufferCount
+                    };
+
+                    result = device_table_->AllocateCommandBuffers(
+                        device, &allocate_info, &copy_cmd_data.command_buffers[command_buffer_count]);
                     if (result != VK_SUCCESS)
                     {
-                        GFXRECON_LOG_ERROR("Failed to create virtual swapchain image for swapchain (ID = %" PRIu64 ")",
+                        GFXRECON_LOG_ERROR("Virtual swapchain failed allocating internal command buffer %d for "
+                                           "swapchain (ID = %" PRIu64 ")",
+                                           queue_family_index,
                                            swapchain_info->capture_id);
-                        break;
+                        return result;
                     }
-                    swapchain_info->virtual_images.emplace_back(std::move(image));
                 }
 
-                VkImageMemoryBarrier barrier = {
-                    VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
-                    nullptr,
-                    VK_ACCESS_NONE,
-                    VK_ACCESS_NONE,
-                    VK_IMAGE_LAYOUT_UNDEFINED,
-                    VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
-                    VK_QUEUE_FAMILY_IGNORED,
-                    VK_QUEUE_FAMILY_IGNORED,
-                    VK_NULL_HANDLE,
-                    VkImageSubresourceRange{
-                        VK_IMAGE_ASPECT_COLOR_BIT,
-                        0,
-                        image_create_info.mipLevels,
-                        0,
-                        image_create_info.arrayLayers,
-                    },
-                };
-
-                for (uint32_t i = 0; i < *replay_image_count; ++i)
+                uint32_t semaphore_count = static_cast<uint32_t>(copy_cmd_data.semaphores.size());
+                if (semaphore_count < capture_image_count)
                 {
-                    barrier.image = replay_swapchain_images[i];
-                    device_table_->CmdPipelineBarrier(command_buffer,
-                                                      VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
-                                                      VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
-                                                      0,
-                                                      0,
-                                                      nullptr,
-                                                      0,
-                                                      nullptr,
-                                                      1,
-                                                      &barrier);
+                    copy_cmd_data.semaphores.resize(capture_image_count);
+
+                    for (uint32_t ii = semaphore_count; ii < capture_image_count; ++ii)
+                    {
+                        VkSemaphoreCreateInfo semaphore_create_info = {
+                            VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, // sType
+                            nullptr,                                 // pNext
+                            0                                        // flags
+                        };
+
+                        VkSemaphore semaphore = 0;
+                        result = device_table_->CreateSemaphore(device, &semaphore_create_info, nullptr, &semaphore);
+                        if (result != VK_SUCCESS)
+                        {
+                            GFXRECON_LOG_ERROR(
+                                "Virtual swapchain failed creating internal copy semaphore for swapchain (ID = %" PRIu64
+                                ")",
+                                swapchain_info->capture_id);
+                            return result;
+                        }
+                        copy_cmd_data.semaphores[ii] = semaphore;
+                    }
                 }
+            }
+        }
 
-                result = device_table_->EndCommandBuffer(command_buffer);
+        uint32_t virtual_swapchain_count = static_cast<uint32_t>(swapchain_resources->virtual_swapchain_images.size());
 
-                VkPipelineStageFlags wait_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
+        // If the call was made more than once because the first call returned VK_INCOMPLETE, only the new images
+        // returned by the second call will have virtual images created and appended to the end of the virtual image
+        // array.
+        if (virtual_swapchain_count < capture_image_count)
+        {
+            // TODO: This is the same code used in VulkanReplayConsumerBase::OverrideGetSwapchainImagesKHR, which
+            // should be moved to a shared graphics utility function.
+
+            //  Create an image for the virtual swapchain.  Based on vkspec.html#swapchain-wsi-image-create-info.
+            VkImageCreateInfo image_create_info = {
+                VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,                                // sType,
+                nullptr,                                                            // pNext
+                0,                                                                  // flags
+                VK_IMAGE_TYPE_2D,                                                   // imageType
+                swapchain_info->format,                                             // format
+                VkExtent3D{ swapchain_info->width, swapchain_info->height, 1 },     // extent
+                1,                                                                  // mipLevels
+                swapchain_info->image_array_layers,                                 // arrayLayers
+                VK_SAMPLE_COUNT_1_BIT,                                              // samples
+                VK_IMAGE_TILING_OPTIMAL,                                            // tiling
+                swapchain_info->image_usage | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,      // usage
+                swapchain_info->image_sharing_mode,                                 // sharingMode
+                static_cast<uint32_t>(swapchain_info->queue_family_indices.size()), // queueFamilyIndexCount
+                swapchain_info->queue_family_indices.data(),                        // pQueueFamilyIndices
+                VK_IMAGE_LAYOUT_UNDEFINED                                           // initialLayout
+            };
+
+            if ((swapchain_info->image_flags & VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR) ==
+                VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR)
+            {
+                image_create_info.flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
+            }
 
-                VkSubmitInfo submit_info       = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
-                submit_info.commandBufferCount = 1;
-                submit_info.pCommandBuffers    = &command_buffer;
+            VkCommandBufferBeginInfo begin_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
+            begin_info.pNext                    = nullptr;
+            begin_info.flags                    = 0;
+            begin_info.pInheritanceInfo         = nullptr;
+
+            auto command_buffer = swapchain_resources->copy_cmd_data[copy_queue_family_index].command_buffers[0];
+
+            result = device_table_->ResetCommandBuffer(command_buffer, 0);
+            if (result != VK_SUCCESS)
+            {
+                GFXRECON_LOG_ERROR(
+                    "Virtual swapchain failed resetting internal command buffer %d for swapchain (ID = %" PRIu64 ")",
+                    copy_queue_family_index,
+                    swapchain_info->capture_id);
+                return result;
+            }
+
+            result = device_table_->BeginCommandBuffer(command_buffer, &begin_info);
+            if (result != VK_SUCCESS)
+            {
+                GFXRECON_LOG_ERROR(
+                    "Virtual swapchain failed starting internal command buffer %d for swapchain (ID = %" PRIu64 ")",
+                    copy_queue_family_index,
+                    swapchain_info->capture_id);
+                return result;
+            }
+
+            for (uint32_t i = virtual_swapchain_count; i < capture_image_count; ++i)
+            {
+                VirtualImage image;
+
+                result = CreateVirtualSwapchainImage(device_info, image_create_info, image);
 
-                result = device_table_->QueueSubmit(swapchain_info->blit_queue, 1, &submit_info, VK_NULL_HANDLE);
-                if (result != VK_SUCCESS)
-                {
-                    return result;
-                }
-                result = device_table_->QueueWaitIdle(swapchain_info->blit_queue);
                 if (result != VK_SUCCESS)
                 {
-                    return result;
+                    GFXRECON_LOG_ERROR("Failed to create virtual swapchain image for swapchain (ID = %" PRIu64 ")",
+                                       swapchain_info->capture_id);
+                    break;
                 }
+                swapchain_resources->virtual_swapchain_images.emplace_back(std::move(image));
             }
-            for (uint32_t i = 0; i < capture_image_count; ++i)
+
+            VkImageMemoryBarrier barrier = {
+                VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, // sType
+                nullptr,                                // pNext
+                VK_ACCESS_NONE,                         // srcAccessMask
+                VK_ACCESS_NONE,                         // dstAccessMask
+                VK_IMAGE_LAYOUT_UNDEFINED,              // oldLayout
+                VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,        // newLayout
+                VK_QUEUE_FAMILY_IGNORED,                // srcQueueFamilyIndex
+                VK_QUEUE_FAMILY_IGNORED,                // dstQueueFamilyIndex
+                VK_NULL_HANDLE,                         // image
+                VkImageSubresourceRange{
+                    VK_IMAGE_ASPECT_COLOR_BIT,
+                    0,
+                    image_create_info.mipLevels,
+                    0,
+                    image_create_info.arrayLayers,
+                }, // subResourceRange
+            };
+
+            for (uint32_t i = 0; i < *replay_image_count; ++i)
             {
-                images[i] = swapchain_info->virtual_images[i].image;
+                barrier.image = swapchain_resources->replay_swapchain_images[i];
+                device_table_->CmdPipelineBarrier(command_buffer,
+                                                  VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
+                                                  VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
+                                                  0,
+                                                  0,
+                                                  nullptr,
+                                                  0,
+                                                  nullptr,
+                                                  1,
+                                                  &barrier);
             }
+
+            result = device_table_->EndCommandBuffer(command_buffer);
+            if (result != VK_SUCCESS)
+            {
+                GFXRECON_LOG_ERROR(
+                    "Virtual swapchain failed ending internal command buffer %d for swapchain (ID = %" PRIu64 ")",
+                    copy_queue_family_index,
+                    swapchain_info->capture_id);
+                return result;
+            }
+
+            VkSubmitInfo submit_info       = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
+            submit_info.commandBufferCount = 1;
+            submit_info.pCommandBuffers    = &command_buffer;
+
+            result = device_table_->QueueSubmit(initial_copy_queue, 1, &submit_info, VK_NULL_HANDLE);
+            if (result != VK_SUCCESS)
+            {
+                GFXRECON_LOG_ERROR(
+                    "Virtual swapchain failed submitting internal command buffer %d for swapchain (ID = %" PRIu64 ")",
+                    copy_queue_family_index,
+                    swapchain_info->capture_id);
+                return result;
+            }
+            result = device_table_->QueueWaitIdle(initial_copy_queue);
+            if (result != VK_SUCCESS)
+            {
+                GFXRECON_LOG_ERROR(
+                    "Virtual swapchain failed waiting for internal command buffer %d for swapchain (ID = %" PRIu64 ")",
+                    copy_queue_family_index,
+                    swapchain_info->capture_id);
+                return result;
+            }
+        }
+
+        for (uint32_t i = 0; i < capture_image_count; ++i)
+        {
+            images[i] = swapchain_resources->virtual_swapchain_images[i].image;
         }
     }
 
@@ -367,6 +570,7 @@ VkResult VulkanVirtualSwapchain::AcquireNextImageKHR(PFN_vkAcquireNextImageKHR f
 {
     VkDevice       device    = VK_NULL_HANDLE;
     VkSwapchainKHR swapchain = VK_NULL_HANDLE;
+    VkResult       result    = VK_NOT_READY;
 
     if (device_info != nullptr)
     {
@@ -378,7 +582,14 @@ VkResult VulkanVirtualSwapchain::AcquireNextImageKHR(PFN_vkAcquireNextImageKHR f
         swapchain = swapchain_info->handle;
     }
 
-    return func(device, swapchain, timeout, semaphore, fence, image_index);
+    result = func(device, swapchain, timeout, semaphore, fence, image_index);
+    if (result != VK_SUCCESS)
+    {
+        GFXRECON_LOG_ERROR("Virtual swapchain failed AcquireNextImageKHR 0x%08x for swapchain (ID = %" PRIu64 ")",
+                           result,
+                           swapchain_info->capture_id);
+    }
+    return result;
 }
 
 VkResult VulkanVirtualSwapchain::AcquireNextImage2KHR(PFN_vkAcquireNextImage2KHR       func,
@@ -395,7 +606,14 @@ VkResult VulkanVirtualSwapchain::AcquireNextImage2KHR(PFN_vkAcquireNextImage2KHR
         device = device_info->handle;
     }
 
-    return func(device, acquire_info, image_index);
+    VkResult result = func(device, acquire_info, image_index);
+    if (result != VK_SUCCESS)
+    {
+        GFXRECON_LOG_ERROR("Virtual swapchain failed AcquireNextImage2KHR 0x%08x for swapchain (ID = %" PRIu64 ")",
+                           result,
+                           swapchain_info->capture_id);
+    }
+    return result;
 }
 
 VkResult VulkanVirtualSwapchain::QueuePresentKHR(PFN_vkQueuePresentKHR                 func,
@@ -404,22 +622,21 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(PFN_vkQueuePresentKHR
                                                  const QueueInfo*                      queue_info,
                                                  const VkPresentInfoKHR*               present_info)
 {
-    VkQueue queue = VK_NULL_HANDLE;
+    VkResult result = VK_ERROR_UNKNOWN;
     if (queue_info == nullptr)
     {
         return VK_ERROR_FEATURE_NOT_PRESENT;
     }
-    queue = queue_info->handle;
 
-    // TODO: Note that this blit could also be used to scale the image, which would allow replay to support an option
+    VkQueue  queue              = queue_info->handle;
+    uint32_t queue_family_index = queue_info->family_index;
+
+    // TODO: Note that this copy could also be used to scale the image, which would allow replay to support an option
     // for changing the window/swapchain size when the virtual swapchain mode is active.  The virtual image would
     // continue to use the captured swapchain image size, and be scaled to the replay swapchain image size with
     // vkCmdBlitImage.
 
-    VkCommandBufferBeginInfo begin_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
-    begin_info.pNext                    = nullptr;
-    begin_info.flags                    = 0;
-    begin_info.pInheritanceInfo         = nullptr;
+    VkCommandBufferBeginInfo begin_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, nullptr, 0, nullptr };
 
     VkImageMemoryBarrier initial_barrier_virtual_image;
     VkImageMemoryBarrier initial_barrier_swapchain_image;
@@ -458,56 +675,89 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(PFN_vkQueuePresentKHR
     final_barrier_swapchain_image           = final_barrier_virtual_image;
     final_barrier_swapchain_image.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
 
-    VkImageSubresourceLayers subresource = {
-        VK_IMAGE_ASPECT_COLOR_BIT,
-        0,
-        0,
-        0,
-    };
+    VkImageSubresourceLayers subresource    = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 0 };
+    VkOffset3D               offset         = { 0, 0, 0 };
+    auto                     swapchainCount = present_info->swapchainCount;
+    std::vector<VkSemaphore> present_wait_semaphores;
+
+    // TODO: There is a potential issue here where a vkQueuePresent comes in on a queue (let's call
+    // it QueueX) which does not support vkCmdCopyImage (i.e. a video-only queue).  In that case,
+    // we would need to insert an emtpy command buffer into the command stream of QueueX which
+    // triggers a semaphore (let's say SemA), then we would need to submit the vkCmdCopyImage in a
+    // command buffer on a queue that supports it (let's say QueueY) which will wait on SemA to
+    // start and signaling another semaphore (SemB) when it is done.  Then, we need to add the
+    // QueuePresent to QueueX, but waiting on SemB before it executes.  And that is assuming that
+    // the buffer image is even accessible on both Queues!
+
+    for (uint32_t i = 0; i < swapchainCount; ++i)
+    {
+        const auto* swapchain_info      = swapchain_infos[i];
+        uint32_t    capture_image_index = capture_image_indices[i];
+        uint32_t    replay_image_index  = present_info->pImageIndices[i];
 
-    VkOffset3D offsets[2] = {
+        // Get the per swapchain resource data so we have access to the virtual swapchain-specific information.
+        if (swapchain_resources_.find(swapchain_info->handle) == swapchain_resources_.end())
         {
-            0,
-            0,
-            0,
-        },
+            GFXRECON_LOG_ERROR(
+                "Virtual swapchain vkQueuePresentKHR missing swapchain resource data for swapchain (ID = %" PRIu64 ")",
+                swapchain_info->capture_id);
+            continue;
+        }
+
+        auto& swapchain_resources = swapchain_resources_[swapchain_info->handle];
+        assert(swapchain_resources != nullptr);
+
+        // Find the appropriate CommandCopyData struct for this queue family
+        if (swapchain_resources->copy_cmd_data.find(queue_family_index) == swapchain_resources->copy_cmd_data.end())
         {
-            0,
-            0,
-            1,
-        },
-    };
+            GFXRECON_LOG_ERROR("Virtual swapchain vkQueuePresentKHR missing swapchain resource copy command data for "
+                               "queue (Handle %" PRIu64 ") in swapchain (ID = %" PRIu64 ")",
+                               queue,
+                               swapchain_info->capture_id);
+            continue;
+        }
 
-    auto                     length = present_info->swapchainCount;
-    std::vector<VkSemaphore> semaphores;
+        const auto& virtual_image = swapchain_resources->virtual_swapchain_images[capture_image_index];
+        const auto& replay_image  = swapchain_resources->replay_swapchain_images[replay_image_index];
 
-    for (uint32_t i = 0; i < length; ++i)
-    {
-        uint32_t    capture_image_index = capture_image_indices[i];
-        uint32_t    replay_image_index  = present_info->pImageIndices[i];
-        const auto* swapchain_info      = swapchain_infos[i];
-        const auto& virtual_image       = swapchain_info->virtual_images[capture_image_index];
-        const auto& swapchain_image     = swapchain_info->swapchain_images[replay_image_index];
+        // Use a command buffer and semaphore from the same queue index
+        auto& copy_cmd_data  = swapchain_resources->copy_cmd_data[queue_family_index];
+        auto  command_buffer = copy_cmd_data.command_buffers[capture_image_index];
+        auto  copy_semaphore = copy_cmd_data.semaphores[capture_image_index];
 
-        auto     command_buffer = swapchain_info->blit_command_buffers[capture_image_index];
-        VkResult result         = device_table_->ResetCommandBuffer(command_buffer, 0);
+        std::vector<VkSemaphore> wait_semaphores;
+        std::vector<VkSemaphore> signal_semaphores;
+
+        // Only wait for the present semaphore dependencies on the first copy command buffer.
+        // The others will automatically inherit that dependency because of their order in the
+        // command buffer.
+        if (i == 0 && present_info->waitSemaphoreCount > 0)
+        {
+            wait_semaphores.assign(present_info->pWaitSemaphores,
+                                   present_info->pWaitSemaphores + present_info->waitSemaphoreCount);
+        }
+
+        // Only trigger a semaphore on the last copy
+        if (i == swapchainCount - 1)
+        {
+            signal_semaphores.push_back(copy_semaphore);
+            present_wait_semaphores.emplace_back(copy_semaphore);
+        }
+
+        result = device_table_->ResetCommandBuffer(command_buffer, 0);
         if (result != VK_SUCCESS)
         {
             return result;
         }
-
         result = device_table_->BeginCommandBuffer(command_buffer, &begin_info);
         if (result != VK_SUCCESS)
         {
             return result;
         }
 
-        auto semaphore = swapchain_info->blit_semaphores[capture_image_index];
-        semaphores.emplace_back(semaphore);
-
         initial_barrier_virtual_image.image                         = virtual_image.image;
         initial_barrier_virtual_image.subresourceRange.layerCount   = swapchain_info->image_array_layers;
-        initial_barrier_swapchain_image.image                       = swapchain_image;
+        initial_barrier_swapchain_image.image                       = replay_image;
         initial_barrier_swapchain_image.subresourceRange.layerCount = swapchain_info->image_array_layers;
 
         device_table_->CmdPipelineBarrier(command_buffer,
@@ -532,29 +782,23 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(PFN_vkQueuePresentKHR
                                           1,
                                           &initial_barrier_swapchain_image);
 
-        subresource.layerCount = swapchain_info->image_array_layers;
-
-        offsets[1].x     = static_cast<int32_t>(swapchain_info->width);
-        offsets[1].y     = static_cast<int32_t>(swapchain_info->height);
-        VkImageBlit blit = {
-            subresource,
-            { offsets[0], offsets[1] },
-            subresource,
-            { offsets[0], offsets[1] },
-        };
+        subresource.layerCount   = swapchain_info->image_array_layers;
+        VkExtent3D  image_extent = { swapchain_info->width, swapchain_info->height, 1 };
+        VkImageCopy image_copy   = { subresource, offset, subresource, offset, image_extent };
 
-        device_table_->CmdBlitImage(command_buffer,
+        // NOTE: vkCmdCopyImage works on Queues of types including Graphics, Compute
+        //       and Transfer.  So should work on any queues we get a vkQueuePresentKHR from.
+        device_table_->CmdCopyImage(command_buffer,
                                     virtual_image.image,
                                     VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
-                                    swapchain_image,
+                                    replay_image,
                                     VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
                                     1,
-                                    &blit,
-                                    VK_FILTER_NEAREST);
+                                    &image_copy);
 
         final_barrier_virtual_image.image                         = virtual_image.image;
         final_barrier_virtual_image.subresourceRange.layerCount   = swapchain_info->image_array_layers;
-        final_barrier_swapchain_image.image                       = swapchain_image;
+        final_barrier_swapchain_image.image                       = replay_image;
         final_barrier_swapchain_image.subresourceRange.layerCount = swapchain_info->image_array_layers;
 
         device_table_->CmdPipelineBarrier(command_buffer,
@@ -587,16 +831,24 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(PFN_vkQueuePresentKHR
 
         VkPipelineStageFlags wait_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
 
-        VkSubmitInfo submit_info         = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
-        submit_info.waitSemaphoreCount   = present_info->waitSemaphoreCount;
-        submit_info.pWaitSemaphores      = present_info->pWaitSemaphores;
+        VkSubmitInfo submit_info       = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
+        submit_info.waitSemaphoreCount = static_cast<uint32_t>(wait_semaphores.size());
+        if (present_info->waitSemaphoreCount > 0)
+        {
+            submit_info.pWaitSemaphores = wait_semaphores.data();
+        }
+        else
+        {
+            submit_info.pWaitSemaphores = nullptr;
+        }
         submit_info.pWaitDstStageMask    = &wait_stage;
+        submit_info.signalSemaphoreCount = static_cast<uint32_t>(signal_semaphores.size());
+        submit_info.pSignalSemaphores    = signal_semaphores.data();
         submit_info.commandBufferCount   = 1;
         submit_info.pCommandBuffers      = &command_buffer;
-        submit_info.signalSemaphoreCount = 1;
-        submit_info.pSignalSemaphores    = &semaphore;
 
-        result = device_table_->QueueSubmit(swapchain_info->blit_queue, 1, &submit_info, VK_NULL_HANDLE);
+        result = device_table_->QueueSubmit(queue, 1, &submit_info, VK_NULL_HANDLE);
+
         if (result != VK_SUCCESS)
         {
             return result;
@@ -604,8 +856,8 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(PFN_vkQueuePresentKHR
     }
 
     VkPresentInfoKHR modified_present_info   = *present_info;
-    modified_present_info.waitSemaphoreCount = static_cast<uint32_t>(semaphores.size());
-    modified_present_info.pWaitSemaphores    = semaphores.data();
+    modified_present_info.waitSemaphoreCount = static_cast<uint32_t>(present_wait_semaphores.size());
+    modified_present_info.pWaitSemaphores    = present_wait_semaphores.data();
     return func(queue, &modified_present_info);
 }
 
@@ -672,9 +924,9 @@ void VulkanVirtualSwapchain::CmdPipelineBarrier(PFN_vkCmdPipelineBarrier     fun
          image_memory_barriers);
 }
 
-VkResult VulkanVirtualSwapchain::CreateSwapchainImage(const DeviceInfo*               device_info,
-                                                      const VkImageCreateInfo&        image_create_info,
-                                                      SwapchainKHRInfo::VirtualImage& image)
+VkResult VulkanVirtualSwapchain::CreateVirtualSwapchainImage(const DeviceInfo*        device_info,
+                                                             const VkImageCreateInfo& image_create_info,
+                                                             VirtualImage&            image)
 {
     // TODO: This is the same code used in VulkanReplayConsumerBase::CreateSwapchainImage, which
     // should be moved to a shared graphics utility function.
@@ -732,70 +984,15 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainImage(const DeviceInfo*
             if (image.memory != VK_NULL_HANDLE)
             {
                 allocator->FreeMemoryDirect(image.memory, nullptr, image.memory_allocator_data);
+                image.memory = VK_NULL_HANDLE;
             }
 
             allocator->DestroyImageDirect(image.image, nullptr, image.resource_allocator_data);
+            image.image = VK_NULL_HANDLE;
         }
     }
     return result;
 }
 
-int32_t VulkanVirtualSwapchain::FindFirstPresentSrcLayout(const VkRenderPassCreateInfo* create_info) const
-{
-    if ((create_info != nullptr) && (create_info->pAttachments != nullptr))
-    {
-        uint32_t count        = create_info->attachmentCount;
-        auto     descriptions = create_info->pAttachments;
-
-        for (uint32_t i = 0; i < count; ++i)
-        {
-            // TODO: This should also look at the initialLayout values.
-            if (descriptions[i].finalLayout == VK_IMAGE_LAYOUT_PRESENT_SRC_KHR)
-            {
-                return i;
-            }
-        }
-    }
-
-    return -1;
-}
-
-int32_t VulkanVirtualSwapchain::FindFirstPresentSrcLayout(const VkRenderPassCreateInfo2* create_info) const
-{
-    if ((create_info != nullptr) && (create_info->pAttachments != nullptr))
-    {
-        uint32_t count        = create_info->attachmentCount;
-        auto     descriptions = create_info->pAttachments;
-
-        for (uint32_t i = 0; i < count; ++i)
-        {
-            // TODO: This should also look at the initialLayout values.
-            if (descriptions[i].finalLayout == VK_IMAGE_LAYOUT_PRESENT_SRC_KHR)
-            {
-                return i;
-            }
-        }
-    }
-
-    return -1;
-}
-
-int32_t VulkanVirtualSwapchain::FindFirstPresentSrcLayout(uint32_t count, const VkImageMemoryBarrier* barriers) const
-{
-    if (barriers != nullptr)
-    {
-        for (uint32_t i = 0; i < count; ++i)
-        {
-            // TODO: This should also look at the oldLayout values.
-            if (barriers[i].newLayout == VK_IMAGE_LAYOUT_PRESENT_SRC_KHR)
-            {
-                return i;
-            }
-        }
-    }
-
-    return -1;
-}
-
 GFXRECON_END_NAMESPACE(decode)
 GFXRECON_END_NAMESPACE(gfxrecon)
diff --git a/framework/decode/vulkan_virtual_swapchain.h b/framework/decode/vulkan_virtual_swapchain.h
index e939bee3d5..baaa6707e0 100644
--- a/framework/decode/vulkan_virtual_swapchain.h
+++ b/framework/decode/vulkan_virtual_swapchain.h
@@ -118,15 +118,43 @@ class VulkanVirtualSwapchain : public VulkanSwapchain
     {}
 
   private:
-    VkResult CreateSwapchainImage(const DeviceInfo*               device_info,
-                                  const VkImageCreateInfo&        image_create_info,
-                                  SwapchainKHRInfo::VirtualImage& image);
-
-    int32_t FindFirstPresentSrcLayout(const VkRenderPassCreateInfo* create_info) const;
-
-    int32_t FindFirstPresentSrcLayout(const VkRenderPassCreateInfo2* create_info) const;
-
-    int32_t FindFirstPresentSrcLayout(uint32_t count, const VkImageMemoryBarrier* barriers) const;
+    // Structure necessary to track the necessary information related to the virtual swapchain images
+    struct VirtualImage
+    {
+        VkDeviceMemory                        memory{ VK_NULL_HANDLE };
+        VkImage                               image{ VK_NULL_HANDLE };
+        VulkanResourceAllocator::MemoryData   memory_allocator_data{ 0 };
+        VulkanResourceAllocator::ResourceData resource_allocator_data{ 0 };
+    };
+
+    // Structure to store data required per device queue family to properly handle creating the
+    // virtual swapchain images and copying that data to the actual swapchain images.
+    struct CopyCmdData
+    {
+        VkCommandPool                command_pool;
+        std::vector<VkCommandBuffer> command_buffers;
+        std::vector<VkSemaphore>     semaphores;
+    };
+
+    // Structure to track VulkanVirtualSwapchain private data specific to a particular VkSwapchainKHR handle.
+    struct SwapchainResourceData
+    {
+        // Create a map that correlates copy command data with a queue family index.
+        std::unordered_map<uint32_t, CopyCmdData> copy_cmd_data;
+
+        // Swapchain images, these include the virtual ones created by this
+        // class that are returned in place of the hardware ones, as well
+        // as a vector of the actual hardware ones used during replay.
+        std::vector<VirtualImage> virtual_swapchain_images;
+        std::vector<VkImage>      replay_swapchain_images;
+    };
+
+    VkResult CreateVirtualSwapchainImage(const DeviceInfo*        device_info,
+                                         const VkImageCreateInfo& image_create_info,
+                                         VirtualImage&            image);
+
+    // Create an unordered map to associate the swapchain resource data with a particular Vulkan swapchain
+    std::unordered_map<VkSwapchainKHR, std::unique_ptr<SwapchainResourceData>> swapchain_resources_;
 };
 
 GFXRECON_END_NAMESPACE(decode)
diff --git a/framework/generated/generated_vulkan_replay_consumer.cpp b/framework/generated/generated_vulkan_replay_consumer.cpp
index f6ceab3470..3ed2b2452d 100644
--- a/framework/generated/generated_vulkan_replay_consumer.cpp
+++ b/framework/generated/generated_vulkan_replay_consumer.cpp
@@ -206,13 +206,14 @@ void VulkanReplayConsumer::Process_vkGetDeviceQueue(
     uint32_t                                    queueIndex,
     HandlePointerDecoder<VkQueue>*              pQueue)
 {
-    VkDevice in_device = MapHandle<DeviceInfo>(device, &VulkanObjectInfoTable::GetDeviceInfo);
+    auto in_device = GetObjectInfoTable().GetDeviceInfo(device);
     if (!pQueue->IsNull()) { pQueue->SetHandleLength(1); }
-    VkQueue* out_pQueue = pQueue->GetHandlePointer();
+    QueueInfo handle_info;
+    pQueue->SetConsumerData(0, &handle_info);
 
-    GetDeviceTable(in_device)->GetDeviceQueue(in_device, queueFamilyIndex, queueIndex, out_pQueue);
+    OverrideGetDeviceQueue(GetDeviceTable(in_device->handle)->GetDeviceQueue, in_device, queueFamilyIndex, queueIndex, pQueue);
 
-    AddHandle<QueueInfo>(device, pQueue->GetPointer(), out_pQueue, &VulkanObjectInfoTable::AddQueueInfo);
+    AddHandle<QueueInfo>(device, pQueue->GetPointer(), pQueue->GetHandlePointer(), std::move(handle_info), &VulkanObjectInfoTable::AddQueueInfo);
 }
 
 void VulkanReplayConsumer::Process_vkQueueSubmit(
@@ -2308,14 +2309,14 @@ void VulkanReplayConsumer::Process_vkGetDeviceQueue2(
     StructPointerDecoder<Decoded_VkDeviceQueueInfo2>* pQueueInfo,
     HandlePointerDecoder<VkQueue>*              pQueue)
 {
-    VkDevice in_device = MapHandle<DeviceInfo>(device, &VulkanObjectInfoTable::GetDeviceInfo);
-    const VkDeviceQueueInfo2* in_pQueueInfo = pQueueInfo->GetPointer();
+    auto in_device = GetObjectInfoTable().GetDeviceInfo(device);
     if (!pQueue->IsNull()) { pQueue->SetHandleLength(1); }
-    VkQueue* out_pQueue = pQueue->GetHandlePointer();
+    QueueInfo handle_info;
+    pQueue->SetConsumerData(0, &handle_info);
 
-    GetDeviceTable(in_device)->GetDeviceQueue2(in_device, in_pQueueInfo, out_pQueue);
+    OverrideGetDeviceQueue2(GetDeviceTable(in_device->handle)->GetDeviceQueue2, in_device, pQueueInfo, pQueue);
 
-    AddHandle<QueueInfo>(device, pQueue->GetPointer(), out_pQueue, &VulkanObjectInfoTable::AddQueueInfo);
+    AddHandle<QueueInfo>(device, pQueue->GetPointer(), pQueue->GetHandlePointer(), std::move(handle_info), &VulkanObjectInfoTable::AddQueueInfo);
 }
 
 void VulkanReplayConsumer::Process_vkCreateSamplerYcbcrConversion(
diff --git a/framework/generated/vulkan_generators/replay_overrides.json b/framework/generated/vulkan_generators/replay_overrides.json
index 811203ead3..dc62dc1ed4 100644
--- a/framework/generated/vulkan_generators/replay_overrides.json
+++ b/framework/generated/vulkan_generators/replay_overrides.json
@@ -14,6 +14,8 @@
     "vkGetPhysicalDeviceMemoryProperties2KHR": "OverrideGetPhysicalDeviceMemoryProperties2",
     "vkGetPhysicalDeviceSurfaceCapabilitiesKHR": "OverrideGetPhysicalDeviceSurfaceCapabilitiesKHR",
     "vkGetPhysicalDeviceSurfaceCapabilities2KHR": "OverrideGetPhysicalDeviceSurfaceCapabilities2KHR",
+    "vkGetDeviceQueue": "OverrideGetDeviceQueue",
+    "vkGetDeviceQueue2": "OverrideGetDeviceQueue2",
     "vkWaitForFences": "OverrideWaitForFences",
     "vkGetFenceStatus": "OverrideGetFenceStatus",
     "vkGetEventStatus": "OverrideGetEventStatus",

From 579ef96d75bea56e86da5fde79a1757d07463730 Mon Sep 17 00:00:00 2001
From: Igor Brusentsev <igor.brusentsev@amd.com>
Date: Fri, 25 Aug 2023 15:26:42 -0400
Subject: [PATCH 4/5] copy gfxrecon_interceptor.dll to bin

---
 tools/launcher/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/launcher/CMakeLists.txt b/tools/launcher/CMakeLists.txt
index d8f6e1c773..bfb1b40f2b 100644
--- a/tools/launcher/CMakeLists.txt
+++ b/tools/launcher/CMakeLists.txt
@@ -42,4 +42,5 @@ if(BUILD_LAUNCHER_AND_INTERCEPTOR)
     common_build_directives(gfxrecon-launcher)
 
     install(TARGETS gfxrecon-launcher RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+    install(FILES ${CMAKE_BINARY_DIR}/layer/gfxrecon_interceptor/$<CONFIG>/gfxrecon_interceptor.dll DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif()

From 31b6c7cca7681470a5af5acae7f3aff416aca3fe Mon Sep 17 00:00:00 2001
From: bradgrantham-lunarg
 <50641407+bradgrantham-lunarg@users.noreply.github.com>
Date: Wed, 30 Aug 2023 07:53:31 -0700
Subject: [PATCH 5/5] cast a float result back to uint32_t (#1241)

* cast a float result back to uint32_t

* fix some strict warnings
---
 framework/decode/vulkan_replay_consumer_base.cpp | 8 ++++----
 tools/tool_settings.h                            | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp
index 612247866d..d9b28717d3 100755
--- a/framework/decode/vulkan_replay_consumer_base.cpp
+++ b/framework/decode/vulkan_replay_consumer_base.cpp
@@ -2147,12 +2147,12 @@ void VulkanReplayConsumerBase::WriteScreenshots(const Decoded_VkPresentInfoKHR*
                 // If both copy_scale and copy_width are provided, use copy_scale.
                 const uint32_t screenshot_width =
                     options_.screenshot_scale
-                        ? (options_.screenshot_scale * swapchain_info->width)
+                        ? static_cast<uint32_t>(options_.screenshot_scale * swapchain_info->width)
                         : (options_.screenshot_width ? options_.screenshot_width : swapchain_info->width);
 
                 const uint32_t screenshot_height =
                     options_.screenshot_scale
-                        ? (options_.screenshot_scale * swapchain_info->height)
+                        ? static_cast<uint32_t>(options_.screenshot_scale * swapchain_info->height)
                         : (options_.screenshot_height ? options_.screenshot_height : swapchain_info->height);
 
                 screenshot_handler_->WriteImage(filename_prefix,
@@ -2224,12 +2224,12 @@ bool VulkanReplayConsumerBase::CheckCommandBufferInfoForFrameBoundary(const Comm
                     // If both copy_scale and copy_width are provided, use copy_scale.
                     const uint32_t screenshot_width =
                         options_.screenshot_scale
-                            ? (options_.screenshot_scale * image_info->extent.width)
+                            ? static_cast<uint32_t>(options_.screenshot_scale * image_info->extent.width)
                             : (options_.screenshot_width ? options_.screenshot_width : image_info->extent.width);
 
                     const uint32_t screenshot_height =
                         options_.screenshot_scale
-                            ? (options_.screenshot_scale * image_info->extent.height)
+                            ? static_cast<uint32_t>(options_.screenshot_scale * image_info->extent.height)
                             : (options_.screenshot_height ? options_.screenshot_height : image_info->extent.height);
 
                     screenshot_handler_->WriteImage(filename_prefix,
diff --git a/tools/tool_settings.h b/tools/tool_settings.h
index bcac446c0c..9e59473232 100644
--- a/tools/tool_settings.h
+++ b/tools/tool_settings.h
@@ -480,7 +480,7 @@ static void GetScreenshotSize(const gfxrecon::util::ArgumentParser& arg_parser,
                 width  = std::stoul(value.substr(0, x));
                 height = std::stoul(value.substr(x + 1));
             }
-            catch (std::exception& e)
+            catch (std::exception&)
             {
                 GFXRECON_LOG_WARNING("Ignoring invalid screenshot width x height option. Expected format is "
                                      "--screenshot-size [width]x[height]");
@@ -510,7 +510,7 @@ static float GetScreenshotScale(const gfxrecon::util::ArgumentParser& arg_parser
         {
             scale = std::stof(value);
         }
-        catch (std::exception& e)
+        catch (std::exception&)
         {
             GFXRECON_LOG_WARNING(
                 "Ignoring invalid screenshot scale option. Expected format is --screenshot-scale [scale]");