Fix undersized buffer for Vulkan pixel history with many fragments

renderdoc/driver/vulkan/vk_debug.h

@@ -97,6 +97,8 @@ class VulkanDebugManager
   bool PixelHistorySetupResources(PixelHistoryResources &resources, VkImage targetImage,
                                   VkExtent3D extent, VkFormat format, VkSampleCountFlagBits samples,
                                   const Subresource &sub, uint32_t numEvents);
+  bool PixelHistorySetupPerFragResources(PixelHistoryResources &resources, uint32_t numEvents,
+                                         uint32_t numFragments);
   bool PixelHistoryDestroyResources(const PixelHistoryResources &resources);
 
   void PixelHistoryCopyPixel(VkCommandBuffer cmd, VkCopyPixelParams &p, size_t offset);

renderdoc/driver/vulkan/vk_pixelhistory.cpp

@@ -3389,9 +3389,9 @@ bool VulkanDebugManager::PixelHistorySetupResources(PixelHistoryResources &resou
                                                     VkFormat format, VkSampleCountFlagBits samples,
                                                     const Subresource &sub, uint32_t numEvents)
 {
-  VkMarkerRegion region(StringFormat::Fmt("PixelHistorySetupResources %ux%ux%u %s %ux MSAA",
-                                          extent.width, extent.height, extent.depth,
-                                          ToStr(format).c_str(), samples));
+  VkMarkerRegion region(
+      StringFormat::Fmt("PixelHistorySetupResources %ux%ux%u %s %ux MSAA, %u events", extent.width,
+                        extent.height, extent.depth, ToStr(format).c_str(), samples, numEvents));
   VulkanCreationInfo::Image targetImageInfo = GetImageInfo(GetResID(targetImage));
 
   VkImage colorImage;
@@ -3501,8 +3501,6 @@ bool VulkanDebugManager::PixelHistorySetupResources(PixelHistoryResources &resou
   CheckVkResult(vkr);
 
   VkBufferCreateInfo bufferInfo = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
-  // TODO: the size for memory is calculated to fit pre and post modification values and
-  // stencil values. But we might run out of space when getting per fragment data.
   bufferInfo.size = AlignUp((uint32_t)(numEvents * sizeof(EventInfo)), 4096U);
   bufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
 
@@ -3558,6 +3556,75 @@ bool VulkanDebugManager::PixelHistorySetupResources(PixelHistoryResources &resou
   return true;
 }
 
+bool VulkanDebugManager::PixelHistorySetupPerFragResources(PixelHistoryResources &resources,
+                                                           uint32_t numEvents, uint32_t numFrags)
+{
+  const uint32_t existingBufferSize = AlignUp((uint32_t)(numEvents * sizeof(EventInfo)), 4096U);
+  const uint32_t requiredBufferSize = AlignUp((uint32_t)(numFrags * sizeof(PerFragmentInfo)), 4096U);
+
+  // If the existing buffer is big enough for all of the fragments, we can re-use it.
+  const bool canReuseBuffer = existingBufferSize >= requiredBufferSize;
+
+  VkMarkerRegion region(StringFormat::Fmt(
+      "PixelHistorySetupPerFragResources %u events %u frags, buffer size %u -> %u, %s old buffer",
+      numEvents, numFrags, existingBufferSize, requiredBufferSize,
+      canReuseBuffer ? "reusing" : "NOT reusing"));
+
+  if(canReuseBuffer)
+    return true;
+
+  // Otherwise, destroy it and create a new one that's big enough in its place.
+  VkDevice dev = m_pDriver->GetDev();
+
+  if(resources.dstBuffer != VK_NULL_HANDLE)
+    m_pDriver->vkDestroyBuffer(dev, resources.dstBuffer, NULL);
+  if(resources.bufferMemory != VK_NULL_HANDLE)
+    m_pDriver->vkFreeMemory(dev, resources.bufferMemory, NULL);
+  resources.dstBuffer = VK_NULL_HANDLE;
+  resources.bufferMemory = VK_NULL_HANDLE;
+
+  VkBufferCreateInfo bufferInfo = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
+  bufferInfo.size = requiredBufferSize;
+  bufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
+
+  VkResult vkr = m_pDriver->vkCreateBuffer(m_Device, &bufferInfo, NULL, &resources.dstBuffer);
+  CheckVkResult(vkr);
+
+  // Allocate memory
+  VkMemoryRequirements mrq = {};
+  m_pDriver->vkGetBufferMemoryRequirements(m_Device, resources.dstBuffer, &mrq);
+  VkMemoryAllocateInfo allocInfo = {
+      VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, NULL, mrq.size,
+      m_pDriver->GetReadbackMemoryIndex(mrq.memoryTypeBits),
+  };
+  vkr = m_pDriver->vkAllocateMemory(m_Device, &allocInfo, NULL, &resources.bufferMemory);
+  CheckVkResult(vkr);
+
+  if(vkr != VK_SUCCESS)
+    return false;
+
+  vkr = m_pDriver->vkBindBufferMemory(m_Device, resources.dstBuffer, resources.bufferMemory, 0);
+  CheckVkResult(vkr);
+
+  VkCommandBuffer cmd = m_pDriver->GetNextCmd();
+  VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL,
+                                        VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
+
+  if(cmd == VK_NULL_HANDLE)
+    return false;
+
+  vkr = ObjDisp(dev)->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
+  CheckVkResult(vkr);
+  ObjDisp(cmd)->CmdFillBuffer(Unwrap(cmd), Unwrap(resources.dstBuffer), 0, VK_WHOLE_SIZE, 0);
+
+  vkr = ObjDisp(dev)->EndCommandBuffer(Unwrap(cmd));
+  CheckVkResult(vkr);
+  m_pDriver->SubmitCmds();
+  m_pDriver->FlushQ();
+
+  return true;
+}
+
 VkDescriptorSet VulkanReplay::GetPixelHistoryDescriptor()
 {
   VkDescriptorSet descSet;
@@ -4057,6 +4124,17 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
 
   if(eventsWithFrags.size() > 0)
   {
+    uint32_t numFrags = 0;
+    for(auto &item : eventsWithFrags)
+    {
+      numFrags += item.second;
+    }
+
+    GetDebugManager()->PixelHistorySetupPerFragResources(resources, (uint32_t)events.size(),
+                                                         numFrags);
+
+    callbackInfo.dstBuffer = resources.dstBuffer;
+
     // Replay to get shader output value, post modification value and primitive ID for every
     // fragment.
     VulkanPixelHistoryPerFragmentCallback perFragmentCB(m_pDriver, shaderCache, callbackInfo,
@@ -4222,6 +4300,8 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
           history[h].depthBoundsFailed = true;
       }
     }
+
+    m_pDriver->vkUnmapMemory(dev, resources.bufferMemory);
   }
 
   SAFE_DELETE(tfCb);

util/test/demos/CMakeLists.txt

@@ -87,6 +87,7 @@ set(VULKAN_SRC
         vk/vk_test.cpp
         vk/vk_test.h
         vk/vk_adv_cbuffer_zoo.cpp
+        vk/vk_blend.cpp
         vk/vk_buffer_truncation.cpp
         vk/vk_cbuffer_zoo.cpp
         vk/vk_compute_only.cpp

util/test/demos/demos.vcxproj

@@ -296,6 +296,7 @@
     <ClCompile Include="main.cpp" />
     <ClCompile Include="test_common.cpp" />
     <ClCompile Include="texture_zoo.cpp" />
+    <ClCompile Include="vk\vk_blend.cpp" />
     <ClCompile Include="vk\vk_compute_only.cpp" />
     <ClCompile Include="vk\vk_custom_border_color.cpp" />
     <ClCompile Include="vk\vk_dedicated_allocation.cpp" />

util/test/demos/demos.vcxproj.filters

@@ -673,6 +673,9 @@
     <ClCompile Include="d3d12\d3d12_template.cpp">
       <Filter>D3D12\demos</Filter>
     </ClCompile>
+    <ClCompile Include="vk\vk_blend.cpp">
+      <Filter>Vulkan\demos</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Filter Include="D3D11">

util/test/demos/vk/vk_blend.cpp

@@ -0,0 +1,223 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019-2023 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "vk_test.h"
+
+RD_TEST(VK_Blend, VulkanGraphicsTest)
+{
+  static constexpr const char *Description =
+      "Draws a triangle repeatedly to test blending within a single drawcall";
+
+  const DefaultA2V TemplateTriangleRed[3] = {
+      {Vec3f(-0.5f, -0.5f, 0.0f), Vec4f(1 / 255.f, 0.0f, 0.0f, 1.0f), Vec2f(0.0f, 0.0f)},
+      {Vec3f(0.0f, 0.5f, 0.0f), Vec4f(1 / 255.f, 0.0f, 0.0f, 1.0f), Vec2f(0.0f, 1.0f)},
+      {Vec3f(0.5f, -0.5f, 0.0f), Vec4f(1 / 255.f, 0.0f, 0.0f, 1.0f), Vec2f(1.0f, 0.0f)},
+  };
+  const int TRIANGLES_RED_INDEX = 0;
+  const int NUM_TRIANGLES_RED = 16;
+  const DefaultA2V TemplateTriangleGreen[3] = {
+      {Vec3f(-0.5f, -0.5f, 0.0f), Vec4f(0.0f, 1 / 255.f, 0.0f, 1.0f), Vec2f(0.0f, 0.0f)},
+      {Vec3f(0.0f, 0.5f, 0.0f), Vec4f(0.0f, 1 / 255.f, 0.0f, 1.0f), Vec2f(0.0f, 1.0f)},
+      {Vec3f(0.5f, -0.5f, 0.0f), Vec4f(0.0f, 1 / 255.f, 0.0f, 1.0f), Vec2f(1.0f, 0.0f)},
+  };
+  const int TRIANGLES_GREEN_INDEX = TRIANGLES_RED_INDEX + NUM_TRIANGLES_RED;
+  const int NUM_TRIANGLES_GREEN = 255;
+  const DefaultA2V TemplateTriangleBlue[3] = {
+      {Vec3f(-0.5f, -0.5f, 0.0f), Vec4f(0.0f, 0.0f, 1 / 255.f, 1.0f), Vec2f(0.0f, 0.0f)},
+      {Vec3f(0.0f, 0.5f, 0.0f), Vec4f(0.0f, 0.0f, 1 / 255.f, 1.0f), Vec2f(0.0f, 1.0f)},
+      {Vec3f(0.5f, -0.5f, 0.0f), Vec4f(0.0f, 0.0f, 1 / 255.f, 1.0f), Vec2f(1.0f, 0.0f)},
+  };
+  const int TRIANGLES_BLUE_INDEX = TRIANGLES_GREEN_INDEX + NUM_TRIANGLES_GREEN;
+  const int NUM_TRIANGLES_BLUE = 512;
+
+  const int NUM_TRIANGLES_TOTAL = TRIANGLES_BLUE_INDEX + NUM_TRIANGLES_BLUE;
+
+  int main()
+  {
+    // initialise, create window, create context, etc
+    if(!Init())
+      return 3;
+
+    VkPipelineLayout layout = createPipelineLayout(vkh::PipelineLayoutCreateInfo());
+
+    std::vector<DefaultA2V> triangles;
+    triangles.reserve(3 * NUM_TRIANGLES_TOTAL);
+    for(int i = 0; i < NUM_TRIANGLES_RED; i++)
+    {
+      triangles.push_back(TemplateTriangleRed[0]);
+      triangles.push_back(TemplateTriangleRed[1]);
+      triangles.push_back(TemplateTriangleRed[2]);
+    }
+    for(int i = 0; i < NUM_TRIANGLES_GREEN; i++)
+    {
+      triangles.push_back(TemplateTriangleGreen[0]);
+      triangles.push_back(TemplateTriangleGreen[1]);
+      triangles.push_back(TemplateTriangleGreen[2]);
+    }
+    for(int i = 0; i < NUM_TRIANGLES_BLUE; i++)
+    {
+      triangles.push_back(TemplateTriangleBlue[0]);
+      triangles.push_back(TemplateTriangleBlue[1]);
+      triangles.push_back(TemplateTriangleBlue[2]);
+    }
+
+    AllocatedBuffer vb(this, vkh::BufferCreateInfo(sizeof(DefaultA2V) * 3 * NUM_TRIANGLES_TOTAL,
+                                                   VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
+                                                       VK_BUFFER_USAGE_TRANSFER_DST_BIT),
+                       VmaAllocationCreateInfo({0, VMA_MEMORY_USAGE_CPU_TO_GPU}));
+
+    vb.upload(triangles.data(), sizeof(DefaultA2V) * 3 * NUM_TRIANGLES_TOTAL);
+
+    AllocatedImage img(
+        this,
+        vkh::ImageCreateInfo(mainWindow->scissor.extent.width, mainWindow->scissor.extent.height, 0,
+                             VK_FORMAT_R32G32B32A32_SFLOAT,
+                             VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+                                 VK_IMAGE_USAGE_TRANSFER_DST_BIT),
+        VmaAllocationCreateInfo({0, VMA_MEMORY_USAGE_GPU_ONLY}));
+
+    VkImageView imgview = createImageView(
+        vkh::ImageViewCreateInfo(img.image, VK_IMAGE_VIEW_TYPE_2D, VK_FORMAT_R32G32B32A32_SFLOAT));
+
+    vkh::RenderPassCreator renderPassCreateInfo;
+
+    renderPassCreateInfo.attachments.push_back(vkh::AttachmentDescription(
+        VK_FORMAT_R32G32B32A32_SFLOAT, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL));
+
+    renderPassCreateInfo.addSubpass({VkAttachmentReference({0, VK_IMAGE_LAYOUT_GENERAL})});
+
+    VkRenderPass renderPass = createRenderPass(renderPassCreateInfo);
+
+    VkFramebuffer framebuffer = createFramebuffer(
+        vkh::FramebufferCreateInfo(renderPass, {imgview}, mainWindow->scissor.extent));
+
+    vkh::GraphicsPipelineCreateInfo pipeCreateInfo;
+
+    pipeCreateInfo.layout = layout;
+    pipeCreateInfo.renderPass = renderPass;
+
+    VkPipelineColorBlendAttachmentState colorBlendAttachment = {};
+    colorBlendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
+                                          VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
+    colorBlendAttachment.blendEnable = VK_TRUE;
+    colorBlendAttachment.srcColorBlendFactor = VK_BLEND_FACTOR_ONE;
+    colorBlendAttachment.dstColorBlendFactor = VK_BLEND_FACTOR_ONE;
+    colorBlendAttachment.colorBlendOp = VK_BLEND_OP_ADD;
+    colorBlendAttachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
+    colorBlendAttachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO;
+    colorBlendAttachment.alphaBlendOp = VK_BLEND_OP_ADD;
+    pipeCreateInfo.colorBlendState.attachments = {colorBlendAttachment};
+
+    pipeCreateInfo.vertexInputState.vertexBindingDescriptions = {vkh::vertexBind(0, DefaultA2V)};
+    pipeCreateInfo.vertexInputState.vertexAttributeDescriptions = {
+        vkh::vertexAttr(0, 0, DefaultA2V, pos), vkh::vertexAttr(1, 0, DefaultA2V, col),
+        vkh::vertexAttr(2, 0, DefaultA2V, uv),
+    };
+
+    pipeCreateInfo.stages = {
+        CompileShaderModule(VKDefaultVertex, ShaderLang::glsl, ShaderStage::vert, "main"),
+        CompileShaderModule(VKDefaultPixel, ShaderLang::glsl, ShaderStage::frag, "main"),
+    };
+
+    VkPipeline pipe = createGraphicsPipeline(pipeCreateInfo);
+
+    while(Running())
+    {
+      VkCommandBuffer cmd = GetCommandBuffer();
+
+      vkBeginCommandBuffer(cmd, vkh::CommandBufferBeginInfo());
+
+      VkImage swapimg = StartUsingBackbuffer(cmd, VK_ACCESS_TRANSFER_WRITE_BIT,
+                                             VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+
+      vkh::cmdPipelineBarrier(cmd, {
+                                       vkh::ImageMemoryBarrier(0, VK_ACCESS_TRANSFER_WRITE_BIT,
+                                                               VK_IMAGE_LAYOUT_UNDEFINED,
+                                                               VK_IMAGE_LAYOUT_GENERAL, img.image),
+                                   });
+
+      pushMarker(cmd, "Clear");
+      vkCmdClearColorImage(cmd, img.image, VK_IMAGE_LAYOUT_GENERAL,
+                           vkh::ClearColorValue(0.0f, 0.0f, 0.0f, 1.0f), 1,
+                           vkh::ImageSubresourceRange());
+
+      popMarker(cmd);
+
+      vkCmdBeginRenderPass(cmd,
+                           vkh::RenderPassBeginInfo(renderPass, framebuffer, mainWindow->scissor),
+                           VK_SUBPASS_CONTENTS_INLINE);
+
+      vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
+      vkCmdSetViewport(cmd, 0, 1, &mainWindow->viewport);
+      vkCmdSetScissor(cmd, 0, 1, &mainWindow->scissor);
+      vkh::cmdBindVertexBuffers(cmd, 0, {vb.buffer}, {0});
+      pushMarker(cmd, "Red: groups of repeated draws");
+      for(int i = 1; i <= NUM_TRIANGLES_RED; i *= 2)
+      {
+        vkCmdDraw(cmd, 3 * i, 1, TRIANGLES_RED_INDEX, 0);
+      }
+      setMarker(cmd, "End of red");
+      popMarker(cmd);
+      pushMarker(cmd, "Green: 255 (the maximum we can handle) in a single drawcall");
+      vkCmdDraw(cmd, 3 * NUM_TRIANGLES_GREEN, 1, 3 * TRIANGLES_GREEN_INDEX, 0);
+      popMarker(cmd);
+      pushMarker(cmd, "Blue: 512 (more than the maximum) in a single drawcall");
+      vkCmdDraw(cmd, 3 * NUM_TRIANGLES_BLUE, 1, 3 * TRIANGLES_BLUE_INDEX, 0);
+      popMarker(cmd);
+
+      vkCmdEndRenderPass(cmd);
+
+      pushMarker(cmd, "Clear");
+      vkCmdClearColorImage(cmd, img.image, VK_IMAGE_LAYOUT_GENERAL,
+                           vkh::ClearColorValue(0.0f, 0.0f, 0.0f, 1.0f), 1,
+                           vkh::ImageSubresourceRange());
+      popMarker(cmd);
+
+      vkCmdBeginRenderPass(cmd,
+                           vkh::RenderPassBeginInfo(renderPass, framebuffer, mainWindow->scissor),
+                           VK_SUBPASS_CONTENTS_INLINE);
+
+      pushMarker(cmd, "All of the above in a single drawcall");
+      vkCmdDraw(cmd, 3 * NUM_TRIANGLES_TOTAL, 1, 3 * TRIANGLES_RED_INDEX, 0);
+      popMarker(cmd);
+
+      setMarker(cmd, "Test End");
+
+      vkCmdEndRenderPass(cmd);
+
+      blitToSwap(cmd, img.image, VK_IMAGE_LAYOUT_GENERAL, swapimg,
+                 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+
+      FinishUsingBackbuffer(cmd, VK_ACCESS_TRANSFER_WRITE_BIT, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+
+      vkEndCommandBuffer(cmd);
+
+      SubmitAndPresent({cmd});
+    }
+
+    return 0;
+  }
+};
+
+REGISTER_TEST();