VK_KHR_timeline_semaphore support.

This adds VK_KHR_timeline_semaphore support to GAPID. This handles the simple cases (always being able to make forward progress) as well as the more complex cases (blocking on host or other commands).
google · Mar 9, 2020 · 11c8ba8 · 11c8ba8
1 parent 4c61a93
commit 11c8ba8
Show file tree

Hide file tree

Showing 26 changed files with 1,110 additions and 243 deletions.
diff --git a/gapii/cc/vulkan_extras.cpp b/gapii/cc/vulkan_extras.cpp
@@ -469,6 +469,9 @@ void VulkanSpy::recordEventSet(CallObserver*, VkEvent) {}
 void VulkanSpy::recordFenceSignal(CallObserver*, uint64_t) {}
 void VulkanSpy::recordFenceWait(CallObserver*, uint64_t) {}
 void VulkanSpy::recordFenceReset(CallObserver*, uint64_t) {}
+void VulkanSpy::recordSemaphoreSignal(CallObserver*, uint64_t, uint64_t) {}
+void VulkanSpy::recordSemaphoreWait(CallObserver*, uint64_t, uint64_t) {}
+
 void VulkanSpy::recordAcquireNextImage(CallObserver*, uint64_t, uint32_t) {}
 void VulkanSpy::recordPresentSwapchainImage(CallObserver*, uint64_t, uint32_t) {
 }
@@ -493,8 +496,11 @@ void VulkanSpy::resetCmd(CallObserver* observer, VkCommandBuffer cmdBuf) {}
 void VulkanSpy::enterSubcontext(CallObserver*) {}
 void VulkanSpy::leaveSubcontext(CallObserver*) {}
 void VulkanSpy::nextSubcontext(CallObserver*) {}
-void VulkanSpy::resetSubcontext(CallObserver*) {}
 void VulkanSpy::onPreSubcommand(CallObserver*, gapil::Ref<CommandReference>) {}
+void VulkanSpy::deferSubmit(CallObserver*, gapil::Ref<Submission>) {}
+void VulkanSpy::executeDeferredSubmit(CallObserver*, gapil::Ref<Submission>) {}
+void VulkanSpy::finishedExecuteDeferredSubmit(CallObserver*,
+                                              gapil::Ref<Submission>) {}
 
 void VulkanSpy::onPostSubcommand(CallObserver*, gapil::Ref<CommandReference>) {}
 void VulkanSpy::onCommandAdded(CallObserver*, VkCommandBuffer) {}
@@ -1102,4 +1108,27 @@ uint32_t VulkanSpy::SpyOverride_vkQueueSubmit(CallObserver* observer,
   return VkResult::VK_SUCCESS;
 }
 
+void VulkanSpy::recordWaitedSemaphores(CallObserver* observer, VkDevice device,
+                                       const VkSemaphoreWaitInfo* wait_info,
+                                       bool isKHR) {
+  auto it = mImports.mVkDeviceFunctions.find(device);
+
+  vulkan_pb::SemaphoreState state;
+
+  for (size_t i = 0; i < wait_info->msemaphoreCount; ++i) {
+    uint64_t value;
+    if (isKHR) {
+      it->second.vkGetSemaphoreCounterValueKHR(
+          device, wait_info->mpSemaphores[i], &value);
+    } else {
+      it->second.vkGetSemaphoreCounterValue(device, wait_info->mpSemaphores[i],
+                                            &value);
+    }
+    state.add_semaphores(wait_info->mpSemaphores[i]);
+    state.add_values(value);
+  }
+
+  observer->encode_message(&state);
+}
+
 }  // namespace gapii
diff --git a/gapir/cc/context.cpp b/gapir/cc/context.cpp
@@ -53,9 +53,7 @@ std::unique_ptr<Context> Context::create(ReplayService* srv,
 // TODO: Make the PostBuffer size dynamic? It currently holds 2MB of data.
 Context::Context(ReplayService* srv, core::CrashHandler& crash_handler,
                  ResourceLoader* resource_loader, MemoryManager* memory_manager)
-    :
-
-      mSrv(srv),
+    : mSrv(srv),
       mCrashHandler(crash_handler),
       mResourceLoader(resource_loader),
       mMemoryManager(memory_manager),
@@ -758,6 +756,72 @@ void Context::registerCallbacks(Interpreter* interpreter) {
           return false;
         }
       });
+
+  interpreter->registerBuiltin(
+      Vulkan::INDEX, Builtins::ReplayGetSemaphoreCounterValue,
+      [this](uint32_t label, Stack* stack, bool pushReturn) {
+        GAPID_DEBUG("[%u]replayGetSemaphoreCounterValue()", label);
+        if (mVulkanRenderer != nullptr) {
+          auto* api = mVulkanRenderer->getApi<Vulkan>();
+          return api->replayGetSemaphoreCounterValue(stack, pushReturn);
+        } else {
+          GAPID_WARNING(
+              "[%u]replayGetSemaphoreCounterValue called without a bound "
+              "Vulkan "
+              "renderer",
+              label);
+          return false;
+        }
+      });
+
+  interpreter->registerBuiltin(
+      Vulkan::INDEX, Builtins::ReplayGetSemaphoreCounterValueKHR,
+      [this](uint32_t label, Stack* stack, bool pushReturn) {
+        GAPID_DEBUG("[%u]replayGetSemaphoreCounterValueKHR()", label);
+        if (mVulkanRenderer != nullptr) {
+          auto* api = mVulkanRenderer->getApi<Vulkan>();
+          return api->replayGetSemaphoreCounterValueKHR(stack, pushReturn);
+        } else {
+          GAPID_WARNING(
+              "[%u]replayGetSemaphoreCounterValueKHR called without a bound "
+              "Vulkan "
+              "renderer",
+              label);
+          return false;
+        }
+      });
+
+  interpreter->registerBuiltin(
+      Vulkan::INDEX, Builtins::ReplayWaitSemaphores,
+      [this](uint32_t label, Stack* stack, bool pushReturn) {
+        GAPID_DEBUG("[%u]replayWaitSemaphores()", label);
+        if (mVulkanRenderer != nullptr) {
+          auto* api = mVulkanRenderer->getApi<Vulkan>();
+          return api->replayWaitSemaphores(stack, pushReturn);
+        } else {
+          GAPID_WARNING(
+              "[%u]replayWaitSemaphores called without a bound Vulkan "
+              "renderer",
+              label);
+          return false;
+        }
+      });
+
+  interpreter->registerBuiltin(
+      Vulkan::INDEX, Builtins::ReplayWaitSemaphoresKHR,
+      [this](uint32_t label, Stack* stack, bool pushReturn) {
+        GAPID_DEBUG("[%u]replayWaitSemaphoresKHR()", label);
+        if (mVulkanRenderer != nullptr) {
+          auto* api = mVulkanRenderer->getApi<Vulkan>();
+          return api->replayWaitSemaphoresKHR(stack, pushReturn);
+        } else {
+          GAPID_WARNING(
+              "[%u]replayWaitSemaphoresKHR called without a bound Vulkan "
+              "renderer",
+              label);
+          return false;
+        }
+      });
 }
 
 bool Context::loadResource(Stack* stack) {

diff --git a/gapir/cc/vulkan_gfx_api.inc b/gapir/cc/vulkan_gfx_api.inc
@@ -135,6 +135,25 @@ bool replayCreateVkDebugReportCallback(Stack* stack, bool pushReturn);
 // might not be enabled.
 bool replayDestroyVkDebugReportCallback(Stack* stack);
 
+// Builtin function for getting a semaphore counter value. This will make sure
+// to block until the semaphore reaches the original value
+bool replayGetSemaphoreCounterValue(Stack* stack, bool pushReturn);
+
+// Builtin function for getting a semaphore counter value. This will make sure
+// to block until the semaphore reaches the original value
+bool replayGetSemaphoreCounterValueKHR(Stack* stack, bool pushReturn);
+
+// This will wait for semaphores, and block until all semaphores are
+// at the appropriate value. This is needed because the timeout may cause
+// replay behavior to diverge from trace behavior
+bool replayWaitSemaphores(Stack* stack, bool pushReturn);
+
+// This will wait for semaphores, and block until all semaphores are
+// at the appropriate value. This is needed because the timeout may cause
+// replay behavior to diverge from trace behavior
+bool replayWaitSemaphoresKHR(Stack* stack, bool pushReturn);
+
+
 // The callback function for VkDebugReportCallbackEXT. It pipes the debug
 // report captured by validation layers to the Renderer::Listener passed in by
 // the pUserData.

diff --git a/gapis/api/sync/data.go b/gapis/api/sync/data.go
@@ -91,6 +91,11 @@ type Data struct {
 	SyncNodes        []SyncNode
 	CmdSyncNodes     map[api.CmdID]SyncNodeIdx
 	SubcommandLookup *api.SubCmdIdxTrie
+	// UnblockingCommands is a map of what command fully unblocks
+	// some other command. Or to put it another way, the command that answers
+	// the question: If we want to play up to command N, what command do we
+	// have to replay until, in order to prevent a deadlock.
+	UnblockingCommands map[api.CmdID]api.CmdID
 }
 
 type subCommandMarkerGroupTrie struct {
@@ -124,6 +129,7 @@ func NewData() *Data {
 		SyncNodes:              []SyncNode{},
 		CmdSyncNodes:           map[api.CmdID]SyncNodeIdx{},
 		SubcommandLookup:       new(api.SubCmdIdxTrie),
+		UnblockingCommands:     map[api.CmdID]api.CmdID{},
 	}
 }
 

diff --git a/gapis/api/templates/vulkan_gfx_api_extras.tmpl b/gapis/api/templates/vulkan_gfx_api_extras.tmpl
@@ -577,6 +577,125 @@ bool Vulkan::replayGetEventStatus(Stack* stack, bool pushReturn) {
     }
 }
 ¶
+bool Vulkan::replayGetSemaphoreCounterValue(Stack* stack, bool pushReturn) {
+   auto expected = stack->pop<uint64_t>();
+   auto pValue = stack->pop<uint64_t*>();
+   auto semaphore = stack->pop<uint64_t>();
+   auto device = static_cast<size_val>(stack->pop<size_val>());
+   if (stack->isValid()) {
+        GAPID_DEBUG("vkGetSemaphoreCountValue(%zu, %" PRIu64 ")", device, semaphore);
+        VkSemaphoreWaitInfo wait_info {
+          VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
+          nullptr, // pNext
+          0, // flags
+          1, // semaphoreCount
+          &semaphore, // pSemaphores
+          &expected // pValues
+        };
+        uint32_t return_value;
+        return_value = mVkDeviceFunctionStubs[device].vkWaitSemaphores(device, &wait_info, uint64_t(-1));
+        *pValue = expected;
+        if (pushReturn) {
+            stack->push(return_value);
+        }
+        return true;
+   }
+   else {
+        GAPID_WARNING("Error during calling function vkGetSemaphoreCountValue");
+        return false;
+   }
+}
+¶
+bool Vulkan::replayGetSemaphoreCounterValueKHR(Stack* stack, bool pushReturn) {
+   auto expected = stack->pop<uint64_t>();
+   auto pValue = stack->pop<uint64_t*>();
+   auto semaphore = stack->pop<uint64_t>();
+   auto device = static_cast<size_val>(stack->pop<size_val>());
+   if (stack->isValid()) {
+        GAPID_DEBUG("vkGetSemaphoreCountValue(%zu, %" PRIu64 ")", device, semaphore);
+        VkSemaphoreWaitInfoKHR wait_info {
+          VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO_KHR,
+          nullptr, // pNext
+          0, // flags
+          1, // semaphoreCount
+          &semaphore, // pSemaphores
+          &expected // pValues
+        };
+        uint32_t return_value;
+        return_value = mVkDeviceFunctionStubs[device].vkWaitSemaphoresKHR(device, &wait_info, uint64_t(-1));
+        if (pushReturn) {
+            stack->push(return_value);
+        }
+        *pValue = expected;
+        return true;
+   }
+   else {
+        GAPID_WARNING("Error during calling function vkGetEventStatus");
+        return false;
+   }
+}
+¶
+bool Vulkan::replayWaitSemaphores(Stack* stack, bool pushReturn) {
+   auto expecteds = stack->pop<uint64_t*>();
+   auto semaphores = stack->pop<uint64_t*>();
+   auto count = stack->pop<uint64_t>();
+   auto device = static_cast<size_val>(stack->pop<size_val>());
+   if (stack->isValid()) {
+        GAPID_DEBUG("vkWaitSemaphores(%zu)", device);
+
+        uint32_t return_value;
+        for (uint64_t i = 0; i < count; ++i) {
+          auto expected = expecteds[i];
+          uint64_t val = 0;
+          do {
+            return_value = mVkDeviceFunctionStubs[device].vkGetSemaphoreCounterValue(device, semaphores[i], &val);
+          } while (val < expected && (return_value == VK_SUCCESS));
+          if (return_value != VK_SUCCESS) {
+            break;
+          }
+        }
+
+        if (pushReturn) {
+            stack->push(return_value);
+        }
+        return true;
+   }
+   else {
+        GAPID_WARNING("Error during calling function vkGetEventStatus");
+        return false;
+   }
+}
+¶
+bool Vulkan::replayWaitSemaphoresKHR(Stack* stack, bool pushReturn) {
+   auto expecteds = stack->pop<uint64_t*>();
+   auto semaphores = stack->pop<uint64_t*>();
+   auto count = stack->pop<uint64_t>();
+   auto device = static_cast<size_val>(stack->pop<size_val>());
+   if (stack->isValid()) {
+        GAPID_DEBUG("vkWaitSemaphoresKHR(%zu)", device);
+        uint32_t return_value;
+        for (uint64_t i = 0; i < count; ++i) {
+          auto expected = expecteds[i];
+          uint64_t val = 0;
+          do {
+            return_value = mVkDeviceFunctionStubs[device].vkGetSemaphoreCounterValueKHR(device, semaphores[i], &val);
+          } while (val < expected && (return_value == VK_SUCCESS));
+          if (return_value != VK_SUCCESS) {
+            break;
+          }
+        }
+
+        if (pushReturn) {
+            stack->push(return_value);
+        }
+        return true;
+   }
+   else {
+        GAPID_WARNING("Error during calling function vkGetEventStatus");
+        return false;
+   }
+}
+¶
 bool Vulkan::replayEnumeratePhysicalDevices(Stack* stack, bool pushReturn) {
     auto physicalDeviceIDs = stack->pop<uint64_t*>();
     auto physicalDevices = stack->pop<VkPhysicalDevice*>();

diff --git a/gapis/api/vulkan/BUILD.bazel b/gapis/api/vulkan/BUILD.bazel
@@ -58,7 +58,6 @@ apic_template(
 go_library(
     name = "go_default_library",
     srcs = [
-        "buffer_command.go",
         "command_buffer_rebuilder.go",
         "command_splitter.go",
         "custom_replay.go",
@@ -67,6 +66,7 @@ go_library(
         "draw_call_mesh.go",
         "external_memory.go",
         "externs.go",
+        "extras.go",
         "find_issues.go",
         "frame_loop.go",
         "graph_visualization.go",

diff --git a/gapis/api/vulkan/api/device.api b/gapis/api/vulkan/api/device.api
@@ -66,6 +66,9 @@
   @unused ref!PhysicalDeviceShaderClockFeaturesKHR PhysicalDeviceShaderClockFeaturesKHR
   @unused ref!PhysicalDeviceVulkanMemoryModelFeaturesKHR PhysicalDeviceVulkanMemoryModelFeaturesKHR
   @unused ref!PhysicalDeviceShaderFloat16Int8FeaturesKHR PhysicalDeviceShaderFloat16Int8FeaturesKHR
+
+  @unused ref!PhysicalDeviceShaderAtomicInt64Features PhysicalDeviceShaderAtomicInt64Features
+  @unused ref!PhysicalDeviceTimelineSemaphoreFeatures PhysicalDeviceTimelineSemaphoreFeatures
 }
 
 @indirect("VkDevice")
@@ -233,6 +236,19 @@ sub ref!DeviceObject createDeviceObject(const VkDeviceCreateInfo* data) {
             ShaderInt8: ext.shaderInt8,
           )
         }
+        case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES_KHR: {
+          ext := as!VkPhysicalDeviceShaderAtomicInt64FeaturesKHR*(next.Ptr)[0]
+          object.PhysicalDeviceShaderAtomicInt64Features = new!PhysicalDeviceShaderAtomicInt64Features(
+            ShaderBufferInt64Atomics: ext.shaderBufferInt64Atomics,
+            ShaderSharedInt64Atomics: ext.shaderSharedInt64Atomics,
+          )
+        }
+        case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES: {
+          ext := as!VkPhysicalDeviceTimelineSemaphoreFeatures*(next.Ptr)[0]
+          object.PhysicalDeviceTimelineSemaphoreFeatures = new!PhysicalDeviceTimelineSemaphoreFeatures(
+            TimelineSemaphore: ext.timelineSemaphore,
+          )
+        }
         default: {
           // do nothing
         }
@@ -312,3 +328,12 @@ sub ref!DeviceObject createDeviceObject(const VkDeviceCreateInfo* data) {
 @internal class SamplerYcbcrConversionFeatures {
   VkBool32        SamplerYcbcrConversion
 }
+
+@internal class PhysicalDeviceShaderAtomicInt64Features {
+  VkBool32        ShaderBufferInt64Atomics
+  VkBool32        ShaderSharedInt64Atomics
+}
+
+@internal class PhysicalDeviceTimelineSemaphoreFeatures {
+  VkBool32        TimelineSemaphore
+}
diff --git a/gapis/api/vulkan/api/enums.api b/gapis/api/vulkan/api/enums.api
@@ -394,6 +394,17 @@ enum VkStructureType: u32 {
 
   // @extension("VK_KHR_driver_properties")
   VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES_KHR = 1000196000,
+
+
+
+  // Vulkan 1.2
+  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES = 1000207000,
+  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_PROPERTIES = 1000207001,
+  VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO = 1000207002,
+  VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO = 1000207003,
+  VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO = 1000207004,
+  VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO = 1000207005,
+
 }
 
 enum VkObjectType: u32 {