Implement win32/linux checking and registering of vulkan layers

baldurk · Feb 16, 2017 · 11a914b · 11a914b
1 parent 72921f2
commit 11a914b
Show file tree

Hide file tree

Showing 14 changed files with 640 additions and 379 deletions.
diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp
@@ -294,6 +294,19 @@ void CaptureDialog::on_vulkanLayerWarn_clicked()
   const bool registerAll = (flags & eVulkan_RegisterAll);
   const bool updateAllowed = (flags & eVulkan_UpdateAllowed);
 
+  if(flags & eVulkan_Unfixable)
+  {
+    QString msg =
+        tr("There is an unfixable problem with your vulkan layer configuration. Please consult the "
+           "RenderDoc documentation, or package/distribution documentation on linux\n\n");
+
+    for(const rdctype::str &j : otherJSONs)
+      msg += ToQStr(j) + "\n";
+
+    RDDialog::critical(this, tr("Unfixable vulkan layer configuration"), msg);
+    return;
+  }
+
   QString msg =
       tr("Vulkan capture happens through the API's layer mechanism. RenderDoc has detected that ");
 

diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.ui b/qrenderdoc/Windows/Dialogs/CaptureDialog.ui
@@ -271,7 +271,7 @@
      <property name="minimumSize">
       <size>
        <width>0</width>
-       <height>36</height>
+       <height>40</height>
       </size>
      </property>
      <property name="cursor">

diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h
@@ -648,7 +648,7 @@ extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess(
 
 extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration(
     uint32_t *flags, rdctype::array<rdctype::str> *myJSONs, rdctype::array<rdctype::str> *otherJSONs);
-extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate);
+extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool systemLevel);
 
 //////////////////////////////////////////////////////////////////////////
 // Miscellaneous!

diff --git a/renderdoc/api/replay/replay_enums.h b/renderdoc/api/replay/replay_enums.h
@@ -610,4 +610,5 @@ enum VulkanFlags
   eVulkan_CouldElevate = 0x8,
   eVulkan_RegisterAll = 0x10,
   eVulkan_UpdateAllowed = 0x20,
+  eVulkan_Unfixable = 0x40,
 };
diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp
@@ -237,6 +237,9 @@ RenderDoc::RenderDoc()
 
   m_Overlay = eRENDERDOC_Overlay_Default;
 
+  m_VulkanCheck = NULL;
+  m_VulkanInstall = NULL;
+
   m_TargetControlThreadShutdown = false;
   m_ControlClientThreadShutdown = false;
 }

diff --git a/renderdoc/core/core.h b/renderdoc/core/core.h
@@ -171,6 +171,10 @@ class IReplayDriver;
 typedef ReplayCreateStatus (*RemoteDriverProvider)(const char *logfile, IRemoteDriver **driver);
 typedef ReplayCreateStatus (*ReplayDriverProvider)(const char *logfile, IReplayDriver **driver);
 
+typedef bool (*VulkanLayerCheck)(uint32_t &flags, std::vector<std::string> &myJSONs,
+                                 std::vector<std::string> &otherJSONs);
+typedef void (*VulkanLayerInstall)(bool systemLevel);
+
 typedef void (*ShutdownFunction)();
 
 // this class mediates everything and owns any 'global' resources such as the crash handler.
@@ -240,6 +244,23 @@ class RenderDoc
   void RegisterReplayProvider(RDCDriver driver, const char *name, ReplayDriverProvider provider);
   void RegisterRemoteProvider(RDCDriver driver, const char *name, RemoteDriverProvider provider);
 
+  void SetVulkanLayerCheck(VulkanLayerCheck callback) { m_VulkanCheck = callback; }
+  void SetVulkanLayerInstall(VulkanLayerInstall callback) { m_VulkanInstall = callback; }
+  bool NeedVulkanLayerRegistration(uint32_t &flags, std::vector<std::string> &myJSONs,
+                                   std::vector<std::string> &otherJSONs)
+  {
+    if(m_VulkanCheck)
+      return m_VulkanCheck(flags, myJSONs, otherJSONs);
+
+    return false;
+  }
+
+  void UpdateVulkanLayerRegistration(bool systemLevel)
+  {
+    if(m_VulkanInstall)
+      m_VulkanInstall(systemLevel);
+  }
+
   ReplayCreateStatus CreateReplayDriver(RDCDriver driverType, const char *logfile,
                                         IReplayDriver **driver);
   ReplayCreateStatus CreateRemoteDriver(RDCDriver driverType, const char *logfile,
@@ -355,6 +376,9 @@ class RenderDoc
   map<RDCDriver, ReplayDriverProvider> m_ReplayDriverProviders;
   map<RDCDriver, RemoteDriverProvider> m_RemoteDriverProviders;
 
+  VulkanLayerCheck m_VulkanCheck;
+  VulkanLayerInstall m_VulkanInstall;
+
   set<ShutdownFunction> m_ShutdownFunctions;
 
   struct FrameCap

diff --git a/renderdoc/driver/vulkan/vk_android.cpp b/renderdoc/driver/vulkan/vk_android.cpp
@@ -57,3 +57,14 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 }
 
 const char *VulkanLibraryName = "libvulkan.so";
+
+bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
+                                    std::vector<std::string> &otherJSONs)
+{
+  // nothing to do
+  return false;
+}
+
+void VulkanReplay::InstallVulkanLayer(bool systemLevel)
+{
+}
diff --git a/renderdoc/driver/vulkan/vk_apple.cpp b/renderdoc/driver/vulkan/vk_apple.cpp
@@ -42,3 +42,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 }
 
 const char *VulkanLibraryName = "libvulkan.so";
+
+bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
+                                    std::vector<std::string> &otherJSONs)
+{
+  return false;
+}
+
+void VulkanReplay::InstallVulkanLayer(bool systemLevel)
+{
+}