Vulkan: Use timestamp queries to get more accurate latency values

Server compositor no longer includes time spent waiting on game
to finish rendering frames.

alvr/server/cpp/platform/linux/CEncoder.cpp

@@ -238,9 +238,6 @@ void CEncoder::Run() {
           continue;
         }
 
-        // Close enough to present
-        ReportPresent(pose->targetTimestampNs, 0);
-
         if (m_captureFrame) {
           m_captureFrame = false;
           render.Wait(frame_info.image, frame_info.semaphore_value);
@@ -250,8 +247,6 @@ void CEncoder::Run() {
 
         render.Render(frame_info.image, frame_info.semaphore_value);
 
-        ReportComposed(pose->targetTimestampNs, 0);
-
         encode_pipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion());
 
         static_assert(sizeof(frame_info.pose) == sizeof(vr::HmdMatrix34_t&));
@@ -263,6 +258,12 @@ void CEncoder::Run() {
           continue;
         }
 
+        auto timestamps = render.GetTimestamps();
+        uint64_t timestamp_present = timestamps.renderBegin;
+        uint64_t timestamp_composed = encode_pipeline->GetTimestamp() ? encode_pipeline->GetTimestamp() : timestamps.renderComplete;
+        ReportPresent(pose->targetTimestampNs, timestamps.now - timestamp_present);
+        ReportComposed(pose->targetTimestampNs, timestamps.now - timestamp_composed);
+
         m_listener->SendVideo(encoded_data.data(), encoded_data.size(), pts);
 
         m_listener->GetStatistics()->EncodeOutput();

alvr/server/cpp/platform/linux/EncodePipeline.cpp

@@ -125,3 +125,8 @@ bool alvr::EncodePipeline::GetEncoded(std::vector<uint8_t> &out, uint64_t *pts)
   av_packet_free(&enc_pkt);
   return true;
 }
+
+uint64_t alvr::EncodePipeline::GetTimestamp()
+{
+  return gpu_timestamp;
+}

alvr/server/cpp/platform/linux/EncodePipeline.h

@@ -21,11 +21,13 @@ class EncodePipeline
 
   virtual void PushFrame(uint64_t targetTimestampNs, bool idr) = 0;
   virtual bool GetEncoded(std::vector<uint8_t> & out, uint64_t *pts);
+  virtual uint64_t GetTimestamp();
 
   virtual void SetBitrate(int64_t bitrate);
   static std::unique_ptr<EncodePipeline> Create(Renderer *render, VkContext &vk_ctx, VkFrame &input_frame, VkFrameCtx &vk_frame_ctx, uint32_t width, uint32_t height);
 protected:
   AVCodecContext *encoder_ctx = nullptr; //shall be initialized by child class
+  uint64_t gpu_timestamp = 0;
 };
 
 }

alvr/server/cpp/platform/linux/EncodePipelineSW.cpp

@@ -116,6 +116,7 @@ alvr::EncodePipelineSW::~EncodePipelineSW()
 void alvr::EncodePipelineSW::PushFrame(uint64_t targetTimestampNs, bool idr)
 {
   rgbtoyuv->Convert(encoder_frame->data, encoder_frame->linesize);
+  gpu_timestamp = rgbtoyuv->GetTimestamp();
 
   encoder_frame->pict_type = idr ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
   encoder_frame->pts = targetTimestampNs;

alvr/server/cpp/platform/linux/FormatConverter.cpp

@@ -16,6 +16,7 @@ FormatConverter::~FormatConverter()
     }
 
     vkDestroySampler(r->m_dev, m_sampler, nullptr);
+    vkDestroyQueryPool(r->m_dev, m_queryPool, nullptr);
     vkDestroyCommandPool(r->m_dev, m_commandPool, nullptr);
     vkDestroyDescriptorSetLayout(r->m_dev, m_descriptorLayout, nullptr);
     vkDestroyImageView(r->m_dev, m_view, nullptr);
@@ -40,6 +41,13 @@ void FormatConverter::init(VkImage image, VkImageCreateInfo imageCreateInfo, int
     samplerInfo.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
     VK_CHECK(vkCreateSampler(r->m_dev, &samplerInfo, nullptr, &m_sampler));
 
+    // Timestamp query
+    VkQueryPoolCreateInfo queryPoolInfo = {};
+    queryPoolInfo.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
+    queryPoolInfo.queryType = VK_QUERY_TYPE_TIMESTAMP;
+    queryPoolInfo.queryCount = 1;
+    VK_CHECK(vkCreateQueryPool(r->m_dev, &queryPoolInfo, nullptr, &m_queryPool));
+
     // Command buffer
     VkCommandPoolCreateInfo cmdPoolInfo = {};
     cmdPoolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
@@ -231,10 +239,14 @@ void FormatConverter::Convert(uint8_t **data, int *linesize)
     commandBufferBegin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
     VK_CHECK(vkBeginCommandBuffer(m_commandBuffer, &commandBufferBegin));
 
+    vkCmdResetQueryPool(m_commandBuffer, m_queryPool, 0, 1);
+
     vkCmdBindPipeline(m_commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, m_pipeline);
     vkCmdBindDescriptorSets(m_commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, m_pipelineLayout, 0, 1, &m_descriptor, 0, nullptr);
     vkCmdDispatch(m_commandBuffer, m_groupCountX, m_groupCountY, 1);
 
+    vkCmdWriteTimestamp(m_commandBuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, m_queryPool, 0);
+
     vkEndCommandBuffer(m_commandBuffer);
 
     VkSubmitInfo submitInfo = {};
@@ -252,6 +264,13 @@ void FormatConverter::Convert(uint8_t **data, int *linesize)
     }
 }
 
+uint64_t FormatConverter::GetTimestamp()
+{
+    uint64_t query;
+    VK_CHECK(vkGetQueryPoolResults(r->m_dev, m_queryPool, 0, 1, sizeof(uint64_t), &query, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT));
+    return query * r->m_timestampPeriod;
+}
+
 RgbToYuv420::RgbToYuv420(Renderer *render, VkImage image, VkImageCreateInfo imageInfo)
     : FormatConverter(render)
 {

alvr/server/cpp/platform/linux/FormatConverter.h

@@ -9,6 +9,8 @@ class FormatConverter
 
     void Convert(uint8_t **data, int *linesize);
 
+    uint64_t GetTimestamp();
+
 protected:
     struct OutputImage {
         VkImage image;
@@ -23,6 +25,7 @@ class FormatConverter
 
     Renderer *r;
     VkSampler m_sampler;
+    VkQueryPool m_queryPool;
     VkCommandPool m_commandPool;
     VkCommandBuffer m_commandBuffer;
     VkDescriptorSet m_descriptor;

alvr/server/cpp/platform/linux/Renderer.cpp

@@ -52,7 +52,12 @@ Renderer::Renderer(const VkInstance &inst, const VkDevice &dev, const VkPhysical
     VK_LOAD_PFN(vkImportSemaphoreFdKHR);
     VK_LOAD_PFN(vkGetMemoryFdKHR);
     VK_LOAD_PFN(vkGetImageDrmFormatModifierPropertiesEXT);
+    VK_LOAD_PFN(vkGetCalibratedTimestampsEXT);
 #undef VK_LOAD_PFN
+
+    VkPhysicalDeviceProperties props = {};
+    vkGetPhysicalDeviceProperties(m_physDev, &props);
+    m_timestampPeriod = props.limits.timestampPeriod;
 }
 
 Renderer::~Renderer()
@@ -76,6 +81,7 @@ Renderer::~Renderer()
     vkFreeMemory(m_dev, m_output.memory, nullptr);
     vkDestroyFramebuffer(m_dev, m_output.framebuffer, nullptr);
 
+    vkDestroyQueryPool(m_dev, m_queryPool, nullptr);
     vkDestroyCommandPool(m_dev, m_commandPool, nullptr);
     vkDestroySampler(m_dev, m_sampler, nullptr);
     vkDestroyBuffer(m_dev, m_vertexBuffer, nullptr);
@@ -95,6 +101,13 @@ void Renderer::Startup(uint32_t width, uint32_t height, VkFormat format)
     vkGetDeviceQueue(m_dev, m_queueFamilyIndex, 0, &m_queue);
     vkGetDeviceQueue(m_dev, m_queueFamilyIndexCompute, 0, &m_queueCompute);
 
+    // Timestamp query
+    VkQueryPoolCreateInfo queryPoolInfo = {};
+    queryPoolInfo.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
+    queryPoolInfo.queryType = VK_QUERY_TYPE_TIMESTAMP;
+    queryPoolInfo.queryCount = 2;
+    VK_CHECK(vkCreateQueryPool(m_dev, &queryPoolInfo, nullptr, &m_queryPool));
+
     // Command buffer
     VkCommandPoolCreateInfo cmdPoolInfo = {};
     cmdPoolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
@@ -575,6 +588,9 @@ void Renderer::Render(uint32_t index, uint64_t waitValue)
     commandBufferBegin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
     VK_CHECK(vkBeginCommandBuffer(m_commandBuffer, &commandBufferBegin));
 
+    vkCmdResetQueryPool(m_commandBuffer, m_queryPool, 0, 2);
+    vkCmdWriteTimestamp(m_commandBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, m_queryPool, 0);
+
     for (size_t i = 0; i < m_pipelines.size(); ++i) {
         VkRect2D rect = {};
         VkDescriptorSet in = VK_NULL_HANDLE;
@@ -595,6 +611,8 @@ void Renderer::Render(uint32_t index, uint64_t waitValue)
         m_pipelines[i]->Render(in, out, rect);
     }
 
+    vkCmdWriteTimestamp(m_commandBuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, m_queryPool, 1);
+
     VK_CHECK(vkEndCommandBuffer(m_commandBuffer));
 
     VkTimelineSemaphoreSubmitInfo timelineInfo = {};
@@ -623,6 +641,24 @@ void Renderer::Render(uint32_t index, uint64_t waitValue)
     }
 }
 
+Renderer::Timestamps Renderer::GetTimestamps()
+{
+    uint64_t queries[2];
+    VK_CHECK(vkGetQueryPoolResults(m_dev, m_queryPool, 0, 2, 2 * sizeof(uint64_t), queries, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT));
+    queries[0] *= m_timestampPeriod;
+    queries[1] *= m_timestampPeriod;
+
+    VkCalibratedTimestampInfoEXT timestampInfo = {};
+    timestampInfo.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT;
+    timestampInfo.timeDomain = VK_TIME_DOMAIN_DEVICE_EXT;
+    uint64_t deviation;
+    uint64_t timestamp;
+    VK_CHECK(d.vkGetCalibratedTimestampsEXT(m_dev, 1, &timestampInfo, &timestamp, &deviation));
+    timestamp *= m_timestampPeriod;
+
+    return {timestamp, queries[0], queries[1]};
+}
+
 void Renderer::Wait(uint32_t index, uint64_t waitValue)
 {
     VkSemaphoreWaitInfo waitInfo = {};

alvr/server/cpp/platform/linux/Renderer.h

@@ -41,6 +41,12 @@ class Renderer
         DrmImage drm;
     };
 
+    struct Timestamps {
+        uint64_t now;
+        uint64_t renderBegin;
+        uint64_t renderComplete;
+    };
+
     explicit Renderer(const VkInstance &inst, const VkDevice &dev, const VkPhysicalDevice &physDev, uint32_t graphicsIdx, uint32_t computeIdx, const std::vector<const char *> &devExtensions);
     virtual ~Renderer();
 
@@ -54,6 +60,8 @@ class Renderer
 
     void Render(uint32_t index, uint64_t waitValue);
 
+    Timestamps GetTimestamps();
+
     void CopyOutput(VkImage image, VkFormat format, VkImageLayout layout, VkSemaphore *semaphore = nullptr, VkFence *fence = nullptr);
 
     void Wait(uint32_t index, uint64_t waitValue);
@@ -89,6 +97,7 @@ class Renderer
         PFN_vkImportSemaphoreFdKHR vkImportSemaphoreFdKHR;
         PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR;
         PFN_vkGetImageDrmFormatModifierPropertiesEXT vkGetImageDrmFormatModifierPropertiesEXT;
+        PFN_vkGetCalibratedTimestampsEXT vkGetCalibratedTimestampsEXT;
         bool haveDmaBuf = false;
         bool haveDrmModifiers = false;
     } d;
@@ -107,6 +116,7 @@ class Renderer
     uint32_t m_queueFamilyIndexCompute;
     VkFormat m_format;
     VkExtent2D m_imageSize;
+    VkQueryPool m_queryPool;
     VkCommandPool m_commandPool;
     VkSampler m_sampler;
     VkBuffer m_vertexBuffer;
@@ -116,6 +126,7 @@ class Renderer
     VkDescriptorSetLayout m_descriptorLayout;
     VkCommandBuffer m_commandBuffer;
     VkFence m_fence;
+    double m_timestampPeriod;
 
     std::string m_inputImageCapture;
     std::string m_outputImageCapture;

alvr/server/cpp/platform/linux/ffmpeg_helper.cpp

@@ -53,6 +53,7 @@ alvr::VkContext::VkContext(const char *deviceName, const std::vector<const char*
       VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
       VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME,
       VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME
+      VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME,
   };
   device_extensions.insert(device_extensions.end(), requiredDeviceExtensions.begin(), requiredDeviceExtensions.end());