Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Statistics: Add offset to ReportPresent/ReportComposed #1361

Merged
merged 2 commits into from Jan 12, 2023
Merged

Statistics: Add offset to ReportPresent/ReportComposed #1361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6ccc70
Statistics: Add offset to ReportPresent/ReportComposed
nowrep Jan 8, 2023
7ca8bf5
Vulkan: Use timestamp queries to get more accurate latency values
nowrep Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions alvr/server/cpp/alvr_server/alvr_server.cpp
View file
Open in desktop
Expand Up @@ -181,8 +181,8 @@ void (*VideoSend)(VideoFrame header, unsigned char *buf, int len);
void (*HapticsSend)(unsigned long long path, float duration_s, float frequency, float amplitude);
void (*ShutdownRuntime)();
unsigned long long (*PathStringToHash)(const char *path);
void (*ReportPresent)(unsigned long long timestamp_ns);
void (*ReportComposed)(unsigned long long timestamp_ns);
void (*ReportPresent)(unsigned long long timestamp_ns, unsigned long long offset_ns);
void (*ReportComposed)(unsigned long long timestamp_ns, unsigned long long offset_ns);
void (*ReportEncoded)(unsigned long long timestamp_ns);
void (*ReportFecFailure)(int percentage);

Expand Down
4 changes: 2 additions & 2 deletions alvr/server/cpp/alvr_server/bindings.h
View file
Open in desktop
Expand Up @@ -130,8 +130,8 @@ extern "C" void (*HapticsSend)(unsigned long long path,
float amplitude);
extern "C" void (*ShutdownRuntime)();
extern "C" unsigned long long (*PathStringToHash)(const char *path);
extern "C" void (*ReportPresent)(unsigned long long timestamp_ns);
extern "C" void (*ReportComposed)(unsigned long long timestamp_ns);
extern "C" void (*ReportPresent)(unsigned long long timestamp_ns, unsigned long long offset_ns);
extern "C" void (*ReportComposed)(unsigned long long timestamp_ns, unsigned long long offset_ns);
extern "C" void (*ReportEncoded)(unsigned long long timestamp_ns);
extern "C" void (*ReportFecFailure)(int percentage);

Expand Down
11 changes: 6 additions & 5 deletions alvr/server/cpp/platform/linux/CEncoder.cpp
View file
Open in desktop
Expand Up @@ -238,9 +238,6 @@ void CEncoder::Run() {
continue;
}

// Close enough to present
ReportPresent(pose->targetTimestampNs);

if (m_captureFrame) {
m_captureFrame = false;
render.Wait(frame_info.image, frame_info.semaphore_value);
Expand All @@ -250,8 +247,6 @@ void CEncoder::Run() {

render.Render(frame_info.image, frame_info.semaphore_value);

ReportComposed(pose->targetTimestampNs);

encode_pipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion());

static_assert(sizeof(frame_info.pose) == sizeof(vr::HmdMatrix34_t&));
Expand All @@ -263,6 +258,12 @@ void CEncoder::Run() {
continue;
}

auto timestamps = render.GetTimestamps();
uint64_t timestamp_present = timestamps.renderBegin;
uint64_t timestamp_composed = encode_pipeline->GetTimestamp() ? encode_pipeline->GetTimestamp() : timestamps.renderComplete;
ReportPresent(pose->targetTimestampNs, timestamps.now - timestamp_present);
ReportComposed(pose->targetTimestampNs, timestamps.now - timestamp_composed);

m_listener->SendVideo(encoded_data.data(), encoded_data.size(), pts);

m_listener->GetStatistics()->EncodeOutput();
Expand Down
5 changes: 5 additions & 0 deletions alvr/server/cpp/platform/linux/EncodePipeline.cpp
View file
Open in desktop
Expand Up @@ -125,3 +125,8 @@ bool alvr::EncodePipeline::GetEncoded(std::vector<uint8_t> &out, uint64_t *pts)
av_packet_free(&enc_pkt);
return true;
}

uint64_t alvr::EncodePipeline::GetTimestamp()
{
return gpu_timestamp;
}
2 changes: 2 additions & 0 deletions alvr/server/cpp/platform/linux/EncodePipeline.h
View file
Open in desktop
Expand Up @@ -21,11 +21,13 @@ class EncodePipeline

virtual void PushFrame(uint64_t targetTimestampNs, bool idr) = 0;
virtual bool GetEncoded(std::vector<uint8_t> & out, uint64_t *pts);
virtual uint64_t GetTimestamp();

virtual void SetBitrate(int64_t bitrate);
static std::unique_ptr<EncodePipeline> Create(Renderer *render, VkContext &vk_ctx, VkFrame &input_frame, VkFrameCtx &vk_frame_ctx, uint32_t width, uint32_t height);
protected:
AVCodecContext *encoder_ctx = nullptr; //shall be initialized by child class
uint64_t gpu_timestamp = 0;
};

}
1 change: 1 addition & 0 deletions alvr/server/cpp/platform/linux/EncodePipelineSW.cpp
View file
Open in desktop
Expand Up @@ -116,6 +116,7 @@ alvr::EncodePipelineSW::~EncodePipelineSW()
void alvr::EncodePipelineSW::PushFrame(uint64_t targetTimestampNs, bool idr)
{
rgbtoyuv->Convert(encoder_frame->data, encoder_frame->linesize);
gpu_timestamp = rgbtoyuv->GetTimestamp();

encoder_frame->pict_type = idr ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
encoder_frame->pts = targetTimestampNs;
Expand Down
19 changes: 19 additions & 0 deletions alvr/server/cpp/platform/linux/FormatConverter.cpp
View file
Open in desktop
Expand Up @@ -16,6 +16,7 @@ FormatConverter::~FormatConverter()
}

vkDestroySampler(r->m_dev, m_sampler, nullptr);
vkDestroyQueryPool(r->m_dev, m_queryPool, nullptr);
vkDestroyCommandPool(r->m_dev, m_commandPool, nullptr);
vkDestroyDescriptorSetLayout(r->m_dev, m_descriptorLayout, nullptr);
vkDestroyImageView(r->m_dev, m_view, nullptr);
Expand All @@ -40,6 +41,13 @@ void FormatConverter::init(VkImage image, VkImageCreateInfo imageCreateInfo, int
samplerInfo.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
VK_CHECK(vkCreateSampler(r->m_dev, &samplerInfo, nullptr, &m_sampler));

// Timestamp query
VkQueryPoolCreateInfo queryPoolInfo = {};
queryPoolInfo.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
queryPoolInfo.queryType = VK_QUERY_TYPE_TIMESTAMP;
queryPoolInfo.queryCount = 1;
VK_CHECK(vkCreateQueryPool(r->m_dev, &queryPoolInfo, nullptr, &m_queryPool));

// Command buffer
VkCommandPoolCreateInfo cmdPoolInfo = {};
cmdPoolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
Expand Down Expand Up @@ -231,10 +239,14 @@ void FormatConverter::Convert(uint8_t **data, int *linesize)
commandBufferBegin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
VK_CHECK(vkBeginCommandBuffer(m_commandBuffer, &commandBufferBegin));

vkCmdResetQueryPool(m_commandBuffer, m_queryPool, 0, 1);

vkCmdBindPipeline(m_commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, m_pipeline);
vkCmdBindDescriptorSets(m_commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, m_pipelineLayout, 0, 1, &m_descriptor, 0, nullptr);
vkCmdDispatch(m_commandBuffer, m_groupCountX, m_groupCountY, 1);

vkCmdWriteTimestamp(m_commandBuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, m_queryPool, 0);

vkEndCommandBuffer(m_commandBuffer);

VkSubmitInfo submitInfo = {};
Expand All @@ -252,6 +264,13 @@ void FormatConverter::Convert(uint8_t **data, int *linesize)
}
}

uint64_t FormatConverter::GetTimestamp()
{
uint64_t query;
VK_CHECK(vkGetQueryPoolResults(r->m_dev, m_queryPool, 0, 1, sizeof(uint64_t), &query, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT));
return query * r->m_timestampPeriod;
}

RgbToYuv420::RgbToYuv420(Renderer *render, VkImage image, VkImageCreateInfo imageInfo)
: FormatConverter(render)
{
Expand Down
3 changes: 3 additions & 0 deletions alvr/server/cpp/platform/linux/FormatConverter.h
View file
Open in desktop
Expand Up @@ -9,6 +9,8 @@ class FormatConverter

void Convert(uint8_t **data, int *linesize);

uint64_t GetTimestamp();

protected:
struct OutputImage {
VkImage image;
Expand All @@ -23,6 +25,7 @@ class FormatConverter

Renderer *r;
VkSampler m_sampler;
VkQueryPool m_queryPool;
VkCommandPool m_commandPool;
VkCommandBuffer m_commandBuffer;
VkDescriptorSet m_descriptor;
Expand Down
36 changes: 36 additions & 0 deletions alvr/server/cpp/platform/linux/Renderer.cpp
View file
Open in desktop
Expand Up @@ -52,7 +52,12 @@ Renderer::Renderer(const VkInstance &inst, const VkDevice &dev, const VkPhysical
VK_LOAD_PFN(vkImportSemaphoreFdKHR);
VK_LOAD_PFN(vkGetMemoryFdKHR);
VK_LOAD_PFN(vkGetImageDrmFormatModifierPropertiesEXT);
VK_LOAD_PFN(vkGetCalibratedTimestampsEXT);
#undef VK_LOAD_PFN

VkPhysicalDeviceProperties props = {};
vkGetPhysicalDeviceProperties(m_physDev, &props);
m_timestampPeriod = props.limits.timestampPeriod;
}

Renderer::~Renderer()
Expand All @@ -76,6 +81,7 @@ Renderer::~Renderer()
vkFreeMemory(m_dev, m_output.memory, nullptr);
vkDestroyFramebuffer(m_dev, m_output.framebuffer, nullptr);

vkDestroyQueryPool(m_dev, m_queryPool, nullptr);
vkDestroyCommandPool(m_dev, m_commandPool, nullptr);
vkDestroySampler(m_dev, m_sampler, nullptr);
vkDestroyBuffer(m_dev, m_vertexBuffer, nullptr);
Expand All @@ -95,6 +101,13 @@ void Renderer::Startup(uint32_t width, uint32_t height, VkFormat format)
vkGetDeviceQueue(m_dev, m_queueFamilyIndex, 0, &m_queue);
vkGetDeviceQueue(m_dev, m_queueFamilyIndexCompute, 0, &m_queueCompute);

// Timestamp query
VkQueryPoolCreateInfo queryPoolInfo = {};
queryPoolInfo.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
queryPoolInfo.queryType = VK_QUERY_TYPE_TIMESTAMP;
queryPoolInfo.queryCount = 2;
VK_CHECK(vkCreateQueryPool(m_dev, &queryPoolInfo, nullptr, &m_queryPool));

// Command buffer
VkCommandPoolCreateInfo cmdPoolInfo = {};
cmdPoolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
Expand Down Expand Up @@ -575,6 +588,9 @@ void Renderer::Render(uint32_t index, uint64_t waitValue)
commandBufferBegin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
VK_CHECK(vkBeginCommandBuffer(m_commandBuffer, &commandBufferBegin));

vkCmdResetQueryPool(m_commandBuffer, m_queryPool, 0, 2);
vkCmdWriteTimestamp(m_commandBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, m_queryPool, 0);

for (size_t i = 0; i < m_pipelines.size(); ++i) {
VkRect2D rect = {};
VkDescriptorSet in = VK_NULL_HANDLE;
Expand All @@ -595,6 +611,8 @@ void Renderer::Render(uint32_t index, uint64_t waitValue)
m_pipelines[i]->Render(in, out, rect);
}

vkCmdWriteTimestamp(m_commandBuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, m_queryPool, 1);

VK_CHECK(vkEndCommandBuffer(m_commandBuffer));

VkTimelineSemaphoreSubmitInfo timelineInfo = {};
Expand Down Expand Up @@ -623,6 +641,24 @@ void Renderer::Render(uint32_t index, uint64_t waitValue)
}
}

Renderer::Timestamps Renderer::GetTimestamps()
{
uint64_t queries[2];
VK_CHECK(vkGetQueryPoolResults(m_dev, m_queryPool, 0, 2, 2 * sizeof(uint64_t), queries, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT));
queries[0] *= m_timestampPeriod;
queries[1] *= m_timestampPeriod;

VkCalibratedTimestampInfoEXT timestampInfo = {};
timestampInfo.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT;
timestampInfo.timeDomain = VK_TIME_DOMAIN_DEVICE_EXT;
uint64_t deviation;
uint64_t timestamp;
VK_CHECK(d.vkGetCalibratedTimestampsEXT(m_dev, 1, &timestampInfo, &timestamp, &deviation));
timestamp *= m_timestampPeriod;

return {timestamp, queries[0], queries[1]};
}

void Renderer::Wait(uint32_t index, uint64_t waitValue)
{
VkSemaphoreWaitInfo waitInfo = {};
Expand Down
11 changes: 11 additions & 0 deletions alvr/server/cpp/platform/linux/Renderer.h
View file
Open in desktop
Expand Up @@ -41,6 +41,12 @@ class Renderer
DrmImage drm;
};

struct Timestamps {
uint64_t now;
uint64_t renderBegin;
uint64_t renderComplete;
};

explicit Renderer(const VkInstance &inst, const VkDevice &dev, const VkPhysicalDevice &physDev, uint32_t graphicsIdx, uint32_t computeIdx, const std::vector<const char *> &devExtensions);
virtual ~Renderer();

Expand All @@ -54,6 +60,8 @@ class Renderer

void Render(uint32_t index, uint64_t waitValue);

Timestamps GetTimestamps();

void CopyOutput(VkImage image, VkFormat format, VkImageLayout layout, VkSemaphore *semaphore = nullptr, VkFence *fence = nullptr);

void Wait(uint32_t index, uint64_t waitValue);
Expand Down Expand Up @@ -89,6 +97,7 @@ class Renderer
PFN_vkImportSemaphoreFdKHR vkImportSemaphoreFdKHR;
PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR;
PFN_vkGetImageDrmFormatModifierPropertiesEXT vkGetImageDrmFormatModifierPropertiesEXT;
PFN_vkGetCalibratedTimestampsEXT vkGetCalibratedTimestampsEXT;
bool haveDmaBuf = false;
bool haveDrmModifiers = false;
} d;
Expand All @@ -107,6 +116,7 @@ class Renderer
uint32_t m_queueFamilyIndexCompute;
VkFormat m_format;
VkExtent2D m_imageSize;
VkQueryPool m_queryPool;
VkCommandPool m_commandPool;
VkSampler m_sampler;
VkBuffer m_vertexBuffer;
Expand All @@ -116,6 +126,7 @@ class Renderer
VkDescriptorSetLayout m_descriptorLayout;
VkCommandBuffer m_commandBuffer;
VkFence m_fence;
double m_timestampPeriod;

std::string m_inputImageCapture;
std::string m_outputImageCapture;
Expand Down
3 changes: 2 additions & 1 deletion alvr/server/cpp/platform/linux/ffmpeg_helper.cpp
View file
Open in desktop
Expand Up @@ -52,7 +52,8 @@ alvr::VkContext::VkContext(const char *deviceName, const std::vector<const char*
VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME,
VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME,
VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME
VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME,
VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME,
};
device_extensions.insert(device_extensions.end(), requiredDeviceExtensions.begin(), requiredDeviceExtensions.end());

Expand Down
4 changes: 2 additions & 2 deletions alvr/server/cpp/platform/win32/OvrDirectModeComponent.cpp
View file
Open in desktop
Expand Up @@ -155,7 +155,7 @@ void OvrDirectModeComponent::SubmitLayer(const SubmitLayerPerEye_t(&perEye)[2])
/** Submits queued layers for display. */
void OvrDirectModeComponent::Present(vr::SharedTextureHandle_t syncTexture)
{
ReportPresent(m_targetTimestampNs);
ReportPresent(m_targetTimestampNs, 0);

bool useMutex = true;

Expand Down Expand Up @@ -203,7 +203,7 @@ void OvrDirectModeComponent::Present(vr::SharedTextureHandle_t syncTexture)
}
}

ReportComposed(m_targetTimestampNs);
ReportComposed(m_targetTimestampNs, 0);

if (m_pEncoder) {
m_pEncoder->NewFrameReady();
Expand Down
14 changes: 10 additions & 4 deletions alvr/server/src/lib.rs
View file
Open in desktop
Expand Up @@ -425,15 +425,21 @@ pub unsafe extern "C" fn HmdDriverFactory(
alvr_common::hash_string(CStr::from_ptr(path).to_str().unwrap())
}

extern "C" fn report_present(timestamp_ns: u64) {
extern "C" fn report_present(timestamp_ns: u64, offset_ns: u64) {
if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
stats.report_frame_present(Duration::from_nanos(timestamp_ns));
stats.report_frame_present(
Duration::from_nanos(timestamp_ns),
Duration::from_nanos(offset_ns),
);
}
}

extern "C" fn report_composed(timestamp_ns: u64) {
extern "C" fn report_composed(timestamp_ns: u64, offset_ns: u64) {
if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
stats.report_frame_composed(Duration::from_nanos(timestamp_ns));
stats.report_frame_composed(
Duration::from_nanos(timestamp_ns),
Duration::from_nanos(offset_ns),
);
}
}

Expand Down
8 changes: 4 additions & 4 deletions alvr/server/src/statistics.rs
View file
Open in desktop
Expand Up @@ -89,13 +89,13 @@ impl StatisticsManager {
}
}

pub fn report_frame_present(&mut self, target_timestamp: Duration) {
pub fn report_frame_present(&mut self, target_timestamp: Duration, offset: Duration) {
if let Some(frame) = self
.history_buffer
.iter_mut()
.find(|frame| frame.target_timestamp == target_timestamp)
{
let now = Instant::now();
let now = Instant::now() - offset;

self.last_frame_present_interval =
now.saturating_duration_since(self.last_frame_present_instant);
Expand All @@ -105,13 +105,13 @@ impl StatisticsManager {
}
}

pub fn report_frame_composed(&mut self, target_timestamp: Duration) {
pub fn report_frame_composed(&mut self, target_timestamp: Duration, offset: Duration) {
if let Some(frame) = self
.history_buffer
.iter_mut()
.find(|frame| frame.target_timestamp == target_timestamp)
{
frame.frame_composed = Instant::now();
frame.frame_composed = Instant::now() - offset;
}
}

Expand Down