From 9074c01b9a1e819e5652af74d4931d714861402b Mon Sep 17 00:00:00 2001 From: Jim Britton Date: Wed, 20 Mar 2024 20:31:50 +0000 Subject: [PATCH] OpenVR and OpenXR symmetric and other projection matrix options (#196) * naive symmetric projection for OpenVR * WIP OpenXR symmetric projection * first cut of OpenXR symmetric projection matrices - desparately needs tidying up but appears to work correctly * tweak a calculation the looked upside down (but is hard to tell with Index's almost vertically symmetric projection) * WIP * more WIP - reworked bounds and FOV calculations to reduce mess; still need to verify OpenXR signs * wip * wip * corrected various calculations; added mirrored projections * add option to grow the render target to accommodate projection cropping with no image quality loss (at the expense of performance) * revert commit hash change * fix memory leak; first part of changes following PR comments * more tidy up - cache the results of the eye projection and texture bounds / scaling calculations * only get the eye positions once per framework sync * ensure changes to near clipping plane trigger eye matrix derivation; added original FOV to log messages when deriving eye matrices * fix incorrect vertical matched for openVR * Remove unused CommitHash.hpp --------- Co-authored-by: praydog --- src/mods/VR.cpp | 9 ++ src/mods/VR.hpp | 52 +++++++- src/mods/vr/D3D11Component.cpp | 15 ++- src/mods/vr/D3D12Component.cpp | 15 ++- src/mods/vr/OverlayComponent.cpp | 6 + src/mods/vr/runtimes/OpenVR.cpp | 96 ++++++++++++--- src/mods/vr/runtimes/OpenXR.cpp | 192 +++++++++++++++++++---------- src/mods/vr/runtimes/OpenXR.hpp | 1 + src/mods/vr/runtimes/VRRuntime.hpp | 14 +++ 9 files changed, 298 insertions(+), 102 deletions(-) diff --git a/src/mods/VR.cpp b/src/mods/VR.cpp index a0573f69..4b22ad0d 100644 --- a/src/mods/VR.cpp +++ b/src/mods/VR.cpp @@ -2533,6 +2533,15 @@ void VR::on_draw_sidebar_entry(std::string_view name) { m_compatibility_skip_pip->draw("Skip PostInitProperties"); m_sceneview_compatibility_mode->draw("SceneView Compatibility Mode"); m_extreme_compat_mode->draw("Extreme Compatibility Mode"); + + // changes to any of these options should trigger a regeneration of the eye projection matrices + const auto horizontal_projection_changed = m_horizontal_projection_override->draw("Horizontal Projection"); + const auto vertical_projection_changed = m_vertical_projection_override->draw("Vertical Projection"); + const auto scale_render = m_grow_rectangle_for_projection_cropping->draw("Scale Render Target"); + const auto scale_render_changed = get_runtime()->is_modifying_eye_texture_scale != scale_render; + get_runtime()->is_modifying_eye_texture_scale = scale_render; + get_runtime()->should_recalculate_eye_projections = horizontal_projection_changed || vertical_projection_changed || scale_render_changed; + ImGui::TreePop(); } diff --git a/src/mods/VR.hpp b/src/mods/VR.hpp index e2101115..743621c9 100644 --- a/src/mods/VR.hpp +++ b/src/mods/VR.hpp @@ -54,6 +54,18 @@ class VR : public Mod { GESTURE_HEAD_RIGHT, }; + enum HORIZONTAL_PROJECTION_OVERRIDE : int32_t { + HORIZONTAL_DEFAULT, + HORIZONTAL_SYMMETRIC, + HORIZONTAL_MIRROR + }; + + enum VERTICAL_PROJECTION_OVERRIDE : int32_t { + VERTICAL_DEFAULT, + VERTICAL_SYMMETRIC, + VERTICAL_MATCHED + }; + static const inline std::string s_action_pose = "/actions/default/in/Pose"; static const inline std::string s_action_grip_pose = "/actions/default/in/GripPose"; static const inline std::string s_action_trigger = "/actions/default/in/Trigger"; @@ -102,6 +114,11 @@ class VR : public Mod { }; } + // texture bounds to tell OpenVR which parts of the submitted texture to render (default - use the whole texture). + // Will be modified to accommodate forced symmetrical eye projection + vr::VRTextureBounds_t m_right_bounds{0.0f, 0.0f, 1.0f, 1.0f}; + vr::VRTextureBounds_t m_left_bounds{0.0f, 0.0f, 1.0f, 1.0f}; + void on_config_load(const utility::Config& cfg, bool set_defaults) override; void on_config_save(utility::Config& cfg) override; @@ -577,6 +594,18 @@ class VR : public Mod { return m_extreme_compat_mode->value(); } + auto get_horizontal_projection_override() const { + return m_horizontal_projection_override->value(); + } + + auto get_vertical_projection_override() const { + return m_vertical_projection_override->value(); + } + + bool should_grow_rectangle_for_projection_cropping() const { + return m_grow_rectangle_for_projection_cropping->value(); + } + vrmod::D3D11Component& d3d11() { return m_d3d11; } @@ -671,9 +700,6 @@ class VR : public Mod { std::vector m_controllers{}; std::unordered_set m_controllers_set{}; - vr::VRTextureBounds_t m_right_bounds{ 0.0f, 0.0f, 1.0f, 1.0f }; - vr::VRTextureBounds_t m_left_bounds{ 0.0f, 0.0f, 1.0f, 1.0f }; - glm::vec3 m_overlay_rotation{-1.550f, 0.0f, -1.330f}; glm::vec4 m_overlay_position{0.0f, 0.06f, -0.07f, 1.0f}; @@ -797,6 +823,18 @@ class VR : public Mod { "Gesture (Head) + Right Joystick", }; + static const inline std::vector s_horizontal_projection_override_names{ + "Raw / default", + "Symmetrical", + "Mirrored", + }; + + static const inline std::vector s_vertical_projection_override_names{ + "Raw / default", + "Symmetrical", + "Matched", + }; + const ModCombo::Ptr m_rendering_method{ ModCombo::create(generate_name("RenderingMethod"), s_rendering_method_names) }; const ModCombo::Ptr m_synced_afr_method{ ModCombo::create(generate_name("SyncedSequentialMethod"), s_synced_afr_method_names, 1) }; const ModToggle::Ptr m_extreme_compat_mode{ ModToggle::create(generate_name("ExtremeCompatibilityMode"), false, true) }; @@ -814,6 +852,9 @@ class VR : public Mod { const ModToggle::Ptr m_2d_screen_mode{ ModToggle::create(generate_name("2DScreenMode"), false) }; const ModToggle::Ptr m_roomscale_movement{ ModToggle::create(generate_name("RoomscaleMovement"), false) }; const ModToggle::Ptr m_swap_controllers{ ModToggle::create(generate_name("SwapControllerInputs"), false) }; + const ModCombo::Ptr m_horizontal_projection_override{ModCombo::create(generate_name("HorizontalProjectionOverride"), s_horizontal_projection_override_names)}; + const ModCombo::Ptr m_vertical_projection_override{ModCombo::create(generate_name("VerticalProjectionOverride"), s_vertical_projection_override_names)}; + const ModToggle::Ptr m_grow_rectangle_for_projection_cropping{ModToggle::create(generate_name("GrowRectangleForProjectionCropping"), false)}; // Snap turn settings and globals void gamepad_snapturn(XINPUT_STATE& state); @@ -952,6 +993,9 @@ class VR : public Mod { *m_2d_screen_mode, *m_roomscale_movement, *m_swap_controllers, + *m_horizontal_projection_override, + *m_vertical_projection_override, + *m_grow_rectangle_for_projection_cropping, *m_snapturn, *m_snapturn_joystick_deadzone, *m_snapturn_angle, @@ -1134,4 +1178,4 @@ class VR : public Mod { friend class vrmod::D3D12Component; friend class vrmod::OverlayComponent; friend class FFakeStereoRenderingHook; -}; \ No newline at end of file +}; diff --git a/src/mods/vr/D3D11Component.cpp b/src/mods/vr/D3D11Component.cpp index f74e3b40..27491c63 100644 --- a/src/mods/vr/D3D11Component.cpp +++ b/src/mods/vr/D3D11Component.cpp @@ -559,8 +559,9 @@ vr::EVRCompositorError D3D11Component::on_frame(VR* vr) { (void*)m_left_eye_tex.Get(), vr::TextureType_DirectX, vr::ColorSpace_Auto, submit_pose }; - - const auto e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &vr->m_left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); + const auto left_bounds = vr::VRTextureBounds_t{runtime->view_bounds[0][0], runtime->view_bounds[0][2], + runtime->view_bounds[0][1], runtime->view_bounds[0][3]}; + const auto e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); if (e != vr::VRCompositorError_None) { spdlog::error("[VR] VRCompositor failed to submit left eye: {}", (int)e); @@ -710,8 +711,9 @@ vr::EVRCompositorError D3D11Component::on_frame(VR* vr) { (void*)m_left_eye_tex.Get(), vr::TextureType_DirectX, vr::ColorSpace_Auto, submit_pose }; - - e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &vr->m_left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); + const auto left_bounds = vr::VRTextureBounds_t{runtime->view_bounds[0][0], runtime->view_bounds[0][2], + runtime->view_bounds[0][1], runtime->view_bounds[0][3]}; + e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); if (e != vr::VRCompositorError_None) { spdlog::error("[VR] VRCompositor failed to submit left eye: {}", (int)e); @@ -760,8 +762,9 @@ vr::EVRCompositorError D3D11Component::on_frame(VR* vr) { (void*)m_right_eye_tex.Get(), vr::TextureType_DirectX, vr::ColorSpace_Auto, submit_pose }; - - e = vr::VRCompositor()->Submit(vr::Eye_Right, &right_eye, &vr->m_right_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); + const auto right_bounds = vr::VRTextureBounds_t{runtime->view_bounds[1][0], runtime->view_bounds[1][2], + runtime->view_bounds[1][1], runtime->view_bounds[1][3]}; + e = vr::VRCompositor()->Submit(vr::Eye_Right, &right_eye, &right_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); runtime->frame_synced = false; bool submitted = true; diff --git a/src/mods/vr/D3D12Component.cpp b/src/mods/vr/D3D12Component.cpp index e2bacc9d..add08453 100644 --- a/src/mods/vr/D3D12Component.cpp +++ b/src/mods/vr/D3D12Component.cpp @@ -398,8 +398,9 @@ vr::EVRCompositorError D3D12Component::on_frame(VR* vr) { (void*)&left, vr::TextureType_DirectX12, vr::ColorSpace_Auto, submit_pose }; - - auto e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &vr->m_left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); + const auto left_bounds = vr::VRTextureBounds_t{runtime->view_bounds[0][0], runtime->view_bounds[0][2], + runtime->view_bounds[0][1], runtime->view_bounds[0][3]}; + auto e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); if (e != vr::VRCompositorError_None) { spdlog::error("[VR] VRCompositor failed to submit left eye: {}", (int)e); @@ -496,8 +497,9 @@ vr::EVRCompositorError D3D12Component::on_frame(VR* vr) { (void*)&left, vr::TextureType_DirectX12, vr::ColorSpace_Auto, submit_pose }; - - auto e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &vr->m_left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); + const auto left_bounds = vr::VRTextureBounds_t{runtime->view_bounds[0][0], runtime->view_bounds[0][2], + runtime->view_bounds[0][1], runtime->view_bounds[0][3]}; + auto e = vr::VRCompositor()->Submit(vr::Eye_Left, &left_eye, &left_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); if (e != vr::VRCompositorError_None) { spdlog::error("[VR] VRCompositor failed to submit left eye: {}", (int)e); @@ -521,8 +523,9 @@ vr::EVRCompositorError D3D12Component::on_frame(VR* vr) { (void*)&right, vr::TextureType_DirectX12, vr::ColorSpace_Auto, submit_pose }; - - auto e = vr::VRCompositor()->Submit(vr::Eye_Right, &right_eye, &vr->m_right_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); + const auto right_bounds = vr::VRTextureBounds_t{runtime->view_bounds[1][0], runtime->view_bounds[1][2], + runtime->view_bounds[1][1], runtime->view_bounds[1][3]}; + auto e = vr::VRCompositor()->Submit(vr::Eye_Right, &right_eye, &right_bounds, vr::EVRSubmitFlags::Submit_TextureWithPose); runtime->frame_synced = false; if (e != vr::VRCompositorError_None) { diff --git a/src/mods/vr/OverlayComponent.cpp b/src/mods/vr/OverlayComponent.cpp index 599e6102..e4e9da80 100644 --- a/src/mods/vr/OverlayComponent.cpp +++ b/src/mods/vr/OverlayComponent.cpp @@ -327,6 +327,8 @@ void OverlayComponent::update_slate_openvr() { const auto is_d3d11 = g_framework->get_renderer_type() == Framework::RendererType::D3D11; + // TODO: do the sizing / scaling calculations below need to take into account non-standard VRTextureBounds_t + // when we force a symmetrical eye projection matrix? vr::VRTextureBounds_t bounds{}; bounds.uMin = 0.0f; bounds.uMax = 1.0f; @@ -424,6 +426,8 @@ bool OverlayComponent::update_wrist_overlay_openvr() { // so it doesn't become too intrusive during gameplay const auto scale = m_closed_ui ? 0.25f : 1.0f; + // TODO: do the sizing / scaling calculations below need to take into account non-standard VRTextureBounds_t + // when we force a symmetrical eye projection matrix? vr::VRTextureBounds_t bounds{}; bounds.uMin = last_window_pos.x / render_target_width ; bounds.uMax = (last_window_pos.x + last_window_size.x) / render_target_width; @@ -667,6 +671,8 @@ void OverlayComponent::update_overlay_openvr() { vr::VROverlay()->ShowOverlay(m_overlay_handle); // Show the entire texture + // TODO: do the sizing / scaling calculations below need to take into account non-standard VRTextureBounds_t + // when we force a symmetrical eye projection matrix? vr::VRTextureBounds_t bounds{}; bounds.uMin = 0.0f; bounds.uMax = 1.0f; diff --git a/src/mods/vr/runtimes/OpenVR.cpp b/src/mods/vr/runtimes/OpenVR.cpp index b1b2a8c5..4777562a 100644 --- a/src/mods/vr/runtimes/OpenVR.cpp +++ b/src/mods/vr/runtimes/OpenVR.cpp @@ -20,8 +20,8 @@ VRRuntime::Error OpenVR::synchronize_frame(std::optional frame_count) this->got_first_valid_poses = true; this->got_first_sync = true; this->frame_synced = true; + this->should_update_eye_matrices = true; } - return (VRRuntime::Error)ret; } @@ -136,11 +136,11 @@ VRRuntime::Error OpenVR::update_render_target_size() { } uint32_t OpenVR::get_width() const { - return this->w; + return this->w * eye_width_adjustment; } uint32_t OpenVR::get_height() const { - return this->h; + return this->h * eye_height_adjustment; } VRRuntime::Error OpenVR::consume_events(std::function callback) { @@ -178,7 +178,14 @@ VRRuntime::Error OpenVR::consume_events(std::function callback) { return VRRuntime::Error::SUCCESS; } -VRRuntime::Error OpenVR::update_matrices(float nearz, float farz){ +VRRuntime::Error OpenVR::update_matrices(float nearz, float farz) { + // exit immediately if we've updated the eye matrices since the last frame sync, so we only do this + // operation once per sync + if (!this->should_update_eye_matrices) { + return VRRuntime::Error::SUCCESS; + } + + // always update the pose: std::unique_lock __{ this->eyes_mtx }; const auto local_left = this->hmd->GetEyeToHeadTransform(vr::Eye_Left); const auto local_right = this->hmd->GetEyeToHeadTransform(vr::Eye_Right); @@ -186,20 +193,63 @@ VRRuntime::Error OpenVR::update_matrices(float nearz, float farz){ this->eyes[vr::Eye_Left] = glm::rowMajor4(Matrix4x4f{ *(Matrix3x4f*)&local_left } ); this->eyes[vr::Eye_Right] = glm::rowMajor4(Matrix4x4f{ *(Matrix3x4f*)&local_right } ); - //auto pleft = this->hmd->GetProjectionMatrix(vr::Eye_Left, nearz, farz); - //auto pright = this->hmd->GetProjectionMatrix(vr::Eye_Right, nearz, farz); + auto get_mat = [&](vr::EVREye eye) { + const auto& vr = VR::get(); + std::array tan_half_fov{}; + + if (vr->get_horizontal_projection_override() == VR::HORIZONTAL_PROJECTION_OVERRIDE::HORIZONTAL_SYMMETRIC) { + tan_half_fov[0] = std::max(std::max(-this->raw_projections[0][0], this->raw_projections[0][1]), + std::max(-this->raw_projections[1][0], this->raw_projections[1][1])); + tan_half_fov[1] = -tan_half_fov[0]; + } else if (vr->get_horizontal_projection_override() == VR::HORIZONTAL_PROJECTION_OVERRIDE::HORIZONTAL_MIRROR) { + const auto max_outer = std::max(-this->raw_projections[0][0], this->raw_projections[1][1]); + const auto max_inner = std::max(this->raw_projections[0][1], -this->raw_projections[1][0]); + tan_half_fov[0] = eye == 0 ? max_outer : max_inner; + tan_half_fov[1] = eye == 0 ? -max_inner : -max_outer; + } else { + tan_half_fov[0] = -this->raw_projections[eye][0]; + tan_half_fov[1] = -this->raw_projections[eye][1]; + } - //this->projections[vr::Eye_Left] = glm::rowMajor4(Matrix4x4f{ *(Matrix4x4f*)&pleft } ); - //this->projections[vr::Eye_Right] = glm::rowMajor4(Matrix4x4f{ *(Matrix4x4f*)&pright } ); + if (vr->get_vertical_projection_override() == VR::VERTICAL_PROJECTION_OVERRIDE::VERTICAL_SYMMETRIC) { + tan_half_fov[2] = std::max(std::max(-this->raw_projections[0][2], this->raw_projections[0][3]), + std::max(-this->raw_projections[1][2], this->raw_projections[1][3])); + tan_half_fov[3] = -tan_half_fov[2]; + } else if (vr->get_vertical_projection_override() == VR::VERTICAL_PROJECTION_OVERRIDE::VERTICAL_MATCHED) { + + tan_half_fov[2] = std::max(-this->raw_projections[0][2], -this->raw_projections[1][2]); + tan_half_fov[3] = -std::max(this->raw_projections[0][3], this->raw_projections[1][3]); + } else { + tan_half_fov[2] = -this->raw_projections[eye][2]; + tan_half_fov[3] = -this->raw_projections[eye][3]; + } + view_bounds[eye][0] = 0.5f + 0.5f * this->raw_projections[eye][0] / tan_half_fov[0]; + view_bounds[eye][1] = 0.5f - 0.5f * this->raw_projections[eye][1] / tan_half_fov[1]; + view_bounds[eye][2] = 0.5f + 0.5f * this->raw_projections[eye][2] / tan_half_fov[2]; + view_bounds[eye][3] = 0.5f - 0.5f * this->raw_projections[eye][3] / tan_half_fov[3]; + + // if we've derived the right eye, we have up to date view bounds for both so adjust the render target if necessary + if (eye == 1) { + if (vr->should_grow_rectangle_for_projection_cropping()) { + eye_width_adjustment = 1 / std::max(view_bounds[0][1] - view_bounds[0][0], view_bounds[1][1] - view_bounds[1][0]); + eye_height_adjustment = 1 / std::max(view_bounds[0][3] - view_bounds[0][2], view_bounds[1][3] - view_bounds[1][2]); + } else { + eye_width_adjustment = 1; + eye_height_adjustment = 1; + } + SPDLOG_INFO("Eye texture proportion scale: {} by {}", eye_width_adjustment, eye_height_adjustment); + } - this->hmd->GetProjectionRaw(vr::Eye_Left, &this->raw_projections[vr::Eye_Left][0], &this->raw_projections[vr::Eye_Left][1], &this->raw_projections[vr::Eye_Left][2], &this->raw_projections[vr::Eye_Left][3]); - this->hmd->GetProjectionRaw(vr::Eye_Right, &this->raw_projections[vr::Eye_Right][0], &this->raw_projections[vr::Eye_Right][1], &this->raw_projections[vr::Eye_Right][2], &this->raw_projections[vr::Eye_Right][3]); + const auto left = tan_half_fov[0]; + const auto right = tan_half_fov[1]; + const auto top = tan_half_fov[2]; + const auto bottom = tan_half_fov[3]; - auto get_mat = [&](vr::EVREye eye) { - const auto left = this->raw_projections[eye][0] * -1.0f; - const auto right = this->raw_projections[eye][1] * -1.0f; - const auto top = this->raw_projections[eye][2] * -1.0f; - const auto bottom = this->raw_projections[eye][3] * -1.0f; + // signs : at this point we expect right [1] and bottom [3] to be negative + SPDLOG_INFO("Original FOV for {} eye: {}, {}, {}, {}", eye == 0 ? "left" : "right", -this->raw_projections[eye][0], -this->raw_projections[eye][1], + -this->raw_projections[eye][2], -this->raw_projections[eye][3]); + SPDLOG_INFO("Derived FOV for {} eye: {}, {}, {}, {}", eye == 0 ? "left" : "right", left, right, top, bottom); + SPDLOG_INFO("Derived texture bounds {} eye: {}, {}, {}, {}", eye == 0 ? "left" : "right", view_bounds[eye][0], view_bounds[eye][1], view_bounds[eye][2], view_bounds[eye][3]); float sum_rl = (left + right); float sum_tb = (top + bottom); float inv_rl = (1.0f / (left - right)); @@ -212,10 +262,18 @@ VRRuntime::Error OpenVR::update_matrices(float nearz, float farz){ 0.0f, 0.0f, nearz, 0.0f }; }; - - this->projections[vr::Eye_Left] = get_mat(vr::Eye_Left); - this->projections[vr::Eye_Right] = get_mat(vr::Eye_Right); - + // if we've not yet derived an eye projection matrix, or we've changed the projection, derive it here + // Hacky way to check for an uninitialised eye matrix - is there something better, is this necessary? + if (this->should_recalculate_eye_projections || this->last_eye_matrix_nearz != nearz || this->projections[vr::Eye_Left][2][3] == 0) { + this->hmd->GetProjectionRaw(vr::Eye_Left, &this->raw_projections[vr::Eye_Left][0], &this->raw_projections[vr::Eye_Left][1], &this->raw_projections[vr::Eye_Left][2], &this->raw_projections[vr::Eye_Left][3]); + this->hmd->GetProjectionRaw(vr::Eye_Right, &this->raw_projections[vr::Eye_Right][0], &this->raw_projections[vr::Eye_Right][1], &this->raw_projections[vr::Eye_Right][2], &this->raw_projections[vr::Eye_Right][3]); + this->projections[vr::Eye_Left] = get_mat(vr::Eye_Left); + this->projections[vr::Eye_Right] = get_mat(vr::Eye_Right); + this->should_recalculate_eye_projections = false; + this->last_eye_matrix_nearz = nearz; + } + // don't allow the eye matrices to be derived again until after the next frame sync + this->should_update_eye_matrices = false; return VRRuntime::Error::SUCCESS; } diff --git a/src/mods/vr/runtimes/OpenXR.cpp b/src/mods/vr/runtimes/OpenXR.cpp index 99a0996f..02602e0e 100644 --- a/src/mods/vr/runtimes/OpenXR.cpp +++ b/src/mods/vr/runtimes/OpenXR.cpp @@ -185,8 +185,8 @@ VRRuntime::Error OpenXR::synchronize_frame(std::optional frame_count) this->got_first_sync = true; this->frame_synced = true; + this->should_update_eye_matrices = true; } - return VRRuntime::Error::SUCCESS; } @@ -382,16 +382,14 @@ uint32_t OpenXR::get_width() const { if (this->view_configs.empty()) { return 0; } - - return (uint32_t)((float)this->view_configs[0].recommendedImageRectWidth * this->resolution_scale->value()); + return (uint32_t)((float)this->view_configs[0].recommendedImageRectWidth * this->resolution_scale->value() * eye_width_adjustment); } uint32_t OpenXR::get_height() const { if (this->view_configs.empty()) { return 0; } - - return (uint32_t)((float)this->view_configs[0].recommendedImageRectHeight * this->resolution_scale->value()); + return (uint32_t)((float)this->view_configs[0].recommendedImageRectHeight * this->resolution_scale->value() * eye_height_adjustment); } VRRuntime::Error OpenXR::consume_events(std::function callback) { @@ -465,46 +463,118 @@ VRRuntime::Error OpenXR::consume_events(std::function callback) { } VRRuntime::Error OpenXR::update_matrices(float nearz, float farz) { + // exit immediately if we've updated the eye matrices since the last frame sync, so we only do this + // operation once per sync + if (!this->should_update_eye_matrices) { + return VRRuntime::Error::SUCCESS; + } + if (!this->session_ready || this->views.empty()) { return VRRuntime::Error::SUCCESS; } - std::unique_lock __{ this->eyes_mtx }; + // always update the pose: std::unique_lock ___{ this->pose_mtx }; + const auto& left_pose = this->views[0].pose; + const auto& right_pose = this->views[1].pose; + this->eyes[0] = Matrix4x4f{OpenXR::to_glm(left_pose.orientation)}; + this->eyes[0][3] = Vector4f{*(Vector3f*)&left_pose.position, 1.0f}; + this->eyes[1] = Matrix4x4f{OpenXR::to_glm(right_pose.orientation)}; + this->eyes[1][3] = Vector4f{*(Vector3f*)&right_pose.position, 1.0f}; + + auto get_mat = [&](int eye) { + const auto& vr = VR::get(); + std::array tan_half_fov{}; + + if (vr->get_horizontal_projection_override() == VR::HORIZONTAL_PROJECTION_OVERRIDE::HORIZONTAL_SYMMETRIC) { + tan_half_fov[0] = -std::max(std::max(-this->raw_projections[0][0], this->raw_projections[0][1]), + std::max(-this->raw_projections[1][0], this->raw_projections[1][1])); + tan_half_fov[1] = -tan_half_fov[0]; + } else if (vr->get_horizontal_projection_override() == VR::HORIZONTAL_PROJECTION_OVERRIDE::HORIZONTAL_MIRROR) { + float max_outer = std::max(-this->raw_projections[0][0], this->raw_projections[1][1]); + float max_inner = std::max(this->raw_projections[0][1], -this->raw_projections[1][0]); + tan_half_fov[0] = eye == 0 ? -max_outer : -max_inner; + tan_half_fov[1] = eye == 0 ? max_inner : max_outer; + } else { + tan_half_fov[0] = this->raw_projections[eye][0]; + tan_half_fov[1] = this->raw_projections[eye][1]; + } - for (auto i = 0; i < 2; ++i) { - const auto& pose = this->views[i].pose; - const auto& fov = this->views[i].fov; - - // Update projection matrix - //XrMatrix4x4f_CreateProjection((XrMatrix4x4f*)&this->projections[i], GRAPHICS_D3D, tan(fov.angleLeft), tan(fov.angleRight), tan(fov.angleUp), tan(fov.angleDown), nearz, farz); - - auto get_mat = [&](int eye) { - const auto top = tan(fov.angleUp); - const auto bottom = tan(fov.angleDown); - const auto left = tan(fov.angleLeft); - const auto right = tan(fov.angleRight); - - float sum_rl = (right + left); - float sum_tb = (top + bottom); - float inv_rl = (1.0f / (right - left)); - float inv_tb = (1.0f / (top - bottom)); - - return Matrix4x4f { - (2.0f * inv_rl), 0.0f, 0.0f, 0.0f, - 0.0f, (2.0f * inv_tb), 0.0f, 0.0f, - (sum_rl * -inv_rl), (sum_tb * -inv_tb), 0.0f, 1.0f, - 0.0f, 0.0f, nearz, 0.0f - }; - }; - - this->projections[i] = get_mat(i); + if (vr->get_vertical_projection_override() == VR::VERTICAL_PROJECTION_OVERRIDE::VERTICAL_SYMMETRIC) { + tan_half_fov[2] = std::max(std::max(this->raw_projections[0][2], -this->raw_projections[0][3]), + std::max(this->raw_projections[1][2], -this->raw_projections[1][3])); + tan_half_fov[3] = -tan_half_fov[2]; + } else if (vr->get_vertical_projection_override() == VR::VERTICAL_PROJECTION_OVERRIDE::VERTICAL_MATCHED) { + float max_top = std::max(this->raw_projections[0][2], this->raw_projections[1][2]); + float max_bottom = std::max(-this->raw_projections[0][3], -this->raw_projections[1][3]); + tan_half_fov[2] = max_top; + tan_half_fov[3] = -max_bottom; + } else { + tan_half_fov[2] = this->raw_projections[eye][2]; + tan_half_fov[3] = this->raw_projections[eye][3]; + } + view_bounds[eye][0] = 0.5f - 0.5f * this->raw_projections[eye][0] / tan_half_fov[0]; + view_bounds[eye][1] = 0.5f + 0.5f * this->raw_projections[eye][1] / tan_half_fov[1]; + view_bounds[eye][2] = 0.5f - 0.5f * this->raw_projections[eye][2] / tan_half_fov[2]; + view_bounds[eye][3] = 0.5f + 0.5f * this->raw_projections[eye][3] / tan_half_fov[3]; + + // if we've derived the right eye, we have up to date view bounds for both so adjust the render target if necessary + if (eye == 1) { + if (vr->should_grow_rectangle_for_projection_cropping()) { + eye_width_adjustment = 1 / std::max(view_bounds[0][1] - view_bounds[0][0], view_bounds[1][1] - view_bounds[1][0]); + eye_height_adjustment = 1 / std::max(view_bounds[0][3] - view_bounds[0][2], view_bounds[1][3] - view_bounds[1][2]); + } else { + eye_width_adjustment = 1; + eye_height_adjustment = 1; + } + SPDLOG_INFO("Eye texture proportion scale: {} by {}", eye_width_adjustment, eye_height_adjustment); + } - // Update view matrix - this->eyes[i] = Matrix4x4f{OpenXR::to_glm(pose.orientation)}; - this->eyes[i][3] = Vector4f{*(Vector3f*)&pose.position, 1.0f}; - } + const auto left = tan_half_fov[0]; + const auto right = tan_half_fov[1]; + const auto top = tan_half_fov[2]; + const auto bottom = tan_half_fov[3]; + + // signs: at this point we expect left[0] and bottom[3] to be negative + SPDLOG_INFO("Original FOV for {} eye: {}, {}, {}, {}", eye == 0 ? "left" : "right", this->raw_projections[eye][0], this->raw_projections[eye][1], + this->raw_projections[eye][2], this->raw_projections[eye][3]); + SPDLOG_INFO("Derived FOV for {} eye: {}, {}, {}, {}", eye == 0 ? "left" : "right", left, right, top, bottom); + SPDLOG_INFO("Derived texture bounds {} eye: {}, {}, {}, {}", eye == 0 ? "left" : "right", view_bounds[eye][0], view_bounds[eye][1], view_bounds[eye][2], view_bounds[eye][3]); + float sum_rl = (right + left); + float sum_tb = (top + bottom); + float inv_rl = (1.0f / (right - left)); + float inv_tb = (1.0f / (top - bottom)); + + return Matrix4x4f { + (2.0f * inv_rl), 0.0f, 0.0f, 0.0f, + 0.0f, (2.0f * inv_tb), 0.0f, 0.0f, + (sum_rl * -inv_rl), (sum_tb * -inv_tb), 0.0f, 1.0f, + 0.0f, 0.0f, nearz, 0.0f + }; + }; + // if we've not yet derived an eye projection matrix, or we've changed the projection, derive it here + // Hacky way to check for an uninitialised eye matrix - is there something better, is this necessary? + if (this->should_recalculate_eye_projections || this->last_eye_matrix_nearz != nearz || this->projections[0][2][3] == 0) { + // deriving the texture bounds when modifying projections requires left and right raw projections so get them all before we start: + std::unique_lock __{this->eyes_mtx}; + const auto& left_fov = this->views[0].fov; + this->raw_projections[0][0] = tan(left_fov.angleLeft); + this->raw_projections[0][1] = tan(left_fov.angleRight); + this->raw_projections[0][2] = tan(left_fov.angleUp); + this->raw_projections[0][3] = tan(left_fov.angleDown); + const auto& right_fov = this->views[1].fov; + this->raw_projections[1][0] = tan(right_fov.angleLeft); + this->raw_projections[1][1] = tan(right_fov.angleRight); + this->raw_projections[1][2] = tan(right_fov.angleUp); + this->raw_projections[1][3] = tan(right_fov.angleDown); + this->projections[0] = get_mat(0); + this->projections[1] = get_mat(1); + this->should_recalculate_eye_projections = false; + this->last_eye_matrix_nearz = nearz; + } + // don't allow the eye matrices to be derived again until after the next frame sync + this->should_update_eye_matrices = false; return VRRuntime::Error::SUCCESS; } @@ -1721,7 +1791,7 @@ XrResult OpenXR::end_frame(const std::vector& qua projection_layer_views.resize(pipelined_stage_views.size(), {XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}); depth_layers.resize(projection_layer_views.size(), {XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR}); - for (auto i = 0; i < projection_layer_views.size(); ++i) { + for (auto i = 0; i < projection_layer_views.size(); ++i) { Swapchain* swapchain = nullptr; if (is_afr) { @@ -1739,13 +1809,23 @@ XrResult OpenXR::end_frame(const std::vector& qua projection_layer_views[i].fov = pipelined_stage_views[i].fov; projection_layer_views[i].subImage.swapchain = swapchain->handle; - if (is_afr) { - projection_layer_views[i].subImage.imageRect.offset = {0, 0}; - projection_layer_views[i].subImage.imageRect.extent = {swapchain->width, swapchain->height}; + int32_t offset_x = 0, offset_y = 0, extent_x = 0, extent_y = 0; + // if we're working with a double-wide texture, use half the view bounds adjustment (as they apply to a single eye) + int texture_area_width = is_afr ? swapchain->width : swapchain->width / 2; + if (is_afr || i == 0) { + offset_x = view_bounds[i][0] * texture_area_width; + extent_x = view_bounds[i][1] * texture_area_width - offset_x; } else { - projection_layer_views[i].subImage.imageRect.offset = {(swapchain->width / 2) * i, 0}; - projection_layer_views[i].subImage.imageRect.extent = {swapchain->width / 2, swapchain->height}; + // right eye double-wide + offset_x = texture_area_width + view_bounds[i][0] * texture_area_width; + extent_x = view_bounds[i][1] * texture_area_width - (offset_x - texture_area_width); } + offset_y = view_bounds[i][2] * swapchain->height; + extent_y = view_bounds[i][3] * swapchain->height - offset_y; + + // SPDLOG_INFO("image calc for eye {} {}, {}, {}, {}", i, offset_x, extent_x, offset_y, extent_y); + projection_layer_views[i].subImage.imageRect.offset = {offset_x, offset_y}; + projection_layer_views[i].subImage.imageRect.extent = {extent_x, extent_y}; if (has_depth) { Swapchain* depth_swapchain = nullptr; @@ -1764,30 +1844,8 @@ XrResult OpenXR::end_frame(const std::vector& qua depth_layers[i].next = nullptr; depth_layers[i].subImage.swapchain = depth_swapchain->handle; - const auto is_afr = VR::get()->is_using_afr(); - bool doublewide_depth = true; - - if (is_afr /*&& depth_swapchain->width == get_width() && depth_swapchain->height == get_height()*/) { - doublewide_depth = false; - } - - XrExtent2Di depth_extent = {depth_swapchain->width, depth_swapchain->height}; - - if (doublewide_depth) { - depth_extent.width /= 2; - } - - depth_extent = {std::min(get_width(), depth_extent.width), std::min(get_height(), depth_extent.height)}; - - if (is_afr) { - // Always the left half of the depth texture. - depth_layers[i].subImage.imageRect.offset = {0, 0}; - depth_layers[i].subImage.imageRect.extent = depth_extent; - } else { - depth_layers[i].subImage.imageRect.offset = {depth_extent.width * i, 0}; - depth_layers[i].subImage.imageRect.extent = depth_extent; - } - + depth_layers[i].subImage.imageRect.offset = {offset_x, offset_y}; + depth_layers[i].subImage.imageRect.extent = {std::min(get_width(), extent_x), std::min(get_height(), extent_y)}; depth_layers[i].minDepth = 0.0f; depth_layers[i].maxDepth = 1.0f; auto wtm = VR::get()->get_world_to_meters(); diff --git a/src/mods/vr/runtimes/OpenXR.hpp b/src/mods/vr/runtimes/OpenXR.hpp index 2ad3d60e..929923a4 100644 --- a/src/mods/vr/runtimes/OpenXR.hpp +++ b/src/mods/vr/runtimes/OpenXR.hpp @@ -485,5 +485,6 @@ struct OpenXR final : public VRRuntime { "/interaction_profiles/microsoft/motion_controller", "/interaction_profiles/htc/vive_controller", }; + }; } \ No newline at end of file diff --git a/src/mods/vr/runtimes/VRRuntime.hpp b/src/mods/vr/runtimes/VRRuntime.hpp index 0ebddec2..f773c1e7 100644 --- a/src/mods/vr/runtimes/VRRuntime.hpp +++ b/src/mods/vr/runtimes/VRRuntime.hpp @@ -193,4 +193,18 @@ struct VRRuntime { uint32_t internal_frame_count{}; uint32_t internal_render_frame_count{}; bool has_render_frame_count{false}; + + // view bounds proportions - left xmin, xmax, ymin, ymax then right xmin, xmax, ymin, ymax + // used to crop the rendered eye textures to account for projection adjustments + float view_bounds[2][4] = {0, 1, 0, 1, 0, 1, 0, 1}; + + float last_eye_matrix_nearz = 0.01f; + bool should_update_eye_matrices{true}; + bool should_recalculate_eye_projections{false}; + bool is_modifying_eye_texture_scale{false}; + + // factor to scale the recommended eye texture size where we're cropping due to projection overrides, but + // want to retain the final eye texture resolution + float eye_width_adjustment{1}; + float eye_height_adjustment{1}; }; \ No newline at end of file