From eda7d955f88c3ddf371c5c1b28f13da05ab8035f Mon Sep 17 00:00:00 2001 From: Imanol Fernandez Date: Wed, 20 May 2020 01:19:47 +0200 Subject: [PATCH] Add support for lay-down mode in VR videos (#3232) Co-authored-by: Randall E. Barker --- .../mozilla/vrbrowser/VRBrowserActivity.java | 8 ++--- .../ui/widgets/NavigationBarWidget.java | 5 ++- .../ui/widgets/WidgetManagerDelegate.java | 7 +++- app/src/main/cpp/BrowserWorld.cpp | 36 +++++++++++++------ app/src/main/cpp/BrowserWorld.h | 3 +- app/src/main/cpp/VRVideo.cpp | 9 +++-- app/src/main/cpp/VRVideo.h | 1 + app/src/oculusvr/cpp/OculusVRLayers.cpp | 1 + 8 files changed, 49 insertions(+), 21 deletions(-) diff --git a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java index ec6d29476..d8fe7ba51 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java @@ -1080,7 +1080,7 @@ void onExitWebXR(long aCallback) { }); // Show the window in front of you when you exit immersive mode. - resetUIYaw(); + recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL); TelemetryWrapper.uploadImmersiveToHistogram(); GleanMetricsService.stopImmersive(); @@ -1646,8 +1646,8 @@ public void hideVRVideo() { } @Override - public void resetUIYaw() { - queueRunnable(this::resetUIYawNative); + public void recenterUIYaw(@YawTarget int aTarget) { + queueRunnable(() -> recenterUIYawNative(aTarget)); } @Override @@ -1742,7 +1742,7 @@ public AppServicesProvider getServicesProvider() { private native void updatePointerColorNative(); private native void showVRVideoNative(int aWindowHandler, int aVideoProjection); private native void hideVRVideoNative(); - private native void resetUIYawNative(); + private native void recenterUIYawNative(@YawTarget int aTarget); private native void setControllersVisibleNative(boolean aVisible); private native void runCallbackNative(long aCallback); private native void setCylinderDensityNative(float aDensity); diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java index 2b489e14e..1e3d92079 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java @@ -11,7 +11,6 @@ import android.graphics.Canvas; import android.graphics.Rect; import android.preference.PreferenceManager; -import android.text.Spannable; import android.util.AttributeSet; import android.util.Log; import android.util.Pair; @@ -828,7 +827,7 @@ private void exitVRVideo() { mMediaControlsWidget.setVisible(false); // Reposition UI in front of the user when exiting a VR video. - mWidgetManager.resetUIYaw(); + mWidgetManager.recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL); } private void setResizePreset(float aMultiplier) { @@ -1049,7 +1048,7 @@ public void onWorldClick() { if (mMediaControlsWidget.isVisible()) { // Reorient the MediaControl UI when the users clicks to show it. // So you can look at any point of the 180/360 video and the UI always shows in front of you. - mWidgetManager.resetUIYaw(); + mWidgetManager.recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_WIDGETS); } } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WidgetManagerDelegate.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WidgetManagerDelegate.java index 9b7b5d69e..86a04f8a9 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WidgetManagerDelegate.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WidgetManagerDelegate.java @@ -56,6 +56,11 @@ interface WebXRListener { int WEBXR_INTERSTITIAL_ALLOW_DISMISS = 1; int WEBXR_INTERSTITIAL_HIDDEN = 2; + @IntDef(value = { YAW_TARGET_ALL, YAW_TARGET_WIDGETS}) + @interface YawTarget {} + int YAW_TARGET_ALL = 0; // Targets widgets and VR videos. + int YAW_TARGET_WIDGETS = 1; // Targets widgets only. + int newWidgetHandle(); void addWidget(Widget aWidget); void updateWidget(Widget aWidget); @@ -80,7 +85,7 @@ interface WebXRListener { void updatePointerColor(); void showVRVideo(int aWindowHandle, @VideoProjectionMenuWidget.VideoProjectionFlags int aVideoProjection); void hideVRVideo(); - void resetUIYaw(); + void recenterUIYaw(@YawTarget int target); void setCylinderDensity(float aDensity); float getCylinderDensity(); void addFocusChangeListener(@NonNull FocusChangeListener aListener); diff --git a/app/src/main/cpp/BrowserWorld.cpp b/app/src/main/cpp/BrowserWorld.cpp index 7bb2332cf..988bd149f 100644 --- a/app/src/main/cpp/BrowserWorld.cpp +++ b/app/src/main/cpp/BrowserWorld.cpp @@ -189,6 +189,7 @@ struct BrowserWorld::State { std::function frameEndHandler; bool wasInGazeMode = false; WebXRInterstialState webXRInterstialState; + vrb::Matrix widgetsYaw; bool wasWebXRRendering = false; State() : paused(true), glInitialized(false), modelsLoaded(false), env(nullptr), cylinderDensity(0.0f), nearClip(0.1f), @@ -217,6 +218,7 @@ struct BrowserWorld::State { monitor->AddPerformanceMonitorObserver(std::make_shared()); wasInGazeMode = false; webXRInterstialState = WebXRInterstialState::FORCED; + widgetsYaw = vrb::Matrix::Identity(); #if defined(WAVEVR) monitor->SetPerformanceDelta(15.0); #endif @@ -463,7 +465,7 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) { if (hitWidget) { vrb::Matrix translation = vrb::Matrix::Translation(hitPoint); vrb::Matrix localRotation = vrb::Matrix::Rotation(hitNormal); - vrb::Matrix reorient = device->GetReorientTransform(); + vrb::Matrix reorient = rootTransparent->GetTransform(); controller.pointer->SetTransform(reorient.AfineInverse().PostMultiply(translation).PostMultiply(localRotation)); controller.pointer->SetScale(hitPoint, device->GetHeadTransform()); } @@ -1388,13 +1390,20 @@ BrowserWorld::SetControllersVisible(const bool aVisible) { } void -BrowserWorld::ResetUIYaw() { +BrowserWorld::RecenterUIYaw(const YawTarget aTarget) { vrb::Matrix head = m.device->GetHeadTransform(); - vrb::Vector vector = head.MultiplyDirection(vrb::Vector(1.0f, 0.0f, 0.0f)); - const float yaw = atan2(vector.z(), vector.x()); - vrb::Matrix matrix = vrb::Matrix::Rotation(vrb::Vector(0.0f, 1.0f, 0.0f), -yaw); - m.device->SetReorientTransform(matrix); + if (aTarget == YawTarget::ALL) { + vrb::Vector vector = head.MultiplyDirection(vrb::Vector(1.0f, 0.0f, 0.0f)); + float yaw = atan2(vector.z(), vector.x()); + vrb::Matrix matrix = vrb::Matrix::Rotation(vrb::Vector(0.0f, 1.0f, 0.0f), -yaw); + m.device->SetReorientTransform(matrix); + m.widgetsYaw = vrb::Matrix::Identity(); + } else { + vrb::Vector vector = m.device->GetReorientTransform().AfineInverse().PostMultiply(head).MultiplyDirection(vrb::Vector(1.0f, 0.0f, 0.0f)); + const float yaw = atan2(vector.z(), vector.x()); + m.widgetsYaw = vrb::Matrix::Rotation(vrb::Vector(0.0f, 1.0f, 0.0f), -yaw); + } } void @@ -1453,7 +1462,10 @@ BrowserWorld::TickWorld() { m.SortWidgets(); m.device->StartFrame(); m.rootOpaque->SetTransform(m.device->GetReorientTransform()); - m.rootTransparent->SetTransform(m.device->GetReorientTransform()); + m.rootTransparent->SetTransform(m.device->GetReorientTransform().PostMultiply(m.widgetsYaw)); + if (m.vrVideo) { + m.vrVideo->SetReorientTransform(m.device->GetReorientTransform()); + } m.drawHandler = [=](device::Eye aEye) { DrawWorld(aEye); @@ -1759,9 +1771,13 @@ JNI_METHOD(void, setControllersVisibleNative) crow::BrowserWorld::Instance().SetControllersVisible(aVisible); } -JNI_METHOD(void, resetUIYawNative) -(JNIEnv*, jobject) { - crow::BrowserWorld::Instance().ResetUIYaw(); +JNI_METHOD(void, recenterUIYawNative) +(JNIEnv*, jobject, jint aTarget) { + crow::BrowserWorld::YawTarget value = crow::BrowserWorld::YawTarget::ALL; + if (aTarget == 1) { + value = crow::BrowserWorld::YawTarget::WIDGETS; + } + crow::BrowserWorld::Instance().RecenterUIYaw(value); } JNI_METHOD(void, setCylinderDensityNative) diff --git a/app/src/main/cpp/BrowserWorld.h b/app/src/main/cpp/BrowserWorld.h index ff141900e..d0ee4b013 100644 --- a/app/src/main/cpp/BrowserWorld.h +++ b/app/src/main/cpp/BrowserWorld.h @@ -65,7 +65,8 @@ class BrowserWorld { void ShowVRVideo(const int aWindowHandle, const int aVideoProjection); void HideVRVideo(); void SetControllersVisible(const bool aVisible); - void ResetUIYaw(); + enum class YawTarget { ALL, WIDGETS }; + void RecenterUIYaw(const YawTarget aTarget); void SetCylinderDensity(const float aDensity); enum class WebXRInterstialState { FORCED, ALLOW_DISMISS, HIDDEN }; void SetWebXRInterstitalState(const WebXRInterstialState aState); diff --git a/app/src/main/cpp/VRVideo.cpp b/app/src/main/cpp/VRVideo.cpp index d7434f77b..b92bec138 100644 --- a/app/src/main/cpp/VRVideo.cpp +++ b/app/src/main/cpp/VRVideo.cpp @@ -33,7 +33,7 @@ struct VRVideo::State { std::weak_ptr deviceWeak; WidgetPtr window; VRVideoProjection projection; - vrb::TogglePtr root; + vrb::TransformPtr root; vrb::TogglePtr leftEye; vrb::TogglePtr rightEye; VRLayerPtr layer; @@ -50,7 +50,7 @@ struct VRVideo::State { vrb::CreationContextPtr create = context.lock(); window = aWindow; projection = aProjection; - root = vrb::Toggle::Create(create); + root = vrb::Transform::Create(create); VRLayerSurfacePtr windowLayer = aWindow->GetLayer(); if (windowLayer) { layerTextureBackup[0] = windowLayer->GetTextureRect(device::Eye::Left); @@ -362,6 +362,11 @@ VRVideo::Exit() { } } +void +VRVideo::SetReorientTransform(const vrb::Matrix& transform) { + m.root->SetTransform(transform); +} + VRVideoPtr VRVideo::Create(vrb::CreationContextPtr aContext, const WidgetPtr& aWindow, diff --git a/app/src/main/cpp/VRVideo.h b/app/src/main/cpp/VRVideo.h index 9d51c48d3..4556667ec 100644 --- a/app/src/main/cpp/VRVideo.h +++ b/app/src/main/cpp/VRVideo.h @@ -44,6 +44,7 @@ class VRVideo { void SelectEye(device::Eye aEye); vrb::NodePtr GetRoot() const; void Exit(); + void SetReorientTransform(const vrb::Matrix& transform); protected: struct State; VRVideo(State& aState, vrb::CreationContextPtr& aContext); diff --git a/app/src/oculusvr/cpp/OculusVRLayers.cpp b/app/src/oculusvr/cpp/OculusVRLayers.cpp index 1ab5c3ba9..ab272c58c 100644 --- a/app/src/oculusvr/cpp/OculusVRLayers.cpp +++ b/app/src/oculusvr/cpp/OculusVRLayers.cpp @@ -221,6 +221,7 @@ OculusLayerEquirect::Update(const ovrTracking2& aTracking, ovrTextureSwapChain* OculusLayerBase::Update(aTracking, aClearSwapChain); vrb::Quaternion q(layer->GetModelTransform(device::Eye::Left)); + q = q.Inverse(); ovrLayer.HeadPose.Pose.Orientation.x = q.x(); ovrLayer.HeadPose.Pose.Orientation.y = q.y(); ovrLayer.HeadPose.Pose.Orientation.z = q.z();