Add support for lay-down mode in VR videos (#3232)

Co-authored-by: Randall E. Barker <simstorm@mac.com>

app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java

@@ -1080,7 +1080,7 @@ void onExitWebXR(long aCallback) {
         });
 
         // Show the window in front of you when you exit immersive mode.
-        resetUIYaw();
+        recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL);
 
         TelemetryWrapper.uploadImmersiveToHistogram();
         GleanMetricsService.stopImmersive();
@@ -1646,8 +1646,8 @@ public void hideVRVideo() {
     }
 
     @Override
-    public void resetUIYaw() {
-        queueRunnable(this::resetUIYawNative);
+    public void recenterUIYaw(@YawTarget int aTarget) {
+        queueRunnable(() -> recenterUIYawNative(aTarget));
     }
 
     @Override
@@ -1742,7 +1742,7 @@ public AppServicesProvider getServicesProvider() {
     private native void updatePointerColorNative();
     private native void showVRVideoNative(int aWindowHandler, int aVideoProjection);
     private native void hideVRVideoNative();
-    private native void resetUIYawNative();
+    private native void recenterUIYawNative(@YawTarget int aTarget);
     private native void setControllersVisibleNative(boolean aVisible);
     private native void runCallbackNative(long aCallback);
     private native void setCylinderDensityNative(float aDensity);

app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/NavigationBarWidget.java

@@ -11,7 +11,6 @@
 import android.graphics.Canvas;
 import android.graphics.Rect;
 import android.preference.PreferenceManager;
-import android.text.Spannable;
 import android.util.AttributeSet;
 import android.util.Log;
 import android.util.Pair;
@@ -828,7 +827,7 @@ private void exitVRVideo() {
         mMediaControlsWidget.setVisible(false);
 
         // Reposition UI in front of the user when exiting a VR video.
-        mWidgetManager.resetUIYaw();
+        mWidgetManager.recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL);
     }
 
     private void setResizePreset(float aMultiplier) {
@@ -1049,7 +1048,7 @@ public void onWorldClick() {
                 if (mMediaControlsWidget.isVisible()) {
                     // Reorient the MediaControl UI when the users clicks to show it.
                     // So you can look at any point of the 180/360 video and the UI always shows in front of you.
-                    mWidgetManager.resetUIYaw();
+                    mWidgetManager.recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_WIDGETS);
                 }
             }
 

app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WidgetManagerDelegate.java

@@ -56,6 +56,11 @@ interface WebXRListener {
     int WEBXR_INTERSTITIAL_ALLOW_DISMISS = 1;
     int WEBXR_INTERSTITIAL_HIDDEN = 2;
 
+    @IntDef(value = { YAW_TARGET_ALL, YAW_TARGET_WIDGETS})
+    @interface YawTarget {}
+    int YAW_TARGET_ALL = 0; // Targets widgets and VR videos.
+    int YAW_TARGET_WIDGETS = 1; // Targets widgets only.
+
     int newWidgetHandle();
     void addWidget(Widget aWidget);
     void updateWidget(Widget aWidget);
@@ -80,7 +85,7 @@ interface WebXRListener {
     void updatePointerColor();
     void showVRVideo(int aWindowHandle, @VideoProjectionMenuWidget.VideoProjectionFlags int aVideoProjection);
     void hideVRVideo();
-    void resetUIYaw();
+    void recenterUIYaw(@YawTarget int target);
     void setCylinderDensity(float aDensity);
     float getCylinderDensity();
     void addFocusChangeListener(@NonNull FocusChangeListener aListener);

app/src/main/cpp/BrowserWorld.cpp

@@ -189,6 +189,7 @@ struct BrowserWorld::State {
   std::function<void()> frameEndHandler;
   bool wasInGazeMode = false;
   WebXRInterstialState webXRInterstialState;
+  vrb::Matrix widgetsYaw;
   bool wasWebXRRendering = false;
 
   State() : paused(true), glInitialized(false), modelsLoaded(false), env(nullptr), cylinderDensity(0.0f), nearClip(0.1f),
@@ -217,6 +218,7 @@ struct BrowserWorld::State {
     monitor->AddPerformanceMonitorObserver(std::make_shared<PerformanceObserver>());
     wasInGazeMode = false;
     webXRInterstialState = WebXRInterstialState::FORCED;
+    widgetsYaw = vrb::Matrix::Identity();
 #if defined(WAVEVR)
     monitor->SetPerformanceDelta(15.0);
 #endif
@@ -463,7 +465,7 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
       if (hitWidget) {
         vrb::Matrix translation = vrb::Matrix::Translation(hitPoint);
         vrb::Matrix localRotation = vrb::Matrix::Rotation(hitNormal);
-        vrb::Matrix reorient = device->GetReorientTransform();
+        vrb::Matrix reorient = rootTransparent->GetTransform();
         controller.pointer->SetTransform(reorient.AfineInverse().PostMultiply(translation).PostMultiply(localRotation));
         controller.pointer->SetScale(hitPoint, device->GetHeadTransform());
       }
@@ -1388,13 +1390,20 @@ BrowserWorld::SetControllersVisible(const bool aVisible) {
 }
 
 void
-BrowserWorld::ResetUIYaw() {
+BrowserWorld::RecenterUIYaw(const YawTarget aTarget) {
   vrb::Matrix head = m.device->GetHeadTransform();
-  vrb::Vector vector = head.MultiplyDirection(vrb::Vector(1.0f, 0.0f, 0.0f));
-  const float yaw = atan2(vector.z(), vector.x());
 
-  vrb::Matrix matrix = vrb::Matrix::Rotation(vrb::Vector(0.0f, 1.0f, 0.0f), -yaw);
-  m.device->SetReorientTransform(matrix);
+  if (aTarget == YawTarget::ALL) {
+    vrb::Vector vector = head.MultiplyDirection(vrb::Vector(1.0f, 0.0f, 0.0f));
+    float yaw = atan2(vector.z(), vector.x());
+    vrb::Matrix matrix = vrb::Matrix::Rotation(vrb::Vector(0.0f, 1.0f, 0.0f), -yaw);
+    m.device->SetReorientTransform(matrix);
+    m.widgetsYaw = vrb::Matrix::Identity();
+  } else {
+    vrb::Vector vector = m.device->GetReorientTransform().AfineInverse().PostMultiply(head).MultiplyDirection(vrb::Vector(1.0f, 0.0f, 0.0f));
+    const float yaw = atan2(vector.z(), vector.x());
+    m.widgetsYaw = vrb::Matrix::Rotation(vrb::Vector(0.0f, 1.0f, 0.0f), -yaw);
+  }
 }
 
 void
@@ -1453,7 +1462,10 @@ BrowserWorld::TickWorld() {
   m.SortWidgets();
   m.device->StartFrame();
   m.rootOpaque->SetTransform(m.device->GetReorientTransform());
-  m.rootTransparent->SetTransform(m.device->GetReorientTransform());
+  m.rootTransparent->SetTransform(m.device->GetReorientTransform().PostMultiply(m.widgetsYaw));
+  if (m.vrVideo) {
+    m.vrVideo->SetReorientTransform(m.device->GetReorientTransform());
+  }
 
   m.drawHandler = [=](device::Eye aEye) {
     DrawWorld(aEye);
@@ -1759,9 +1771,13 @@ JNI_METHOD(void, setControllersVisibleNative)
   crow::BrowserWorld::Instance().SetControllersVisible(aVisible);
 }
 
-JNI_METHOD(void, resetUIYawNative)
-(JNIEnv*, jobject) {
-  crow::BrowserWorld::Instance().ResetUIYaw();
+JNI_METHOD(void, recenterUIYawNative)
+(JNIEnv*, jobject, jint aTarget) {
+  crow::BrowserWorld::YawTarget value = crow::BrowserWorld::YawTarget::ALL;
+  if (aTarget == 1) {
+    value = crow::BrowserWorld::YawTarget::WIDGETS;
+  }
+  crow::BrowserWorld::Instance().RecenterUIYaw(value);
 }
 
 JNI_METHOD(void, setCylinderDensityNative)

app/src/main/cpp/BrowserWorld.h

@@ -65,7 +65,8 @@ class BrowserWorld {
   void ShowVRVideo(const int aWindowHandle, const int aVideoProjection);
   void HideVRVideo();
   void SetControllersVisible(const bool aVisible);
-  void ResetUIYaw();
+  enum class YawTarget { ALL, WIDGETS };
+  void RecenterUIYaw(const YawTarget aTarget);
   void SetCylinderDensity(const float aDensity);
   enum class WebXRInterstialState { FORCED, ALLOW_DISMISS, HIDDEN };
   void SetWebXRInterstitalState(const WebXRInterstialState aState);

app/src/main/cpp/VRVideo.cpp

@@ -33,7 +33,7 @@ struct VRVideo::State {
   std::weak_ptr<DeviceDelegate> deviceWeak;
   WidgetPtr window;
   VRVideoProjection projection;
-  vrb::TogglePtr root;
+  vrb::TransformPtr root;
   vrb::TogglePtr leftEye;
   vrb::TogglePtr rightEye;
   VRLayerPtr layer;
@@ -50,7 +50,7 @@ struct VRVideo::State {
     vrb::CreationContextPtr create = context.lock();
     window = aWindow;
     projection = aProjection;
-    root = vrb::Toggle::Create(create);
+    root = vrb::Transform::Create(create);
     VRLayerSurfacePtr windowLayer = aWindow->GetLayer();
     if (windowLayer) {
       layerTextureBackup[0] = windowLayer->GetTextureRect(device::Eye::Left);
@@ -362,6 +362,11 @@ VRVideo::Exit() {
   }
 }
 
+void
+VRVideo::SetReorientTransform(const vrb::Matrix& transform) {
+  m.root->SetTransform(transform);
+}
+
 VRVideoPtr
 VRVideo::Create(vrb::CreationContextPtr aContext,
                 const WidgetPtr& aWindow,

app/src/main/cpp/VRVideo.h

@@ -44,6 +44,7 @@ class VRVideo {
   void SelectEye(device::Eye aEye);
   vrb::NodePtr GetRoot() const;
   void Exit();
+  void SetReorientTransform(const vrb::Matrix& transform);
 protected:
   struct State;
   VRVideo(State& aState, vrb::CreationContextPtr& aContext);

app/src/oculusvr/cpp/OculusVRLayers.cpp

@@ -221,6 +221,7 @@ OculusLayerEquirect::Update(const ovrTracking2& aTracking, ovrTextureSwapChain*
   OculusLayerBase<VRLayerEquirectPtr, ovrLayerEquirect2>::Update(aTracking, aClearSwapChain);
 
   vrb::Quaternion q(layer->GetModelTransform(device::Eye::Left));
+  q = q.Inverse();
   ovrLayer.HeadPose.Pose.Orientation.x  = q.x();
   ovrLayer.HeadPose.Pose.Orientation.y  = q.y();
   ovrLayer.HeadPose.Pose.Orientation.z  = q.z();