Pico Gaze Mode support

app/src/main/cpp/BrowserWorld.cpp

@@ -189,6 +189,7 @@ struct BrowserWorld::State {
   std::unordered_map<vrb::Node*, std::pair<Widget*, float>> depthSorting;
   std::function<void(device::Eye)> drawHandler;
   std::function<void()> frameEndHandler;
+  bool wasInGazeMode = false;
 
   State() : paused(true), glInitialized(false), modelsLoaded(false), env(nullptr), cylinderDensity(0.0f), nearClip(0.1f),
             farClip(300.0f), activity(nullptr), windowsInitialized(false), exitImmersiveRequested(false), loaderDelay(0) {
@@ -215,6 +216,7 @@ struct BrowserWorld::State {
     splashAnimation = SplashAnimation::Create(create);
     monitor = PerformanceMonitor::Create(create);
     monitor->AddPerformanceMonitorObserver(std::make_shared<PerformanceObserver>());
+    wasInGazeMode = false;
 #if defined(WAVEVR)
     monitor->SetPerformanceDelta(15.0);
 #endif
@@ -224,6 +226,7 @@ struct BrowserWorld::State {
   bool CheckExitImmersive();
   void EnsureControllerFocused();
   void ChangeControllerFocus(const Controller& aController);
+  void UpdateGazeModeState();
   void UpdateControllers(bool& aRelayoutWidgets);
   WidgetPtr GetWidget(int32_t aHandle) const;
   WidgetPtr FindWidget(const std::function<bool(const WidgetPtr&)>& aCondition) const;
@@ -336,6 +339,25 @@ ScaleScrollDelta(const float aValue, const double aStartTime, const double aCurr
   return aValue * scale;
 }
 
+void
+BrowserWorld::State::UpdateGazeModeState() {
+  bool isInGazeMode = device->IsInGazeMode();
+  if (isInGazeMode != wasInGazeMode) {
+    int32_t gazeIndex = device->GazeModeIndex();
+    if (isInGazeMode && gazeIndex >= 0) {
+      VRB_LOG("Gaze mode ON")
+      controllers->SetEnabled(gazeIndex, true);
+      controllers->SetVisible(gazeIndex, true);
+
+    } else {
+      VRB_LOG("Gaze mode OFF")
+      controllers->SetEnabled(gazeIndex, false);
+      controllers->SetVisible(gazeIndex, false);
+    }
+    wasInGazeMode = isInGazeMode;
+  }
+}
+
 void
 BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
   EnsureControllerFocused();
@@ -711,6 +733,7 @@ BrowserWorld::RegisterDeviceDelegate(DeviceDelegatePtr aDelegate) {
     m.leftCamera = m.device->GetCamera(device::Eye::Left);
     m.rightCamera = m.device->GetCamera(device::Eye::Right);
     ControllerDelegatePtr delegate = m.controllers;
+    delegate->SetGazeModeIndex(m.device->GazeModeIndex());
     m.device->SetClipPlanes(m.nearClip, m.farClip);
     m.device->SetControllerDelegate(delegate);
     m.gestures = m.device->GetGestureDelegate();
@@ -893,6 +916,7 @@ BrowserWorld::StartFrame() {
     TickImmersive();
   } else {
     bool relayoutWidgets = false;
+    m.UpdateGazeModeState();
     m.UpdateControllers(relayoutWidgets);
     if (relayoutWidgets) {
       UpdateVisibleWidgets();

app/src/main/cpp/ControllerContainer.cpp

@@ -35,6 +35,7 @@ struct ControllerContainer::State {
   GeometryPtr beamModel;
   bool visible = false;
   vrb::Color pointerColor;
+  int gazeIndex = -1;
 
   void Initialize(vrb::CreationContextPtr& aContext) {
     context = aContext;
@@ -201,33 +202,36 @@ ControllerContainer::CreateController(const int32_t aControllerIndex, const int3
   controller.transform = Transform::Create(create);
   controller.pointer = Pointer::Create(create);
   controller.pointer->SetVisible(true);
-  if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) {
-    controller.transform->AddNode(m.models[aModelIndex]);
-    controller.beamToggle = vrb::Toggle::Create(create);
-    controller.beamToggle->ToggleAll(true);
-    if (aBeamTransform.IsIdentity()) {
-      controller.beamParent = controller.beamToggle;
+  if (aControllerIndex != m.gazeIndex) {
+    if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) {
+      controller.transform->AddNode(m.models[aModelIndex]);
+      controller.beamToggle = vrb::Toggle::Create(create);
+      controller.beamToggle->ToggleAll(true);
+      if (aBeamTransform.IsIdentity()) {
+        controller.beamParent = controller.beamToggle;
+      } else {
+        vrb::TransformPtr beamTransform = Transform::Create(create);
+        beamTransform->SetTransform(aBeamTransform);
+        controller.beamParent = beamTransform;
+        controller.beamToggle->AddNode(beamTransform);
+      }
+      controller.transform->AddNode(controller.beamToggle);
+      if (m.beamModel && controller.beamParent) {
+        controller.beamParent->AddNode(m.beamModel);
+      }
     } else {
-      vrb::TransformPtr beamTransform = Transform::Create(create);
-      beamTransform->SetTransform(aBeamTransform);
-      controller.beamParent = beamTransform;
-      controller.beamToggle->AddNode(beamTransform);
-    }
-    controller.transform->AddNode(controller.beamToggle);
-    if (m.beamModel && controller.beamParent) {
-      controller.beamParent->AddNode(m.beamModel);
+      VRB_ERROR("Failed to add controller model");
     }
-    if (m.root) {
-      m.root->AddNode(controller.transform);
-      m.root->ToggleChild(*controller.transform, false);
-    }
-    if (m.pointerContainer) {
-      m.pointerContainer->AddNode(controller.pointer->GetRoot());
-    }
-    m.updatePointerColor(controller);
-  } else {
-    VRB_ERROR("Failed to add controller model");
   }
+
+  if (m.root) {
+    m.root->AddNode(controller.transform);
+    m.root->ToggleChild(*controller.transform, false);
+  }
+  if (m.pointerContainer) {
+    m.pointerContainer->AddNode(controller.pointer->GetRoot());
+  }
+  m.updatePointerColor(controller);
 }
 
 void
@@ -458,6 +462,10 @@ ControllerContainer::SetVisible(const bool aVisible) {
   }
 }
 
+void ControllerContainer::SetGazeModeIndex(const int32_t aControllerIndex) {
+  m.gazeIndex = aControllerIndex;
+}
+
 ControllerContainer::ControllerContainer(State& aState, vrb::CreationContextPtr& aContext) : m(aState) {
   m.Initialize(aContext);
 }

app/src/main/cpp/ControllerContainer.h

@@ -54,6 +54,7 @@ class ControllerContainer : public crow::ControllerDelegate  {
   void SetScrolledDelta(const int32_t aControllerIndex, const float aScrollDeltaX, const float aScrollDeltaY) override;
   void SetPointerColor(const vrb::Color& color) const;
   void SetVisible(const bool aVisible);
+  void SetGazeModeIndex(const int32_t aControllerIndex) override;
 protected:
   struct State;
   ControllerContainer(State& aState, vrb::CreationContextPtr& aContext);

app/src/main/cpp/ControllerDelegate.h

@@ -51,6 +51,7 @@ class ControllerDelegate {
   virtual void SetTouchPosition(const int32_t aControllerIndex, const float aTouchX, const float aTouchY) = 0;
   virtual void EndTouch(const int32_t aControllerIndex) = 0;
   virtual void SetScrolledDelta(const int32_t aControllerIndex, const float aScrollDeltaX, const float aScrollDeltaY) = 0;
+  virtual void SetGazeModeIndex(const int32_t aControllerIndex) = 0;
 protected:
   ControllerDelegate() {}
 private:

app/src/main/cpp/DeviceDelegate.h

@@ -66,6 +66,8 @@ class DeviceDelegate {
   virtual void StartFrame() = 0;
   virtual void BindEye(const device::Eye aWhich) = 0;
   virtual void EndFrame(bool aDiscard = false) = 0;
+  virtual bool IsInGazeMode() const { return false; };
+  virtual int32_t GazeModeIndex() const { return -1; };
   virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,
                                          VRLayerSurface::SurfaceType aSurfaceType) { return nullptr; }
   virtual VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) { return nullptr; }

app/src/main/cpp/ExternalVR.cpp

@@ -414,12 +414,12 @@ ExternalVR::PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<
     immersiveController.numButtons = controller.numButtons;
     immersiveController.buttonPressed = controller.immersivePressedState;
     immersiveController.buttonTouched = controller.immersiveTouchedState;
-    for (int i = 0; i< controller.numButtons; ++i) {
-      immersiveController.triggerValue[i] = controller.immersiveTriggerValues[i];
+    for (int j = 0; j < controller.numButtons; ++j) {
+      immersiveController.triggerValue[j] = controller.immersiveTriggerValues[j];
     }
     immersiveController.numAxes = controller.numAxes;
-    for (int i = 0; i< controller.numAxes; ++i) {
-      immersiveController.axisValue[i] = controller.immersiveAxes[i];
+    for (int j = 0; j < controller.numAxes; ++j) {
+      immersiveController.axisValue[j] = controller.immersiveAxes[j];
     }
     immersiveController.numHaptics = controller.numHaptics;
     immersiveController.hand = controller.leftHanded ? mozilla::gfx::ControllerHand::Left : mozilla::gfx::ControllerHand::Right;

app/src/picovr/cpp/DeviceDelegatePicoVR.cpp

@@ -29,9 +29,10 @@ namespace crow {
 
 static const vrb::Vector kAverageHeight(0.0f, 1.7f, 0.0f);
 // TODO: support different controllers & buttons
-static const int32_t kMaxControllerCount = 2;
+static const int32_t kMaxControllerCount = 3;
 static const int32_t kNumButtons = 6;
 static const int32_t kNumG2Buttons = 2;
+static const int32_t kNumGazeButtons = 2;
 static const int32_t kNumAxes = 2;
 static const int32_t kTypeNeo2 = 1;
 static const int32_t kButtonApp       = 1;
@@ -93,6 +94,8 @@ struct DeviceDelegatePicoVR::State {
   float fov = (float) (51.0 * M_PI / 180.0);
   int32_t focusIndex = 0;
   bool recentered = false;
+  bool isInGazeMode = false;
+  int32_t gazeIndex = -1;
 
   void Initialize() {
     vrb::RenderContextPtr localContext = context.lock();
@@ -103,6 +106,8 @@ struct DeviceDelegatePicoVR::State {
     UpdatePerspective();
     UpdateEyeTransform();
 
+    gazeIndex = VRBrowserPico::GetGazeIndex();
+
     for (int32_t index = 0; index < kMaxControllerCount; index++) {
       controllers[index].index = index;
       if (index == 0) {
@@ -211,12 +216,22 @@ struct DeviceDelegatePicoVR::State {
 
       vrb::Matrix transform = controller.transform;
       if (renderMode == device::RenderMode::StandAlone) {
-        if (type == kTypeNeo2) {
-          transform.TranslateInPlace(headOffset);
+        if (isInGazeMode) {
+          if (i == gazeIndex) {
+            vrb::Matrix head = vrb::Matrix::Rotation(orientation);
+            head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
+            controller.transform = head;
+            transform = controller.transform;
+          }
+
         } else {
-          vrb::Matrix head = vrb::Matrix::Rotation(orientation);
-          head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
-          transform = elbow->GetTransform(controller.hand, head, transform);
+          if (type == kTypeNeo2) {
+            transform.TranslateInPlace(headOffset);
+          } else {
+            vrb::Matrix head = vrb::Matrix::Rotation(orientation);
+            head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
+            transform = elbow->GetTransform(controller.hand, head, transform);
+          }
         }
       }
 
@@ -227,6 +242,10 @@ struct DeviceDelegatePicoVR::State {
       }
     }
   }
+
+  int32_t GazeModeIndex() {
+    return gazeIndex;
+  }
 };
 
 DeviceDelegatePicoVRPtr
@@ -311,18 +330,25 @@ DeviceDelegatePicoVR::SetControllerDelegate(ControllerDelegatePtr& aController)
   for (State::Controller& controller: m.controllers) {
     const int32_t index = controller.index;
 
-    if (m.type == kTypeNeo2) {
-      vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
-      beam.TranslateInPlace(vrb::Vector(0.0f, 0.012f, -0.06f));
-      m.controllerDelegate->CreateController(index, int32_t(controller.hand), controller.IsRightHand() ? "Pico Neo 2 (Right)" : "Pico Neo 2 (LEFT)", beam);
-      m.controllerDelegate->SetButtonCount(index, kNumButtons);
-      m.controllerDelegate->SetHapticCount(index, 1);
-    } else {
-      vrb::Matrix beam =  vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
-
-      m.controllerDelegate->CreateController(index, 0, "Pico G2 Controller", beam);
-      m.controllerDelegate->SetButtonCount(index, kNumG2Buttons);
+    if (index == m.gazeIndex) {
+      vrb::Matrix beam = vrb::Matrix::Identity();
+      m.controllerDelegate->CreateController(index, 0, "Pico Gaze Controller", beam);
+      m.controllerDelegate->SetButtonCount(index, kNumGazeButtons);
       m.controllerDelegate->SetHapticCount(index, 0);
+
+    } else {
+      if (m.type == kTypeNeo2) {
+        vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
+        beam.TranslateInPlace(vrb::Vector(0.0f, 0.012f, -0.06f));
+        m.controllerDelegate->CreateController(index, int32_t(controller.hand), controller.IsRightHand() ? "Pico Neo 2 (Right)" : "Pico Neo 2 (LEFT)", beam);
+        m.controllerDelegate->SetButtonCount(index, kNumButtons);
+        m.controllerDelegate->SetHapticCount(index, 1);
+      } else {
+        vrb::Matrix beam =  vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
+        m.controllerDelegate->CreateController(index, 0, "Pico G2 Controller", beam);
+        m.controllerDelegate->SetButtonCount(index, kNumG2Buttons);
+        m.controllerDelegate->SetHapticCount(index, 0);
+      }
     }
     controller.created = true;
   }
@@ -373,6 +399,22 @@ DeviceDelegatePicoVR::StartFrame() {
 
   m.cameras[0]->SetHeadTransform(head);
   m.cameras[1]->SetHeadTransform(head);
+
+
+  // Update te gaze mode state based on controllers availability
+  m.isInGazeMode = true;
+  for (int32_t i = 0; i < m.controllers.size(); ++i) {
+    if (i != m.gazeIndex && m.controllers[i].enabled) {
+      m.isInGazeMode = false;
+      break;
+    }
+  }
+
+  if (m.isInGazeMode) {
+    m.controllers[m.gazeIndex].enabled = m.isInGazeMode;
+    m.controllers[m.gazeIndex].transform = GetHeadTransform();
+  }
+
   m.UpdateControllers();
 }
 
@@ -388,6 +430,16 @@ DeviceDelegatePicoVR::EndFrame(const bool aDiscard) {
 
 }
 
+bool
+DeviceDelegatePicoVR::IsInGazeMode() const {
+  return m.isInGazeMode;
+}
+
+int32_t
+DeviceDelegatePicoVR::GazeModeIndex() const {
+  return m.gazeIndex;
+}
+
 void
 DeviceDelegatePicoVR::Pause() {
   m.paused = true;

app/src/picovr/cpp/DeviceDelegatePicoVR.h

@@ -40,6 +40,8 @@ class DeviceDelegatePicoVR : public DeviceDelegate {
   void StartFrame() override;
   void BindEye(const device::Eye aWhich) override;
   void EndFrame(const bool aDiscard) override;
+  bool IsInGazeMode() const override;
+  int32_t GazeModeIndex() const override;
   // Custom methods
   void Pause();
   void Resume();

app/src/picovr/cpp/VRBrowserPico.cpp

@@ -39,6 +39,8 @@ VRBrowserPico::InitializeJava(JNIEnv* aEnv, jobject aActivity) {
     return;
   }
 
+  sGetGazeIndex = FindJNIMethodID(sEnv, sBrowserClass, kGetGazeIndex, kGetGazeIndexSignature);
+
   sUpdateHaptics = FindJNIMethodID(sEnv, sBrowserClass, sUpdateHapticsName, sUpdateHapticsSignature);
   sCancelAllHaptics = FindJNIMethodID(sEnv, sBrowserClass, sCancelAllHapticsName, sCancelAllHapticsSignature);
 }
@@ -73,4 +75,13 @@ VRBrowserPico::CancelAllHaptics() {
   CheckJNIException(sEnv, __FUNCTION__);
 }
 
+int32_t
+VRBrowserPico::GetGazeIndex() {
+  if (!ValidateMethodID(sEnv, sActivity, sGetGazeIndex, __FUNCTION__)) { return -1; }
+  jint jGazeIndex = (jint) sEnv->CallIntMethod(sActivity, sGetGazeIndex);
+  CheckJNIException(sEnv, __FUNCTION__);
+
+  return (int32_t )jGazeIndex;
+}
+
 } // namespace crow

app/src/picovr/cpp/VRBrowserPico.h

@@ -11,13 +11,21 @@
 #include <jni.h>
 #include <functional>
 
+namespace {
+const char *kGetGazeIndex = "getGazeIndex";
+const char *kGetGazeIndexSignature = "()I";
+
+jmethodID sGetGazeIndex = nullptr;
+}
+
 namespace crow {
 
-namespace VRBrowserPico {
-void InitializeJava(JNIEnv* aEnv, jobject aActivity);
-void ShutdownJava();
-void UpdateHaptics(jint aControllerIndex, jfloat aIntensity, jfloat aDuration);
-void CancelAllHaptics();
-} // namespace VRBrowser;
+  namespace VRBrowserPico {
+    void InitializeJava(JNIEnv* aEnv, jobject aActivity);
+    void ShutdownJava();
+    void UpdateHaptics(jint aControllerIndex, jfloat aIntensity, jfloat aDuration);
+    void CancelAllHaptics();
+    int32_t GetGazeIndex();
+  } // namespace VRBrowser;
 
 } // namespace crow