Skip to content

Commit

Permalink
Pico Gaze Mode support (#2959)
Browse files Browse the repository at this point in the history 
* Pico Gaze Mode support

* PR comments updates

* Rebase fixes

* UPdate gazeIndex update in container

Co-authored-by: Randall E. Barker <simstorm@mac.com>
  • Loading branch information
@keianhzo @bluemarvin
keianhzo and bluemarvin committed Mar 18, 2020
1 parent 25cca5a commit 15eae62
Show file tree
Hide file tree
Showing 11 changed files with 181 additions and 51 deletions.
24 changes: 24 additions & 0 deletions app/src/main/cpp/BrowserWorld.cpp
View file
Expand Up @@ -189,6 +189,7 @@ struct BrowserWorld::State {
std::unordered_map<vrb::Node*, std::pair<Widget*, float>> depthSorting;
std::function<void(device::Eye)> drawHandler;
std::function<void()> frameEndHandler;
bool wasInGazeMode = false;

State() : paused(true), glInitialized(false), modelsLoaded(false), env(nullptr), cylinderDensity(0.0f), nearClip(0.1f),
farClip(300.0f), activity(nullptr), windowsInitialized(false), exitImmersiveRequested(false), loaderDelay(0) {
Expand All @@ -215,6 +216,7 @@ struct BrowserWorld::State {
splashAnimation = SplashAnimation::Create(create);
monitor = PerformanceMonitor::Create(create);
monitor->AddPerformanceMonitorObserver(std::make_shared<PerformanceObserver>());
wasInGazeMode = false;
#if defined(WAVEVR)
monitor->SetPerformanceDelta(15.0);
#endif
Expand All @@ -224,6 +226,7 @@ struct BrowserWorld::State {
bool CheckExitImmersive();
void EnsureControllerFocused();
void ChangeControllerFocus(const Controller& aController);
void UpdateGazeModeState();
void UpdateControllers(bool& aRelayoutWidgets);
WidgetPtr GetWidget(int32_t aHandle) const;
WidgetPtr FindWidget(const std::function<bool(const WidgetPtr&)>& aCondition) const;
Expand Down Expand Up @@ -336,6 +339,25 @@ ScaleScrollDelta(const float aValue, const double aStartTime, const double aCurr
return aValue * scale;
}

void
BrowserWorld::State::UpdateGazeModeState() {
bool isInGazeMode = device->IsInGazeMode();
if (isInGazeMode != wasInGazeMode) {
int32_t gazeIndex = device->GazeModeIndex();
if (isInGazeMode && gazeIndex >= 0) {
VRB_LOG("Gaze mode ON")
controllers->SetEnabled(gazeIndex, true);
controllers->SetVisible(gazeIndex, true);

} else {
VRB_LOG("Gaze mode OFF")
controllers->SetEnabled(gazeIndex, false);
controllers->SetVisible(gazeIndex, false);
}
wasInGazeMode = isInGazeMode;
}
}

void
BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
EnsureControllerFocused();
Expand Down Expand Up @@ -711,6 +733,7 @@ BrowserWorld::RegisterDeviceDelegate(DeviceDelegatePtr aDelegate) {
m.leftCamera = m.device->GetCamera(device::Eye::Left);
m.rightCamera = m.device->GetCamera(device::Eye::Right);
ControllerDelegatePtr delegate = m.controllers;
delegate->SetGazeModeIndex(m.device->GazeModeIndex());
m.device->SetClipPlanes(m.nearClip, m.farClip);
m.device->SetControllerDelegate(delegate);
m.gestures = m.device->GetGestureDelegate();
Expand Down Expand Up @@ -893,6 +916,7 @@ BrowserWorld::StartFrame() {
TickImmersive();
} else {
bool relayoutWidgets = false;
m.UpdateGazeModeState();
m.UpdateControllers(relayoutWidgets);
if (relayoutWidgets) {
UpdateVisibleWidgets();
Expand Down
56 changes: 32 additions & 24 deletions app/src/main/cpp/ControllerContainer.cpp
View file
Expand Up @@ -35,6 +35,7 @@ struct ControllerContainer::State {
GeometryPtr beamModel;
bool visible = false;
vrb::Color pointerColor;
int gazeIndex = -1;

void Initialize(vrb::CreationContextPtr& aContext) {
context = aContext;
Expand Down Expand Up @@ -201,33 +202,36 @@ ControllerContainer::CreateController(const int32_t aControllerIndex, const int3
controller.transform = Transform::Create(create);
controller.pointer = Pointer::Create(create);
controller.pointer->SetVisible(true);
if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) {
controller.transform->AddNode(m.models[aModelIndex]);
controller.beamToggle = vrb::Toggle::Create(create);
controller.beamToggle->ToggleAll(true);
if (aBeamTransform.IsIdentity()) {
controller.beamParent = controller.beamToggle;
if (aControllerIndex != m.gazeIndex) {
if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) {
controller.transform->AddNode(m.models[aModelIndex]);
controller.beamToggle = vrb::Toggle::Create(create);
controller.beamToggle->ToggleAll(true);
if (aBeamTransform.IsIdentity()) {
controller.beamParent = controller.beamToggle;
} else {
vrb::TransformPtr beamTransform = Transform::Create(create);
beamTransform->SetTransform(aBeamTransform);
controller.beamParent = beamTransform;
controller.beamToggle->AddNode(beamTransform);
}
controller.transform->AddNode(controller.beamToggle);
if (m.beamModel && controller.beamParent) {
controller.beamParent->AddNode(m.beamModel);
}
} else {
vrb::TransformPtr beamTransform = Transform::Create(create);
beamTransform->SetTransform(aBeamTransform);
controller.beamParent = beamTransform;
controller.beamToggle->AddNode(beamTransform);
}
controller.transform->AddNode(controller.beamToggle);
if (m.beamModel && controller.beamParent) {
controller.beamParent->AddNode(m.beamModel);
VRB_ERROR("Failed to add controller model");
}
if (m.root) {
m.root->AddNode(controller.transform);
m.root->ToggleChild(*controller.transform, false);
}
if (m.pointerContainer) {
m.pointerContainer->AddNode(controller.pointer->GetRoot());
}
m.updatePointerColor(controller);
} else {
VRB_ERROR("Failed to add controller model");
}

if (m.root) {
m.root->AddNode(controller.transform);
m.root->ToggleChild(*controller.transform, false);
}
if (m.pointerContainer) {
m.pointerContainer->AddNode(controller.pointer->GetRoot());
}
m.updatePointerColor(controller);
}

void
Expand Down Expand Up @@ -458,6 +462,10 @@ ControllerContainer::SetVisible(const bool aVisible) {
}
}

void ControllerContainer::SetGazeModeIndex(const int32_t aControllerIndex) {
m.gazeIndex = aControllerIndex;
}

ControllerContainer::ControllerContainer(State& aState, vrb::CreationContextPtr& aContext) : m(aState) {
m.Initialize(aContext);
}
Expand Down
1 change: 1 addition & 0 deletions app/src/main/cpp/ControllerContainer.h
View file
Expand Up @@ -54,6 +54,7 @@ class ControllerContainer : public crow::ControllerDelegate {
void SetScrolledDelta(const int32_t aControllerIndex, const float aScrollDeltaX, const float aScrollDeltaY) override;
void SetPointerColor(const vrb::Color& color) const;
void SetVisible(const bool aVisible);
void SetGazeModeIndex(const int32_t aControllerIndex) override;
protected:
struct State;
ControllerContainer(State& aState, vrb::CreationContextPtr& aContext);
Expand Down
1 change: 1 addition & 0 deletions app/src/main/cpp/ControllerDelegate.h
View file
Expand Up @@ -51,6 +51,7 @@ class ControllerDelegate {
virtual void SetTouchPosition(const int32_t aControllerIndex, const float aTouchX, const float aTouchY) = 0;
virtual void EndTouch(const int32_t aControllerIndex) = 0;
virtual void SetScrolledDelta(const int32_t aControllerIndex, const float aScrollDeltaX, const float aScrollDeltaY) = 0;
virtual void SetGazeModeIndex(const int32_t aControllerIndex) = 0;
protected:
ControllerDelegate() {}
private:
Expand Down
2 changes: 2 additions & 0 deletions app/src/main/cpp/DeviceDelegate.h
View file
Expand Up @@ -66,6 +66,8 @@ class DeviceDelegate {
virtual void StartFrame() = 0;
virtual void BindEye(const device::Eye aWhich) = 0;
virtual void EndFrame(bool aDiscard = false) = 0;
virtual bool IsInGazeMode() const { return false; };
virtual int32_t GazeModeIndex() const { return -1; };
virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,
VRLayerSurface::SurfaceType aSurfaceType) { return nullptr; }
virtual VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) { return nullptr; }
Expand Down
8 changes: 4 additions & 4 deletions app/src/main/cpp/ExternalVR.cpp
View file
Expand Up @@ -414,12 +414,12 @@ ExternalVR::PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<
immersiveController.numButtons = controller.numButtons;
immersiveController.buttonPressed = controller.immersivePressedState;
immersiveController.buttonTouched = controller.immersiveTouchedState;
for (int i = 0; i< controller.numButtons; ++i) {
immersiveController.triggerValue[i] = controller.immersiveTriggerValues[i];
for (int j = 0; j < controller.numButtons; ++j) {
immersiveController.triggerValue[j] = controller.immersiveTriggerValues[j];
}
immersiveController.numAxes = controller.numAxes;
for (int i = 0; i< controller.numAxes; ++i) {
immersiveController.axisValue[i] = controller.immersiveAxes[i];
for (int j = 0; j < controller.numAxes; ++j) {
immersiveController.axisValue[j] = controller.immersiveAxes[j];
}
immersiveController.numHaptics = controller.numHaptics;
immersiveController.hand = controller.leftHanded ? mozilla::gfx::ControllerHand::Left : mozilla::gfx::ControllerHand::Right;
Expand Down
85 changes: 68 additions & 17 deletions app/src/picovr/cpp/DeviceDelegatePicoVR.cpp
View file
Expand Up @@ -29,9 +29,10 @@ namespace crow {

static const vrb::Vector kAverageHeight(0.0f, 1.7f, 0.0f);
// TODO: support different controllers & buttons
static const int32_t kMaxControllerCount = 2;
static const int32_t kMaxControllerCount = 3;
static const int32_t kNumButtons = 6;
static const int32_t kNumG2Buttons = 2;
static const int32_t kNumGazeButtons = 2;
static const int32_t kNumAxes = 2;
static const int32_t k6DofHeadSet = 1;
static const int32_t kButtonApp = 1;
Expand Down Expand Up @@ -93,6 +94,8 @@ struct DeviceDelegatePicoVR::State {
float fov = (float) (51.0 * M_PI / 180.0);
int32_t focusIndex = 0;
bool recentered = false;
bool isInGazeMode = false;
int32_t gazeIndex = -1;

void Initialize() {
vrb::RenderContextPtr localContext = context.lock();
Expand All @@ -103,6 +106,8 @@ struct DeviceDelegatePicoVR::State {
UpdatePerspective();
UpdateEyeTransform();

gazeIndex = VRBrowserPico::GetGazeIndex();

for (int32_t index = 0; index < kMaxControllerCount; index++) {
controllers[index].index = index;
if (index == 0) {
Expand Down Expand Up @@ -211,12 +216,21 @@ struct DeviceDelegatePicoVR::State {

vrb::Matrix transform = controller.transform;
if (renderMode == device::RenderMode::StandAlone) {
if (type == k6DofHeadSet) {
transform.TranslateInPlace(headOffset);
if (isInGazeMode) {
if (i == gazeIndex) {
vrb::Matrix head = vrb::Matrix::Rotation(orientation);
head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
controller.transform = head;
transform = controller.transform;
}
} else {
vrb::Matrix head = vrb::Matrix::Rotation(orientation);
head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
transform = elbow->GetTransform(controller.hand, head, transform);
if (type == k6DofHeadSet) {
transform.TranslateInPlace(headOffset);
} else {
vrb::Matrix head = vrb::Matrix::Rotation(orientation);
head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
transform = elbow->GetTransform(controller.hand, head, transform);
}
}
}

Expand All @@ -227,6 +241,10 @@ struct DeviceDelegatePicoVR::State {
}
}
}

int32_t GazeModeIndex() {
return gazeIndex;
}
};

DeviceDelegatePicoVRPtr
Expand Down Expand Up @@ -315,18 +333,25 @@ DeviceDelegatePicoVR::SetControllerDelegate(ControllerDelegatePtr& aController)
for (State::Controller& controller: m.controllers) {
const int32_t index = controller.index;

if (m.type == k6DofHeadSet) {
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
beam.TranslateInPlace(vrb::Vector(0.0f, 0.012f, -0.06f));
m.controllerDelegate->CreateController(index, int32_t(controller.hand), controller.IsRightHand() ? "Pico Neo 2 (Right)" : "Pico Neo 2 (LEFT)", beam);
m.controllerDelegate->SetButtonCount(index, kNumButtons);
m.controllerDelegate->SetHapticCount(index, 1);
} else {
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);

m.controllerDelegate->CreateController(index, 0, "Pico G2 Controller", beam);
m.controllerDelegate->SetButtonCount(index, kNumG2Buttons);
if (index == m.gazeIndex) {
vrb::Matrix beam = vrb::Matrix::Identity();
m.controllerDelegate->CreateController(index, 0, "Pico Gaze Controller", beam);
m.controllerDelegate->SetButtonCount(index, kNumGazeButtons);
m.controllerDelegate->SetHapticCount(index, 0);

} else {
if (m.type == k6DofHeadSet) {
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
beam.TranslateInPlace(vrb::Vector(0.0f, 0.012f, -0.06f));
m.controllerDelegate->CreateController(index, int32_t(controller.hand), controller.IsRightHand() ? "Pico Neo 2 (Right)" : "Pico Neo 2 (LEFT)", beam);
m.controllerDelegate->SetButtonCount(index, kNumButtons);
m.controllerDelegate->SetHapticCount(index, 1);
} else {
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
m.controllerDelegate->CreateController(index, 0, "Pico G2 Controller", beam);
m.controllerDelegate->SetButtonCount(index, kNumG2Buttons);
m.controllerDelegate->SetHapticCount(index, 0);
}
}
controller.created = true;
}
Expand Down Expand Up @@ -377,6 +402,22 @@ DeviceDelegatePicoVR::StartFrame() {

m.cameras[0]->SetHeadTransform(head);
m.cameras[1]->SetHeadTransform(head);


// Update te gaze mode state based on controllers availability
m.isInGazeMode = true;
for (int32_t i = 0; i < m.controllers.size(); ++i) {
if (i != m.gazeIndex && m.controllers[i].enabled) {
m.isInGazeMode = false;
break;
}
}

if (m.isInGazeMode) {
m.controllers[m.gazeIndex].enabled = m.isInGazeMode;
m.controllers[m.gazeIndex].transform = GetHeadTransform();
}

m.UpdateControllers();
}

Expand All @@ -392,6 +433,16 @@ DeviceDelegatePicoVR::EndFrame(const bool aDiscard) {

}

bool
DeviceDelegatePicoVR::IsInGazeMode() const {
return m.isInGazeMode;
}

int32_t
DeviceDelegatePicoVR::GazeModeIndex() const {
return m.gazeIndex;
}

void
DeviceDelegatePicoVR::Pause() {
m.paused = true;
Expand Down
2 changes: 2 additions & 0 deletions app/src/picovr/cpp/DeviceDelegatePicoVR.h
View file
Expand Up @@ -40,6 +40,8 @@ class DeviceDelegatePicoVR : public DeviceDelegate {
void StartFrame() override;
void BindEye(const device::Eye aWhich) override;
void EndFrame(const bool aDiscard) override;
bool IsInGazeMode() const override;
int32_t GazeModeIndex() const override;
// Custom methods
void Pause();
void Resume();
Expand Down
11 changes: 11 additions & 0 deletions app/src/picovr/cpp/VRBrowserPico.cpp
View file
Expand Up @@ -39,6 +39,8 @@ VRBrowserPico::InitializeJava(JNIEnv* aEnv, jobject aActivity) {
return;
}

sGetGazeIndex = FindJNIMethodID(sEnv, sBrowserClass, kGetGazeIndex, kGetGazeIndexSignature);

sUpdateHaptics = FindJNIMethodID(sEnv, sBrowserClass, sUpdateHapticsName, sUpdateHapticsSignature);
sCancelAllHaptics = FindJNIMethodID(sEnv, sBrowserClass, sCancelAllHapticsName, sCancelAllHapticsSignature);
}
Expand Down Expand Up @@ -73,4 +75,13 @@ VRBrowserPico::CancelAllHaptics() {
CheckJNIException(sEnv, __FUNCTION__);
}

int32_t
VRBrowserPico::GetGazeIndex() {
if (!ValidateMethodID(sEnv, sActivity, sGetGazeIndex, __FUNCTION__)) { return -1; }
jint jGazeIndex = (jint) sEnv->CallIntMethod(sActivity, sGetGazeIndex);
CheckJNIException(sEnv, __FUNCTION__);

return (int32_t )jGazeIndex;
}

} // namespace crow
20 changes: 14 additions & 6 deletions app/src/picovr/cpp/VRBrowserPico.h
View file
Expand Up @@ -11,13 +11,21 @@
#include <jni.h>
#include <functional>

namespace {
const char *kGetGazeIndex = "getGazeIndex";
const char *kGetGazeIndexSignature = "()I";

jmethodID sGetGazeIndex = nullptr;
}

namespace crow {

namespace VRBrowserPico {
void InitializeJava(JNIEnv* aEnv, jobject aActivity);
void ShutdownJava();
void UpdateHaptics(jint aControllerIndex, jfloat aIntensity, jfloat aDuration);
void CancelAllHaptics();
} // namespace VRBrowser;
namespace VRBrowserPico {
void InitializeJava(JNIEnv* aEnv, jobject aActivity);
void ShutdownJava();
void UpdateHaptics(jint aControllerIndex, jfloat aIntensity, jfloat aDuration);
void CancelAllHaptics();
int32_t GetGazeIndex();
} // namespace VRBrowser;

} // namespace crow

0 comments on commit 15eae62

Please sign in to comment.