Skip to content

Commit

Permalink
[OpenXR] Add support for hand interaction profile
Browse files Browse the repository at this point in the history 
OpenXR defines the XR_EXT_hand_interaction extension that allows
users of the API to handle the input from hands as if they were
physical controllers.

The hand tracking extensions we have been using so far, were
completely disconnected to the OpenXR input model based on actions
and input profiles. Hand gestures did not trigger events that
can be handled using the OpenXR input model. That's why we had
to use vendor specific extensions to compute aim or detect
pinches or special system actions. For those systems not providing
those we were inferring them from the position of the hand joints.

However with the hand interaction extension we can use the
same input model based on actions and suggest bindings for
certain paths provided by that extension. Those paths include
poses (aim, grip, poke...) and also actions (pinch, grip...).

Apart from adding the input profiles we had to slightly modify
the current code so that it does not assume that all the events
coming from actions are from physical controllers. Likewise
we should also retrieve hand joint information not only if
there is no physical controller, but also if the system
is using the hand interaction profile.
  • Loading branch information
@svillar
svillar committed Jun 10, 2024
1 parent 4812847 commit 60245a4
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 41 deletions.
3 changes: 3 additions & 0 deletions app/src/openxr/cpp/DeviceDelegateOpenXR.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,9 @@ struct DeviceDelegateOpenXR::State {
if (OpenXRExtensions::IsExtensionSupported(XR_ML_FRAME_END_INFO_EXTENSION_NAME))
extensions.push_back(XR_ML_FRAME_END_INFO_EXTENSION_NAME);

if (OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME))
extensions.push_back(XR_EXT_HAND_INTERACTION_EXTENSION_NAME);

java = {XR_TYPE_INSTANCE_CREATE_INFO_ANDROID_KHR};
java.applicationVM = javaContext->vm;
java.applicationActivity = javaContext->activity;
Expand Down
3 changes: 3 additions & 0 deletions app/src/openxr/cpp/OpenXRActionSet.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ XrResult OpenXRActionSet::GetOrCreateButtonActions(OpenXRButtonType type, OpenXR
if (flags & OpenXRButtonFlags::Value) {
RETURN_IF_XR_FAILED(CreateAction(XR_ACTION_TYPE_FLOAT_INPUT, key + "_value", hand, actions.value));
}
if (flags & OpenXRButtonFlags::Ready) {
RETURN_IF_XR_FAILED(CreateAction(XR_ACTION_TYPE_BOOLEAN_INPUT, key + "_ready_ext", hand, actions.ready));
}

mButtonActions.emplace(key, actions);

Expand Down
1 change: 1 addition & 0 deletions app/src/openxr/cpp/OpenXRActionSet.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ namespace crow {
XrAction click { XR_NULL_HANDLE };
XrAction touch { XR_NULL_HANDLE };
XrAction value { XR_NULL_HANDLE };
XrAction ready { XR_NULL_HANDLE };
};
private:
OpenXRActionSet(XrInstance, XrSession);
Expand Down
3 changes: 3 additions & 0 deletions app/src/openxr/cpp/OpenXRExtensions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ void OpenXRExtensions::Initialize() {
#elif PICOXR
// Pico incorrectly advertises this extension as supported but it makes Wolvic not work.
sSupportedExtensions.erase(XR_EXTX_OVERLAY_EXTENSION_NAME);
#elif SPACES
// Spaces incorrectly advertises this extension as supported but it does not really work.
sSupportedExtensions.erase(XR_EXT_HAND_INTERACTION_EXTENSION_NAME);
#endif

// Adding this check here is ugly but required to have a working build for VRX. With the current
Expand Down
25 changes: 23 additions & 2 deletions app/src/openxr/cpp/OpenXRInputMappings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,13 @@ namespace crow {
constexpr const char* kPathButtonB { "input/b" };
constexpr const char* kPathButtonX { "input/x" };
constexpr const char* kPathButtonY { "input/y" };
constexpr const char* kPinchPose { "input/pinch_ext/pose" };
constexpr const char* kPokePose { "input/poke_ext/pose" };
constexpr const char* kPathActionClick { "click" };
constexpr const char* kPathActionTouch { "touch" };
constexpr const char* kPathActionValue { "value" };
constexpr const char* kPathActionReady { "ready_ext" };
constexpr const char* kInteractionProfileHandInteraction { "/interaction_profiles/ext/hand_interaction_ext" };

// OpenXR Button List
enum class OpenXRButtonType {
Expand Down Expand Up @@ -73,9 +77,11 @@ namespace crow {
Click = (1u << 0),
Touch = (1u << 1),
Value = (1u << 2),
Ready = (1u << 3),
ValueTouch = Touch | Value,
ClickTouch = Click | Touch,
ClickValue = Click | Value,
ReadyValue = Ready | Value,
All = Click | Touch | Value,
};

Expand Down Expand Up @@ -391,6 +397,21 @@ namespace crow {
},
};

const OpenXRInputMapping HandInteraction {
kInteractionProfileHandInteraction,
IS_6DOF,
"",
"",
device::UnknownType,
std::vector<OpenXRInputProfile> { "generic-hand-select-grasp", "generic-hand-select", "generic-hand" },
std::vector<OpenXRButton> {
{ OpenXRButtonType::Trigger, "input/pinch_ext", OpenXRButtonFlags::ReadyValue, OpenXRHandFlags::Both },
/* Not adding aim_activate_ext nor grasp_ext as we don't need them yet */
},
{},
{}
};

// Default fallback: https://github.com/immersive-web/webxr-input-profiles/blob/master/packages/registry/profiles/generic/generic-button.json
const OpenXRInputMapping KHRSimple {
"/interaction_profiles/khr/simple_controller",
Expand All @@ -408,8 +429,8 @@ namespace crow {
},
};

const std::array<OpenXRInputMapping, 11> OpenXRInputMappings {
OculusTouch, OculusTouch2, MetaQuestTouchPro, Pico4x, PicoNeo3, Hvr6DOF, Hvr3DOF, LenovoVRX, MagicLeap2, MetaTouchPlus, KHRSimple
const std::array<OpenXRInputMapping, 12> OpenXRInputMappings {
OculusTouch, OculusTouch2, MetaQuestTouchPro, Pico4x, PicoNeo3, Hvr6DOF, Hvr3DOF, LenovoVRX, MagicLeap2, MetaTouchPlus, HandInteraction, KHRSimple
};

} // namespace crow
118 changes: 83 additions & 35 deletions app/src/openxr/cpp/OpenXRInputSource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ namespace crow {
// Threshold to consider a trigger value as a click
// Used when devices don't map the click value for triggers;
const float kClickThreshold = 0.91f;
const float kClickLowFiThreshold = 0.8f;

OpenXRInputSourcePtr OpenXRInputSource::Create(XrInstance instance, XrSession session, OpenXRActionSet& actionSet, const XrSystemProperties& properties, OpenXRHandFlags handeness, int index)
{
Expand Down Expand Up @@ -51,13 +52,18 @@ XrResult OpenXRInputSource::Initialize()
{
mSubactionPathName = mHandeness == OpenXRHandFlags::Left ? kPathLeftHand : kPathRightHand;
mSubactionPath = mActionSet.GetSubactionPath(mHandeness);
mIsHandInteractionEXTSupported = OpenXRExtensions::IsExtensionSupported(XR_EXT_HAND_INTERACTION_EXTENSION_NAME);

// Initialize Action Set.
std::string prefix = std::string("input_") + (mHandeness == OpenXRHandFlags::Left ? "left" : "right");

// Initialize pose actions and spaces.
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "grip", OpenXRHandFlags::Both, mGripAction));
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "pointer", OpenXRHandFlags::Both, mPointerAction));
if (mIsHandInteractionEXTSupported) {
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "pinch_ext", OpenXRHandFlags::Both, mPinchPoseAction));
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_POSE_INPUT, "poke_ext", OpenXRHandFlags::Both, mPokePoseAction));
}
RETURN_IF_XR_FAILED(mActionSet.GetOrCreateAction(XR_ACTION_TYPE_VIBRATION_OUTPUT, "haptic", OpenXRHandFlags::Both, mHapticAction));

// Filter mappings
Expand Down Expand Up @@ -140,19 +146,14 @@ XrResult OpenXRInputSource::Initialize()
#else
mSupportsFBHandTrackingAim = OpenXRExtensions::IsExtensionSupported(XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME);
#endif
VRB_LOG("OpenXR: using %s to compute hands aim", mSupportsFBHandTrackingAim ? "XR_FB_HAND_TRACKING_AIM" : "hand joints");

if (mSupportsFBHandTrackingAim)
mGestureManager = std::make_unique<OpenXRGestureManagerFBHandTrackingAim>();
else {
else if (!mIsHandInteractionEXTSupported) {
// TODO: fine tune params for different devices.
OpenXRGestureManagerHandJoints::OneEuroFilterParams params;
if (deviceType == device::MagicLeap2)
params = {0.25, 2, 1};
else
params = { 0.25, 0.1, 1 };
OpenXRGestureManagerHandJoints::OneEuroFilterParams params= { 0.25, 1, 1 };
mGestureManager = std::make_unique<OpenXRGestureManagerHandJoints>(mHandJoints, &params);
}
VRB_LOG("OpenXR: using %s to compute hands aim", mSupportsFBHandTrackingAim ? XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME : (mIsHandInteractionEXTSupported ? XR_EXT_HAND_INTERACTION_EXTENSION_NAME : "hand joints"));
}

// Initialize double buffers for storing XR_MSFT_hand_tracking_mesh geometry
Expand Down Expand Up @@ -265,6 +266,7 @@ std::optional<OpenXRInputSource::OpenXRButtonState> OpenXRInputSource::GetButton
bool clickedHasValue = hasValue;
queryActionState(button.flags & OpenXRButtonFlags::Touch, actions.touch, result.touched, result.clicked);
queryActionState(button.flags & OpenXRButtonFlags::Value, actions.value, result.value, result.clicked ? 1.0 : 0.0);
queryActionState(button.flags & OpenXRButtonFlags::Ready, actions.ready, result.ready, true);

if (!clickedHasValue && result.value > kClickThreshold) {
result.clicked = true;
Expand Down Expand Up @@ -420,6 +422,13 @@ XrResult OpenXRInputSource::SuggestBindings(SuggestedBindings& bindings) const
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, mGripAction, mSubactionPathName + "/" + kPathGripPose, bindings));
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, mPointerAction, mSubactionPathName + "/" + kPathAimPose, bindings));

// FIXME: reenable this once MagicLeap OS v1.8 is released as it has a fix for this.
// Otherwise the hand interaction profile is not properly used.
if (mIsHandInteractionEXTSupported && (DeviceUtils::GetDeviceTypeFromSystem(true) != device::MagicLeap2)) {
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, mPinchPoseAction, mSubactionPathName + "/" + kPinchPose, bindings));
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, mPokePoseAction, mSubactionPathName + "/" + kPokePose, bindings));
}

// Suggest binding for button actions.
for (auto& button: mapping.buttons) {
if ((button.hand & mHandeness) == 0) {
Expand All @@ -443,6 +452,10 @@ XrResult OpenXRInputSource::SuggestBindings(SuggestedBindings& bindings) const
assert(actions.value != XR_NULL_HANDLE);
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, actions.value, mSubactionPathName + "/" + button.path + "/" + kPathActionValue, bindings));
}
if (button.flags & OpenXRButtonFlags::Ready) {
assert(actions.ready != XR_NULL_HANDLE);
RETURN_IF_XR_FAILED(CreateBinding(mapping.path, actions.ready, mSubactionPathName + "/" + button.path + "/" + kPathActionReady, bindings));
}
}

// Suggest binding for axis actions.
Expand Down Expand Up @@ -529,7 +542,8 @@ bool OpenXRInputSource::GetHandTrackingInfo(XrTime predictedDisplayTime, XrSpace
XrHandJointLocationsEXT jointLocations { XR_TYPE_HAND_JOINT_LOCATIONS_EXT };
jointLocations.jointCount = XR_HAND_JOINT_COUNT_EXT;
jointLocations.jointLocations = mHandJoints.data();
mGestureManager->populateNextStructureIfNeeded(jointLocations);
if (mGestureManager)
mGestureManager->populateNextStructureIfNeeded(jointLocations);

CHECK_XRCMD(OpenXRExtensions::sXrLocateHandJointsEXT(mHandTracker, &locateInfo, &jointLocations));
mHasHandJoints = jointLocations.isActive;
Expand All @@ -553,7 +567,6 @@ bool OpenXRInputSource::GetHandTrackingInfo(XrTime predictedDisplayTime, XrSpace
mHasHandJoints = mHasHandJoints && hasAtLeastOneValidJoint(jointLocations);

// Rest of the method deal with XR_MSFT_hand_tracking_mesh extension

if (!OpenXRExtensions::IsExtensionSupported(XR_MSFT_HAND_TRACKING_MESH_EXTENSION_NAME) || !mHasHandJoints)
return mHasHandJoints;

Expand Down Expand Up @@ -601,12 +614,8 @@ bool OpenXRInputSource::GetHandTrackingInfo(XrTime predictedDisplayTime, XrSpace
return mHasHandJoints;
}

void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, ControllerDelegate& delegate)
{
// Prepare and submit hand joint locations data for rendering
assert(mHasHandJoints);
std::vector<vrb::Matrix> jointTransforms;
std::vector<float> jointRadii;
void
OpenXRInputSource::PopulateHandJointLocations(device::RenderMode renderMode, std::vector<vrb::Matrix>& jointTransforms, std::vector<float>& jointRadii) {
jointTransforms.resize(mHandJoints.size());
jointRadii.resize(mHandJoints.size());
for (int i = 0; i < mHandJoints.size(); i++) {
Expand All @@ -623,16 +632,6 @@ void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode,
jointTransforms[i] = transform;
jointRadii[i] = mHandJoints[i].radius;
}

// This is not really needed. It's just an optimization for devices taking over the control of
// hands when facing head. In those cases we don't need to do all the matrix computations.
bool systemTakesOverWhenHandsFacingHead = false;
#if defined(OCULUSVR)
systemTakesOverWhenHandsFacingHead = true;
#endif
bool hasAim = mGestureManager->hasAim();
bool systemGestureDetected = mGestureManager->systemGestureDetected(jointTransforms[HAND_JOINT_FOR_AIM], head);

#if defined(PICOXR)
// Scale joints according to their radius (for rendering). This is currently only
// relevant on Pico with system version earlier than 5.7.1, where we are using spheres
Expand All @@ -648,10 +647,23 @@ void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode,
}
}
#endif
}

void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode, XrTime predictedDisplayTime, const vrb::Matrix& head, const vrb::Matrix& handJointForAim, ControllerDelegate& delegate)
{
assert(mHasHandJoints);

// This is not really needed. It's just an optimization for devices taking over the control of
// hands when facing head. In those cases we don't need to do all the matrix computations.
bool systemTakesOverWhenHandsFacingHead = false;
#if defined(OCULUSVR)
systemTakesOverWhenHandsFacingHead = true;
#endif
bool hasAim = mGestureManager->hasAim();
bool systemGestureDetected = mGestureManager->systemGestureDetected(handJointForAim, head);

// We should handle the gesture whenever the system does not handle it.
bool isHandActionEnabled = systemGestureDetected && (!systemTakesOverWhenHandsFacingHead || mHandeness == Left);
delegate.SetHandJointLocations(mIndex, std::move(jointTransforms), std::move(jointRadii));
delegate.SetAimEnabled(mIndex, hasAim);
delegate.SetHandActionEnabled(mIndex, isHandActionEnabled);
delegate.SetMode(mIndex, ControllerMode::Hand);
Expand Down Expand Up @@ -752,6 +764,12 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
if (mPointerSpace == XR_NULL_HANDLE) {
CHECK_XRCMD(CreateActionSpace(mPointerAction, mPointerSpace));
}
if (mIsHandInteractionEXTSupported) {
if (mPinchSpace == XR_NULL_HANDLE)
CHECK_XRCMD(CreateActionSpace(mPinchPoseAction, mPinchSpace));
if (mPokeSpace == XR_NULL_HANDLE)
CHECK_XRCMD(CreateActionSpace(mPokePoseAction, mPokeSpace));
}

// Pose transforms.
bool isPoseActive { false };
Expand All @@ -776,12 +794,27 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
#else
bool isControllerUnavailable = (poseLocation.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) == 0;
#endif
if (isControllerUnavailable && GetHandTrackingInfo(frameState.predictedDisplayTime, localSpace, head)) {
EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, delegate);
return;

auto gotHandTrackingInfo = false;
if (isControllerUnavailable || mUsingHandInteractionProfile) {
gotHandTrackingInfo = GetHandTrackingInfo(frameState.predictedDisplayTime, localSpace, head);
if (gotHandTrackingInfo) {
std::vector<vrb::Matrix> jointTransforms;
std::vector<float> jointRadii;
PopulateHandJointLocations(renderMode, jointTransforms, jointRadii);
if (!mIsHandInteractionEXTSupported) {
EmulateControllerFromHand(renderMode, frameState.predictedDisplayTime, head, jointTransforms[HAND_JOINT_FOR_AIM], delegate);
delegate.SetHandJointLocations(mIndex, std::move(jointTransforms), std::move(jointRadii));
return;
}
delegate.SetHandJointLocations(mIndex, std::move(jointTransforms), std::move(jointRadii));
}
}

if ((poseLocation.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) == 0) {
bool hasAim = isPoseActive && (poseLocation.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT);
delegate.SetAimEnabled(mIndex, hasAim);

if (!hasAim && !mUsingHandInteractionProfile) {
delegate.SetEnabled(mIndex, false);
return;
}
Expand All @@ -795,9 +828,13 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
};
adjustPoseLocation(offsets);

delegate.SetMode(mIndex, ControllerMode::Device);
// XR_EXT_hand_interaction does not really require the hand joints data, but if we
// set ControllerMode::Hand then Wolvic code assumes that it does.
delegate.SetMode(mIndex, mUsingHandInteractionProfile && gotHandTrackingInfo ? ControllerMode::Hand : ControllerMode::Device);
delegate.SetEnabled(mIndex, true);
delegate.SetAimEnabled(mIndex, true);

bool isHandActionEnabled = !hasAim && mUsingHandInteractionProfile;
delegate.SetHandActionEnabled(mIndex, isHandActionEnabled);

device::CapabilityFlags flags = device::Orientation;
vrb::Matrix pointerTransform = XrPoseToMatrix(poseLocation.pose);
Expand Down Expand Up @@ -866,10 +903,20 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
buttonCount++;
auto browserButton = GetBrowserButton(button);
auto immersiveButton = GetImmersiveButton(button);
delegate.SetButtonState(mIndex, browserButton, immersiveButton.has_value() ? immersiveButton.value() : -1, state->clicked, state->touched, state->value);

if (button.type == OpenXRButtonType::Trigger)
if (isHandActionEnabled) {
// When hand faces head, tracking systems do not have the same level of precision
// detecting pinches, that's why we need to lower the bar to detect them.
delegate.SetButtonState(mIndex, ControllerDelegate::BUTTON_APP, -1, state->value >= kClickLowFiThreshold,
state->value > 0, 1.0);
} else {
delegate.SetButtonState(mIndex, browserButton, immersiveButton.has_value() ? immersiveButton.value() : -1,
state->clicked, state->touched, state->value);
}

if (button.type == OpenXRButtonType::Trigger && state->ready) {
delegate.SetSelectFactor(mIndex, state->value);
}

// Select action
if (renderMode == device::RenderMode::Immersive && button.type == OpenXRButtonType::Trigger && state->clicked != selectActionStarted) {
Expand Down Expand Up @@ -969,6 +1016,7 @@ XrResult OpenXRInputSource::UpdateInteractionProfile(ControllerDelegate& delegat
for (auto& mapping : mMappings) {
if (!strncmp(mapping.path, path, path_len)) {
mActiveMapping = &mapping;
mUsingHandInteractionProfile = !strncmp(path, kInteractionProfileHandInteraction, path_len);
break;
}
}
Expand Down
Loading

0 comments on commit 60245a4

Please sign in to comment.