Skip to content

Commit

Permalink
Enable hit testing against different entityTypes in BabylonNative (#364)
Browse files Browse the repository at this point in the history 
* Implement hit test filtering on entity types for Android

* Move getHitPose into hitTestQualified call.

* Make bitfield enum more explicit.

* iOS support for raycast query filtering.

* Update stub method for OpenXR

* Cleanup

* Apply suggestions from code review

Co-authored-by: Gary Hsu <bghgary@users.noreply.github.com>

* Add |= and use std::underlying_type

Co-authored-by: Gary Hsu <bghgary@users.noreply.github.com>
  • Loading branch information
@Alex-MSFT @bghgary
Alex-MSFT and bghgary committed Aug 13, 2020
1 parent 5e6bf16 commit 6868472
Show file tree
Hide file tree
Showing 7 changed files with 1,714 additions and 409 deletions.
1,872 changes: 1,524 additions & 348 deletions Apps/BabylonScripts/babylon.max.js
View file

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion Apps/Playground/Scripts/experience.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,13 @@ CreateBoxAsync().then(function () {
setTimeout(function () {
scene.createDefaultXRExperienceAsync({ disableDefaultUI: true, disableTeleportation: true }).then((xr) => {
if (xrHitTest) {
// Create the hit test module. OffsetRay specifies the target direction, and entityTypes can be any combination of "mesh", "plane", and "point".
const xrHitTestModule = xr.baseExperience.featuresManager.enableFeature(
BABYLON.WebXRFeatureName.HIT_TEST,
"latest",
{offsetRay: {origin: {x: 0, y: 0, z: 0}, direction: {x: 0, y: 0, z: -1}}});
{offsetRay: {origin: {x: 0, y: 0, z: 0}, direction: {x: 0, y: 0, z: -1}}, entityTypes: ["mesh"]});

// When we receive hit test results, if there were any valid hits move the position of the mesh to the hit test point.
xrHitTestModule.onHitTestResultObservable.add((results) => {
if (results.length) {
scene.meshes[0].position.x = results[0].position.x;
Expand Down
25 changes: 24 additions & 1 deletion Dependencies/xr/Include/XR.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,29 @@ namespace xr
XYZ
};

enum class HitTestTrackableType {
NONE = 0,
POINT = 1 << 0,
PLANE = 1 << 1,
MESH = 1 << 2,
};

constexpr enum HitTestTrackableType operator |(const enum HitTestTrackableType selfValue, const enum HitTestTrackableType inValue)
{
return static_cast<const enum HitTestTrackableType>(std::underlying_type_t<HitTestTrackableType>(selfValue) | std::underlying_type_t<HitTestTrackableType>(inValue));
}

constexpr enum HitTestTrackableType operator &(const enum HitTestTrackableType selfValue, const enum HitTestTrackableType inValue)
{
return static_cast<const enum HitTestTrackableType>(std::underlying_type_t<HitTestTrackableType>(selfValue) & std::underlying_type_t<HitTestTrackableType>(inValue));
}

constexpr enum HitTestTrackableType& operator |=(enum HitTestTrackableType& selfValue, const enum HitTestTrackableType inValue)
{
selfValue = selfValue | inValue;
return selfValue;
}

struct Size
{
size_t Width{};
Expand Down Expand Up @@ -172,7 +195,7 @@ namespace xr
Frame(System::Session::Impl&);
~Frame();

void GetHitTestResults(std::vector<HitResult>&, Ray) const;
void GetHitTestResults(std::vector<HitResult>&, Ray, HitTestTrackableType) const;
Anchor CreateAnchor(Pose, NativeAnchorPtr) const;
void UpdateAnchor(Anchor&) const;
void DeleteAnchor(Anchor&) const;
Expand Down
49 changes: 33 additions & 16 deletions Dependencies/xr/Source/ARCore/XR.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,7 @@ namespace xr
}
}

void GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay)
void GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay, xr::HitTestTrackableType validHitTestTypes)
{
if (!IsTracking())
{
Expand Down Expand Up @@ -667,27 +667,44 @@ namespace xr
ArTrackableType trackableType{};
ArTrackable* trackable;

bool hitTestResultValid{false};
ArHitResultList_getItem(session, hitResultList, i, hitResult);
ArHitResult_acquireTrackable(session, hitResult, &trackable);
ArTrackable_getType(session, trackable, &trackableType);
if (trackableType == AR_TRACKABLE_PLANE)
{
int32_t isPoseInPolygon{};
ArHitResult_getHitPose(session, hitResult, tempPose);
ArPlane_isPoseInPolygon(session, (ArPlane*) trackable, tempPose, &isPoseInPolygon);

if (isPoseInPolygon != 0)
// If we are only hit testing against planes then mark the hit test as valid otherwise check
// if the hit result is inside the plane mesh.
if ((validHitTestTypes & xr::HitTestTrackableType::PLANE) != xr::HitTestTrackableType::NONE)
{
hitTestResultValid = true;
}
else if ((validHitTestTypes & xr::HitTestTrackableType::MESH) != xr::HitTestTrackableType::NONE)
{
float rawPose[7]{};
ArPose_getPoseRaw(session, tempPose, rawPose);
HitResult hitResult{};
RawToPose(rawPose, hitResult.Pose);

hitResult.NativeTrackable = reinterpret_cast<NativeTrackablePtr>(trackable);
filteredResults.push_back(hitResult);
frameTrackables.push_back(trackable);
int32_t isPoseInPolygon{};
ArHitResult_getHitPose(session, hitResult, tempPose);
ArPlane_isPoseInPolygon(session, reinterpret_cast<ArPlane*>(trackable), tempPose, &isPoseInPolygon);
hitTestResultValid = isPoseInPolygon != 0;
}
}
else if (trackableType == AR_TRACKABLE_POINT && (validHitTestTypes & xr::HitTestTrackableType::POINT) != xr::HitTestTrackableType::NONE)
{
// Hit a feature point, which is valid for this hit test source.
hitTestResultValid = true;
}

if (hitTestResultValid)
{
float rawPose[7]{};
ArHitResult_getHitPose(session, hitResult, tempPose);
ArPose_getPoseRaw(session, tempPose, rawPose);
HitResult hitResult{};
RawToPose(rawPose, hitResult.Pose);

hitResult.NativeTrackable = reinterpret_cast<NativeTrackablePtr>(trackable);
filteredResults.push_back(hitResult);
frameTrackables.push_back(trackable);
}
}
}

Expand Down Expand Up @@ -1047,9 +1064,9 @@ namespace xr
m_impl->sessionImpl.UpdatePlanes(UpdatedPlanes, RemovedPlanes);
}

void System::Session::Frame::GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay) const
void System::Session::Frame::GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay, xr::HitTestTrackableType trackableTypes) const
{
m_impl->sessionImpl.GetHitTestResults(filteredResults, offsetRay);
m_impl->sessionImpl.GetHitTestResults(filteredResults, offsetRay, trackableTypes);
}

Anchor System::Session::Frame::CreateAnchor(Pose pose, NativeTrackablePtr trackable) const
Expand Down
128 changes: 87 additions & 41 deletions Dependencies/xr/Source/ARKit/XR.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -805,53 +805,18 @@ void DrawFrame() {
}
}

void GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay) const {
void GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay, xr::HitTestTrackableType trackableTypes) const {
@autoreleasepool {
if (session != nil && session.currentFrame != nil && session.currentFrame.camera != nil && [session.currentFrame.camera trackingState] == ARTrackingStateNormal) {
if (@available(iOS 13.0, *)) {
// Push the camera origin into a simd_float3.
auto cameraOrigin = simd_make_float3(
ActiveFrameViews[0].Space.Pose.Position.X,
ActiveFrameViews[0].Space.Pose.Position.Y,
ActiveFrameViews[0].Space.Pose.Position.Z);

// Push the camera direction into a simd_quaternion.
auto cameraDirection = simd_quaternion(
ActiveFrameViews[0].Space.Pose.Orientation.X,
ActiveFrameViews[0].Space.Pose.Orientation.Y,
ActiveFrameViews[0].Space.Pose.Orientation.Z,
ActiveFrameViews[0].Space.Pose.Orientation.W);

// Load the offset ray and direction into simd equivalents.
auto offsetOrigin = simd_make_float3(offsetRay.Origin.X, offsetRay.Origin.Y, offsetRay.Origin.Z);
auto offsetDirection = simd_make_float3(offsetRay.Direction.X, offsetRay.Direction.Y, offsetRay.Direction.Z);

// Construct the ARRaycast query compositing the camera and offset ray origin + direction targetting existing plane geometry.
auto raycastQuery = [[ARRaycastQuery alloc]
initWithOrigin:(cameraOrigin + offsetOrigin)
direction:simd_act(cameraDirection, offsetDirection)
allowingTarget:ARRaycastTargetExistingPlaneGeometry
alignment:ARRaycastTargetAlignmentAny];

// Perform the actual raycast.
auto rayCastResults = [session raycast:raycastQuery];
[raycastQuery release];

// Process the results and push them into the results list.
for (ARRaycastResult* result in rayCastResults) {
filteredResults.push_back(transformToHitResult(result.worldTransform));
}
GetHitTestResultsForiOS13(filteredResults, offsetRay, trackableTypes);
} else {
// On iOS versions prior to 13, fall back to doing a raycast from a screen point, for now don't support translating the offset ray.
auto hitTestResults = [session.currentFrame hitTest:CGPointMake(.5, .5) types:(ARHitTestResultTypeExistingPlane)];
for (ARHitTestResult* result in hitTestResults) {
filteredResults.push_back(transformToHitResult(result.worldTransform));
}
GetHitTestResultsLegacy(filteredResults, trackableTypes);
}
}
}
}

/**
Create an ARKit anchor for the given pose.
*/
Expand Down Expand Up @@ -1070,6 +1035,87 @@ void UpdatePlane(std::vector<Frame::Plane::Identifier>& updatedPlanes, Frame::Pl
plane.PolygonFormat = PolygonFormat::XYZ;
updatedPlanes.push_back(plane.ID);
}

// For iOS 13.0 and up make use of the ARRaycastQuery protocol for raycasting against all target trackable types.
API_AVAILABLE(ios(13.0))
void GetHitTestResultsForiOS13(std::vector<HitResult>& filteredResults, xr::Ray offsetRay, xr::HitTestTrackableType trackableTypes) const{
// Push the camera origin into a simd_float3.
auto cameraOrigin = simd_make_float3(
ActiveFrameViews[0].Space.Pose.Position.X,
ActiveFrameViews[0].Space.Pose.Position.Y,
ActiveFrameViews[0].Space.Pose.Position.Z);

// Push the camera direction into a simd_quaternion.
auto cameraDirection = simd_quaternion(
ActiveFrameViews[0].Space.Pose.Orientation.X,
ActiveFrameViews[0].Space.Pose.Orientation.Y,
ActiveFrameViews[0].Space.Pose.Orientation.Z,
ActiveFrameViews[0].Space.Pose.Orientation.W);

// Load the offset ray and direction into simd equivalents.
auto offsetOrigin = simd_make_float3(offsetRay.Origin.X, offsetRay.Origin.Y, offsetRay.Origin.Z);
auto offsetDirection = simd_make_float3(offsetRay.Direction.X, offsetRay.Direction.Y, offsetRay.Direction.Z);
auto rayOrigin = cameraOrigin + offsetOrigin;
auto rayDirection = simd_act(cameraDirection, offsetDirection);

// Check which types we are meant to raycast against and perform their respective queries.
if ((trackableTypes & xr::HitTestTrackableType::MESH) != xr::HitTestTrackableType::NONE) {
PerformRaycastQueryAgainstTarget(filteredResults, ARRaycastTargetExistingPlaneGeometry, rayOrigin, rayDirection);
}

if ((trackableTypes & xr::HitTestTrackableType::POINT) != xr::HitTestTrackableType::NONE) {
PerformRaycastQueryAgainstTarget(filteredResults, ARRaycastTargetEstimatedPlane, rayOrigin, rayDirection);
}

if ((trackableTypes & xr::HitTestTrackableType::PLANE) != xr::HitTestTrackableType::NONE) {
PerformRaycastQueryAgainstTarget(filteredResults, ARRaycastTargetExistingPlaneInfinite, rayOrigin, rayDirection);
}
}

API_AVAILABLE(ios(13.0))
void PerformRaycastQueryAgainstTarget(std::vector<HitResult>& filteredResults, ARRaycastTarget targetType, simd_float3 origin, simd_float3 direction) const {
auto raycastQuery = [[ARRaycastQuery alloc]
initWithOrigin:origin
direction:direction
allowingTarget:targetType
alignment:ARRaycastTargetAlignmentAny];

// Perform the actual raycast.
auto rayCastResults = [session raycast:raycastQuery];
[raycastQuery release];

// Process the results and push them into the results list.
for (ARRaycastResult* result in rayCastResults) {
filteredResults.push_back(transformToHitResult(result.worldTransform));
}
}

// On iOS versions prior to 13, fall back to doing a raycast from a screen point, for now don't support translating the offset ray.
void GetHitTestResultsLegacy(std::vector<HitResult>& filteredResults, xr::HitTestTrackableType trackableTypes) const {
// First set the type filter based on the requested trackable types.
ARHitTestResultType typeFilter = 0;
if ((trackableTypes & xr::HitTestTrackableType::POINT) != xr::HitTestTrackableType::NONE) {
typeFilter |= ARHitTestResultTypeFeaturePoint;
}

if ((trackableTypes & xr::HitTestTrackableType::PLANE) != xr::HitTestTrackableType::NONE) {
typeFilter |= ARHitTestResultTypeExistingPlane;
}

if ((trackableTypes & xr::HitTestTrackableType::POINT) != xr::HitTestTrackableType::NONE) {
if (@available(iOS 11.3, *)) {
typeFilter |= ARHitTestResultTypeExistingPlaneUsingGeometry;
} else {
typeFilter |= ARHitTestResultTypeExistingPlaneUsingExtent;
}
}

// Now perform the actual hit test and process the results
auto hitTestResults = [session.currentFrame hitTest:CGPointMake(.5, .5) types:(typeFilter)];
for (ARHitTestResult* result in hitTestResults) {
filteredResults.push_back(transformToHitResult(result.worldTransform));
}
}
};

struct System::Session::Frame::Impl {
Expand All @@ -1096,8 +1142,8 @@ void UpdatePlane(std::vector<Frame::Plane::Identifier>& updatedPlanes, Frame::Pl
m_impl->sessionImpl.DrawFrame();
}

void System::Session::Frame::GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay) const {
m_impl->sessionImpl.GetHitTestResults(filteredResults, offsetRay);
void System::Session::Frame::GetHitTestResults(std::vector<HitResult>& filteredResults, xr::Ray offsetRay, xr::HitTestTrackableType trackableTypes) const {
m_impl->sessionImpl.GetHitTestResults(filteredResults, offsetRay, trackableTypes);
}

Anchor System::Session::Frame::CreateAnchor(Pose pose, NativeTrackablePtr) const {
Expand Down
2 changes: 1 addition & 1 deletion Dependencies/xr/Source/OpenXR/XR.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ namespace xr
}
}

void System::Session::Frame::GetHitTestResults(std::vector<HitResult>&, Ray) const {
void System::Session::Frame::GetHitTestResults(std::vector<HitResult>&, Ray, xr::HitTestTrackableType) const {
// Stubbed out for now, should be implemented if we want to support OpenXR based passthrough AR devices.
}

Expand Down
43 changes: 42 additions & 1 deletion Plugins/NativeXr/Source/NativeXr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,13 @@ namespace Babylon
static constexpr auto UNBOUNDED{"unbounded"};
};

struct XRHitTestTrackableType
{
static constexpr auto POINT{"point"};
static constexpr auto PLANE{"plane"};
static constexpr auto MESH{"mesh"};
};

struct XREye
{
// static constexpr auto NONE{"none"};
Expand Down Expand Up @@ -1126,6 +1133,33 @@ namespace Babylon
m_offsetRay = Napi::Persistent(options.Get("offsetRay").As<Napi::Object>());
hasOffsetRay = true;
}

if (options.Has("entityTypes"))
{
const auto entityTypeArray = options.Get("entityTypes").As<Napi::Array>();
for (uint32_t i = 0; i < entityTypeArray.Length(); i++)
{
const auto entityType = entityTypeArray.Get(i).As<Napi::String>().Utf8Value();
if (entityType == XRHitTestTrackableType::POINT)
{
m_entityTypes |= xr::HitTestTrackableType::POINT;
}
else if (entityType == XRHitTestTrackableType::PLANE)
{
m_entityTypes |= xr::HitTestTrackableType::PLANE;
}
else if (entityType == XRHitTestTrackableType::MESH)
{
m_entityTypes |= xr::HitTestTrackableType::MESH;
}
}
}

// Default to MESH if unspecified.
if (m_entityTypes == xr::HitTestTrackableType::NONE)
{
m_entityTypes = xr::HitTestTrackableType::MESH;
}
}

XRRay* OffsetRay()
Expand All @@ -1138,6 +1172,11 @@ namespace Babylon
return hasSpace ? XRReferenceSpace::Unwrap(m_space.Value()) : nullptr;
}

xr::HitTestTrackableType GetEntityTypes()
{
return m_entityTypes;
}

private:
void Cancel(const Napi::CallbackInfo& /*info*/)
{
Expand All @@ -1149,6 +1188,8 @@ namespace Babylon

bool hasOffsetRay = false;
Napi::ObjectReference m_offsetRay;

xr::HitTestTrackableType m_entityTypes{xr::HitTestTrackableType::NONE};
};

// Implementation of the XRHitTestResult interface: https://immersive-web.github.io/hit-test/#xr-hit-test-result-interface
Expand Down Expand Up @@ -1457,7 +1498,7 @@ namespace Babylon

// Get the native results
std::vector<xr::HitResult> nativeHitResults{};
m_frame->GetHitTestResults(nativeHitResults, nativeRay);
m_frame->GetHitTestResults(nativeHitResults, nativeRay, hitTestSource->GetEntityTypes());

// Translate those results into a napi array.
auto results = Napi::Array::New(info.Env(), nativeHitResults.size());
Expand Down

0 comments on commit 6868472

Please sign in to comment.