Add support for input and gestures for hand controllers

XRTK-Core/Packages/com.xrtk.core/Editor/Data/Controllers/Simulation/Hands/SimulatedHandControllerPoseDataInspector.cs → XRTK-Core/Packages/com.xrtk.core/Editor/Data/Controllers/Hands/HandControllerPoseDefinitionInspector.cs

@@ -2,17 +2,18 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
 using UnityEditor;
-using XRTK.Definitions.Controllers.Simulation.Hands;
+using XRTK.Definitions.Controllers.Hands;
 using XRTK.Editor.Profiles;
 
-namespace XRTK.Editor.Data.Controllers.Simulation.Hands
+namespace XRTK.Editor.Data.Controllers.Hands
 {
-    [CustomEditor(typeof(SimulatedHandControllerPoseData))]
-    public class SimulatedHandControllerPoseDataInspector : BaseMixedRealityProfileInspector
+    [CustomEditor(typeof(HandControllerPoseDefinition))]
+    public class HandControllerPoseDefinitionInspector : BaseMixedRealityProfileInspector
     {
         private SerializedProperty id;
         private SerializedProperty description;
         private SerializedProperty isDefault;
+        private SerializedProperty isSelectionPose;
         private SerializedProperty keyCode;
         private SerializedProperty data;
 
@@ -23,6 +24,7 @@ protected override void OnEnable()
             id = serializedObject.FindProperty(nameof(id));
             description = serializedObject.FindProperty(nameof(description));
             isDefault = serializedObject.FindProperty(nameof(isDefault));
+            isSelectionPose = serializedObject.FindProperty(nameof(isSelectionPose));
             keyCode = serializedObject.FindProperty(nameof(keyCode));
             data = serializedObject.FindProperty(nameof(data));
         }
@@ -36,6 +38,7 @@ public override void OnInspectorGUI()
             EditorGUILayout.PropertyField(id);
             EditorGUILayout.PropertyField(description);
             EditorGUILayout.PropertyField(isDefault);
+            EditorGUILayout.PropertyField(isSelectionPose);
             EditorGUILayout.PropertyField(keyCode);
             EditorGUILayout.PropertyField(data);
 

XRTK-Core/Packages/com.xrtk.core/Editor/Profiles/InputSystem/Controllers/Hands/BaseMixedRealityHandControllerDataProviderProfileInspector.cs

@@ -15,6 +15,7 @@ public class BaseMixedRealityHandControllerDataProviderProfileInspector : BaseMi
         private SerializedProperty handPhysicsEnabled;
         private SerializedProperty useTriggers;
         private SerializedProperty boundsMode;
+        private SerializedProperty trackedPoses;
 
         private bool showHandTrackingSettings = true;
         private static readonly GUIContent handTrackingSettingsFoldoutHeader = new GUIContent("Hand Tracking Settings");
@@ -27,6 +28,7 @@ protected override void OnEnable()
             handPhysicsEnabled = serializedObject.FindProperty(nameof(handPhysicsEnabled));
             useTriggers = serializedObject.FindProperty(nameof(useTriggers));
             boundsMode = serializedObject.FindProperty(nameof(boundsMode));
+            trackedPoses = serializedObject.FindProperty(nameof(trackedPoses));
         }
 
         public override void OnInspectorGUI()
@@ -39,18 +41,27 @@ public override void OnInspectorGUI()
             if (showHandTrackingSettings)
             {
                 EditorGUI.indentLevel++;
+
                 EditorGUILayout.LabelField("Hand Rendering Settings");
                 EditorGUI.indentLevel++;
                 EditorGUILayout.PropertyField(handMeshingEnabled);
-                EditorGUI.indentLevel--;
                 EditorGUILayout.Space();
+                EditorGUI.indentLevel--;
+
                 EditorGUILayout.LabelField("Hand Physics Settings");
                 EditorGUI.indentLevel++;
                 EditorGUILayout.PropertyField(handPhysicsEnabled);
                 EditorGUILayout.PropertyField(useTriggers);
                 EditorGUILayout.PropertyField(boundsMode);
                 EditorGUILayout.Space();
                 EditorGUI.indentLevel--;
+
+                EditorGUILayout.LabelField("Tracked Hand Poses");
+                EditorGUI.indentLevel++;
+                EditorGUILayout.PropertyField(trackedPoses, true);
+                EditorGUILayout.Space();
+                EditorGUI.indentLevel--;
+
                 EditorGUI.indentLevel--;
             }
 

XRTK-Core/Packages/com.xrtk.core/Editor/Profiles/InputSystem/Controllers/Simulation/SimulatedHandControllerDataProviderProfileInspector.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
 using UnityEditor;
-using UnityEditorInternal;
 using UnityEngine;
 using XRTK.Definitions.Controllers.Simulation.Hands;
 using XRTK.Editor.Extensions;
@@ -13,17 +12,14 @@ namespace XRTK.Editor.Profiles.InputSystem.Controllers.Simulation
     public class SimulatedHandControllerDataProviderProfileInspector : SimulatedControllerDataProviderProfileInspector
     {
         private static readonly GUIContent SimulatedHandSettingsFoldoutHeader = new GUIContent("Simulated Hand Tracking Settings");
+        private static readonly GUIContent handPoseAnimationSpeedLabel = new GUIContent("Hand Pose Animation Speed");
 
         private SerializedProperty handMeshingEnabled;
         private SerializedProperty handPhysicsEnabled;
         private SerializedProperty useTriggers;
         private SerializedProperty boundsMode;
-
+        private SerializedProperty trackedPoses;
         private SerializedProperty handPoseAnimationSpeed;
-        private SerializedProperty poseDefinitions;
-
-        private ReorderableList poseList;
-        private int currentlySelectedPoseElement;
 
         private bool showSimulatedHandTrackingSettings = true;
 
@@ -36,22 +32,9 @@ protected override void OnEnable()
             useTriggers = serializedObject.FindProperty(nameof(useTriggers));
             boundsMode = serializedObject.FindProperty(nameof(boundsMode));
 
-            poseDefinitions = serializedObject.FindProperty(nameof(poseDefinitions));
-            handPoseAnimationSpeed = serializedObject.FindProperty(nameof(handPoseAnimationSpeed));
-
-            poseList = new ReorderableList(serializedObject, poseDefinitions, true, false, true, true)
-            {
-                elementHeight = EditorGUIUtility.singleLineHeight * 1.5f
-            };
-            poseList.drawHeaderCallback += DrawHeaderCallback;
-            poseList.drawElementCallback += DrawConfigurationOptionElement;
-            poseList.onAddCallback += OnConfigurationOptionAdded;
-            poseList.onRemoveCallback += OnConfigurationOptionRemoved;
-        }
+            trackedPoses = serializedObject.FindProperty(nameof(trackedPoses));
 
-        private void DrawHeaderCallback(Rect rect)
-        {
-            EditorGUI.LabelField(rect, "Pose Definitions");
+            handPoseAnimationSpeed = serializedObject.FindProperty(nameof(handPoseAnimationSpeed));
         }
 
         public override void OnInspectorGUI()
@@ -66,67 +49,37 @@ public override void OnInspectorGUI()
             if (showSimulatedHandTrackingSettings)
             {
                 EditorGUI.indentLevel++;
+
                 EditorGUILayout.LabelField("Hand Rendering Settings");
                 EditorGUI.indentLevel++;
                 EditorGUILayout.PropertyField(handMeshingEnabled);
-                EditorGUI.indentLevel--;
                 EditorGUILayout.Space();
+                EditorGUI.indentLevel--;
+
                 EditorGUILayout.LabelField("Hand Physics Settings");
                 EditorGUI.indentLevel++;
                 EditorGUILayout.PropertyField(handPhysicsEnabled);
                 EditorGUILayout.PropertyField(useTriggers);
                 EditorGUILayout.PropertyField(boundsMode);
                 EditorGUILayout.Space();
                 EditorGUI.indentLevel--;
-                EditorGUILayout.LabelField("Simulated Poses");
+
+                EditorGUILayout.LabelField("Tracked Hand Poses");
                 EditorGUI.indentLevel++;
-                EditorGUILayout.PropertyField(handPoseAnimationSpeed);
-                poseList.DoLayoutList();
-                EditorGUI.indentLevel--;
+                EditorGUILayout.PropertyField(trackedPoses, true);
+                EditorGUILayout.Space();
                 EditorGUI.indentLevel--;
-            }
 
-            serializedObject.ApplyModifiedProperties();
-        }
-
-        private void DrawConfigurationOptionElement(Rect rect, int index, bool isActive, bool isFocused)
-        {
-            if (isFocused)
-            {
-                currentlySelectedPoseElement = index;
-            }
-
-            rect.height = EditorGUIUtility.singleLineHeight;
-            rect.y += 3;
-            var poseDataProperty = poseDefinitions.GetArrayElementAtIndex(index);
-            var selectedPoseData = EditorGUI.ObjectField(rect, poseDataProperty.objectReferenceValue, typeof(SimulatedHandControllerPoseData), false) as SimulatedHandControllerPoseData;
-
-            if (selectedPoseData != null)
-            {
-                selectedPoseData.ParentProfile = ThisProfile;
-            }
-
-            poseDataProperty.objectReferenceValue = selectedPoseData;
-        }
-
-        private void OnConfigurationOptionAdded(ReorderableList list)
-        {
-            poseDefinitions.arraySize += 1;
-            var index = poseDefinitions.arraySize - 1;
+                EditorGUILayout.LabelField("Simulated Poses");
+                EditorGUI.indentLevel++;
+                handPoseAnimationSpeed.floatValue = EditorGUILayout.Slider(handPoseAnimationSpeedLabel, handPoseAnimationSpeed.floatValue, 1, 10);
+                EditorGUILayout.Space();
+                EditorGUI.indentLevel--;
 
-            var mappingProfileProperty = poseDefinitions.GetArrayElementAtIndex(index);
-            mappingProfileProperty.objectReferenceValue = null;
-            serializedObject.ApplyModifiedProperties();
-        }
+                EditorGUI.indentLevel--;
 
-        private void OnConfigurationOptionRemoved(ReorderableList list)
-        {
-            if (currentlySelectedPoseElement >= 0)
-            {
-                poseDefinitions.DeleteArrayElementAtIndex(currentlySelectedPoseElement);
+                serializedObject.ApplyModifiedProperties();
             }
-
-            serializedObject.ApplyModifiedProperties();
         }
     }
 }

XRTK-Core/Packages/com.xrtk.core/Editor/Profiles/InputSystem/MixedRealityInputSystemProfileInspector.cs

@@ -35,6 +35,7 @@ public class MixedRealityInputSystemProfileInspector : MixedRealityServiceProfil
         private SerializedProperty handPhysicsEnabled;
         private SerializedProperty useTriggers;
         private SerializedProperty boundsMode;
+        private SerializedProperty trackedPoses;
 
         private SerializedProperty inputActionsProfile;
         private SerializedProperty speechCommandsProfile;
@@ -63,6 +64,7 @@ protected override void OnEnable()
             handPhysicsEnabled = serializedObject.FindProperty(nameof(handPhysicsEnabled));
             useTriggers = serializedObject.FindProperty(nameof(useTriggers));
             boundsMode = serializedObject.FindProperty(nameof(boundsMode));
+            trackedPoses = serializedObject.FindProperty(nameof(trackedPoses));
 
             inputActionsProfile = serializedObject.FindProperty(nameof(inputActionsProfile));
             gesturesProfile = serializedObject.FindProperty(nameof(gesturesProfile));
@@ -154,6 +156,7 @@ public override void OnInspectorGUI()
                 EditorGUILayout.PropertyField(handPhysicsEnabled);
                 EditorGUILayout.PropertyField(useTriggers);
                 EditorGUILayout.PropertyField(boundsMode);
+                EditorGUILayout.PropertyField(trackedPoses, true);
                 EditorGUI.indentLevel--;
             }
 

XRTK-Core/Packages/com.xrtk.core/Runtime/Definitions/Controllers/Hands/BaseHandControllerDataProviderProfile.cs

@@ -1,7 +1,9 @@
 // Copyright (c) XRTK. All rights reserved.
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
+using System.Collections.Generic;
 using UnityEngine;
+using XRTK.Definitions.Controllers.Simulation.Hands;
 using XRTK.Definitions.Utilities;
 using XRTK.Providers.Controllers.Hands;
 
@@ -48,6 +50,15 @@ public abstract class BaseHandControllerDataProviderProfile : BaseMixedRealityCo
         /// </summary>
         public HandBoundsMode BoundsMode => boundsMode;
 
+        [SerializeField]
+        [Tooltip("Tracked hand poses for pose detection.")]
+        private List<HandControllerPoseDefinition> trackedPoses = new List<HandControllerPoseDefinition>();
+
+        /// <summary>
+        /// Tracked hand poses for pose detection.
+        /// </summary>
+        public IReadOnlyList<HandControllerPoseDefinition> TrackedPoses => trackedPoses;
+
         public override ControllerDefinition[] GetDefaultControllerOptions()
         {
             return new[]

XRTK-Core/Packages/com.xrtk.core/Runtime/Definitions/Controllers/Simulation/Hands/SimulatedHandControllerPoseData.cs → XRTK-Core/Packages/com.xrtk.core/Runtime/Definitions/Controllers/Hands/HandControllerPoseDefinition.cs

@@ -4,30 +4,30 @@
 using UnityEngine;
 using XRTK.Definitions.Utilities;
 
-namespace XRTK.Definitions.Controllers.Simulation.Hands
+namespace XRTK.Definitions.Controllers.Hands
 {
     /// <summary>
-    /// A hand pose definition with recorded hand joint data.
-    /// Assign a collection of these poses to the data provider's profile to simulate the pose.
+    /// A hand controller pose definition with recorded hand joint data.
+    /// Defined hand poses can be recognized and trigger input actions.
     /// </summary>
-    [CreateAssetMenu(menuName = "Mixed Reality Toolkit/Input System/Simulated Hand Controller Pose Data", fileName = "SimulatedHandControllerPoseData", order = (int)CreateProfileMenuItemIndices.Input)]
-    public class SimulatedHandControllerPoseData : BaseMixedRealityProfile
+    [CreateAssetMenu(menuName = "Mixed Reality Toolkit/Input System/Hand Controller Pose Definition", fileName = "HandControllerPoseDefinition", order = (int)CreateProfileMenuItemIndices.Input)]
+    public class HandControllerPoseDefinition : BaseMixedRealityProfile
     {
         [SerializeField]
-        [Tooltip("Assign a unique ID to the pose. Can be used as a descriptive name, but must be unique!")]
+        [Tooltip("A unique ID to the pose. Can be a descriptive name, but must be unique!")]
         private string id = string.Empty;
 
         /// <summary>
-        /// A unique identifier for the hand pose.
+        /// A unique ID to the pose.
         /// </summary>
         public string Id => id;
 
         [SerializeField]
-        [Tooltip("Describe the hand pose.")]
+        [Tooltip("Describes the hand pose.")]
         private string description = string.Empty;
 
         /// <summary>
-        /// Gets the gesture's description.
+        /// Describes the hand pose.
         /// </summary>
         public string Description => description;
 
@@ -36,10 +36,19 @@ public class SimulatedHandControllerPoseData : BaseMixedRealityProfile
         private bool isDefault = false;
 
         /// <summary>
-        /// Gets whether this hand pose is the hand's idle pose.
+        /// Is this the default (idle) hand pose?
         /// </summary>
         public bool IsDefault => isDefault;
 
+        [SerializeField]
+        [Tooltip("Is this a selection pose used to select things?")]
+        private bool isSelectionPose = false;
+
+        /// <summary>
+        /// Is this a selection pose used to select things?
+        /// </summary>
+        public bool IsSelectionPose => isSelectionPose;
+
         [SerializeField]
         [Tooltip("Key used to trigger the gesture simulation.")]
         private KeyCode keyCode = KeyCode.None;

XRTK-Core/Packages/com.xrtk.core/Runtime/Definitions/Controllers/Hands/HandData.cs

@@ -27,6 +27,32 @@ public class HandData
         /// </summary>
         public bool IsTracked { get; set; } = false;
 
+        /// <summary>
+        /// Is the hand currently in a pinch pose?
+        /// </summary>
+        public bool IsPinching { get; set; } = false;
+
+        /// <summary>
+        /// Is the hand currently in a pointing pose?
+        /// </summary>
+        public bool IsPointing { get; set; } = false;
+
+        /// <summary>
+        /// Is the hand currently in a grabbing pose?
+        /// </summary>
+        public bool IsGrabbing { get; set; } = false;
+
+        /// <summary>
+        /// The hand's pointer pose.
+        /// </summary>
+        public MixedRealityPose PointerPose { get; set; } = MixedRealityPose.ZeroIdentity;
+
+        /// <summary>
+        /// Recognized hand pose, if any.
+        /// Recognizable hand poses are defined in <see cref="BaseHandControllerDataProviderProfile.TrackedPoses"/>.
+        /// </summary>
+        public HandControllerPoseDefinition TrackedPose { get; set; } = null;
+
         /// <summary>
         /// Pose information for each hand joint.
         /// </summary>