Avatar Hand Track / IK Improvements (#7920)

* Add initial hand tracking support, refactor duplicate code

* Add offset

* Account for hand rig alignment weirdness

* Move matrix objects to module level

* Add height adjustment functions

* Math object cleanup

* Add scale adjustment

* cleanup

* refactor IK targets to be their own entities

* Move quaternion offset to source pose

* optimize ik and skeleton matrix updates

* update screenshare to overwrite existing materials

* dev

---------

Co-authored-by: HexaField <joshfield999@gmail.com>

packages/client-core/src/components/ARPlacement/index.tsx

@@ -17,9 +17,8 @@ export const ARPlacement = () => {
 
   const engineState = useHookstate(getMutableState(EngineState))
   const xrState = useHookstate(getMutableState(XRState))
-  const supportsAR = xrState.supportedSessionModes['immersive-ar'].value
-  const xrSessionActive = xrState.sessionActive.value
-  if (!supportsAR || !engineState.sceneLoaded.value || !xrSessionActive) return <></>
+  const isARSession = xrState.sessionMode.value === 'immersive-ar'
+  if (!isARSession || !engineState.sceneLoaded.value) return <></>
 
   const inPlacingMode = xrState.scenePlacementMode.value === 'placing'
 

packages/client-core/src/components/World/EngineHooks.tsx

@@ -121,10 +121,7 @@ export const usePortalTeleport = () => {
       if (!activePortal) return
 
       const currentLocation = locationState.locationName.value.split('/')[1]
-      if (
-        currentLocation === activePortal.location ||
-        UUIDComponent.entitiesByUUID[activePortal.linkedPortalId]?.value
-      ) {
+      if (currentLocation === activePortal.location || UUIDComponent.entitiesByUUID[activePortal.linkedPortalId]) {
         teleportAvatar(
           Engine.instance.localClientEntity!,
           activePortal.remoteSpawnPosition

packages/client-core/src/media/webcam/WebcamInput.ts

@@ -260,7 +260,7 @@ const avatarSpawnQueue = defineActionQueue(WorldNetworkAction.spawnAvatar.matche
 
 const execute = () => {
   for (const action of avatarSpawnQueue()) {
-    const entity = UUIDComponent.entitiesByUUID.value[action.uuid]
+    const entity = UUIDComponent.entitiesByUUID[action.uuid]
     setComponent(entity, WebcamInputComponent)
   }
   for (const entity of webcamQuery()) setAvatarExpression(entity)

packages/client-core/src/recording/RecordingService.ts

@@ -1,5 +1,4 @@
 import { RecordingResult } from '@etherealengine/common/src/interfaces/Recording'
-import { IKSerialization } from '@etherealengine/engine/src/avatar/IKSerialization'
 import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
 import { ECSRecordingActions } from '@etherealengine/engine/src/ecs/ECSRecording'
 import { defineSystem } from '@etherealengine/engine/src/ecs/functions/SystemFunctions'
@@ -34,9 +33,6 @@ export const RecordingFunctions = {
         schema.push(PhysicsSerialization.ID)
       }
       if (state.config.mocap) {
-        schema.push(IKSerialization.headID)
-        schema.push(IKSerialization.leftHandID)
-        schema.push(IKSerialization.rightHandID)
         schema.push(mocapDataChannelType)
       }
       if (state.config.video) {

packages/client-core/src/systems/ui/WidgetMenuView/index.tsx

@@ -10,6 +10,7 @@ import { WidgetAppActions, WidgetAppState } from '@etherealengine/engine/src/xru
 import { dispatchAction, getMutableState } from '@etherealengine/hyperflux'
 import Icon from '@etherealengine/ui/src/Icon'
 
+import { setTrackingSpace } from '../../../../../engine/src/xr/XRScaleAdjustmentFunctions'
 import { useMediaInstance } from '../../../common/services/MediaInstanceConnectionService'
 import { useChatState } from '../../../social/services/ChatService'
 import { MediaStreamState } from '../../../transports/MediaStreams'
@@ -91,6 +92,10 @@ const WidgetButtons = () => {
     respawnAvatar(Engine.instance.localClientEntity)
   }
 
+  const handleHeightAdjustment = () => {
+    setTrackingSpace()
+  }
+
   const widgets = Object.entries(widgetMutableState.widgets.value).map(([id, widgetMutableState]) => ({
     id,
     ...widgetMutableState,
@@ -122,6 +127,7 @@ const WidgetButtons = () => {
       <style>{styleString}</style>
       <div className="container" style={{ gridTemplateColumns }} xr-pixel-ratio="8" xr-layer="true">
         <WidgetButton icon="Refresh" toggle={handleRespawnAvatar} label={'Respawn'} />
+        <WidgetButton icon="Person" toggle={handleHeightAdjustment} label={'Reset Height'} />
         {mediaInstanceState?.value && (
           <WidgetButton
             icon={isCamAudioEnabled ? 'Mic' : 'MicOff'}

packages/editor/src/components/properties/PropertiesPanelContainer.tsx

@@ -66,7 +66,7 @@ export const PropertiesPanelContainer = () => {
   const lockedNode = editorState.lockPropertiesPanel.value
   const multiEdit = selectedEntities.length > 1
   let nodeEntity = lockedNode
-    ? UUIDComponent.entitiesByUUID[lockedNode].value ?? lockedNode
+    ? UUIDComponent.entitiesByUUID[lockedNode] ?? lockedNode
     : selectedEntities[selectedEntities.length - 1]
   const isMaterial =
     typeof nodeEntity === 'string' &&

packages/editor/src/components/properties/Util.ts

@@ -47,7 +47,7 @@ export const updateProperties = <C extends Component>(
   const affectedNodes = nodes
     ? nodes
     : editorState.lockPropertiesPanel.value
-    ? [UUIDComponent.entitiesByUUID[editorState.lockPropertiesPanel.value]?.value]
+    ? [UUIDComponent.entitiesByUUID[editorState.lockPropertiesPanel.value]]
     : (getEntityNodeArrayFromEntities(selectionState.selectedEntities.value) as EntityOrObjectUUID[])
 
   EditorControlFunctions.modifyProperty(affectedNodes, component, properties)

packages/engine/src/avatar/AvatarAnimationSystem.ts

@@ -1,48 +1,46 @@
 import { useEffect } from 'react'
-import { Bone, MathUtils, Vector3 } from 'three'
+import { AxesHelper, Bone, Euler, MathUtils, Quaternion, Vector3 } from 'three'
 
+import { EntityUUID } from '@etherealengine/common/src/interfaces/EntityUUID'
 import { insertionSort } from '@etherealengine/common/src/utils/insertionSort'
-import { defineState, getState } from '@etherealengine/hyperflux'
+import { defineActionQueue, defineState, dispatchAction, getState } from '@etherealengine/hyperflux'
 
 import { Axis } from '../common/constants/Axis3D'
 import { V_000 } from '../common/constants/MathConstants'
+import { proxifyQuaternion } from '../common/proxies/createThreejsProxy'
 import { Engine } from '../ecs/classes/Engine'
 import { Entity } from '../ecs/classes/Entity'
-import {
-  defineQuery,
-  getComponent,
-  getOptionalComponent,
-  hasComponent,
-  removeComponent,
-  setComponent
-} from '../ecs/functions/ComponentFunctions'
+import { defineQuery, getComponent, getOptionalComponent, setComponent } from '../ecs/functions/ComponentFunctions'
+import { removeEntity } from '../ecs/functions/EntityFunctions'
 import { defineSystem } from '../ecs/functions/SystemFunctions'
 import { createPriorityQueue } from '../ecs/PriorityQueue'
+import { NetworkObjectComponent } from '../networking/components/NetworkObjectComponent'
 import { RigidBodyComponent } from '../physics/components/RigidBodyComponent'
+import { addObjectToGroup } from '../scene/components/GroupComponent'
+import { NameComponent } from '../scene/components/NameComponent'
+import { UUIDComponent } from '../scene/components/UUIDComponent'
 import { VisibleComponent } from '../scene/components/VisibleComponent'
 import {
   compareDistance,
   DistanceFromCameraComponent,
   FrustumCullCameraComponent
 } from '../transform/components/DistanceComponents'
+import { TransformComponent } from '../transform/components/TransformComponent'
 import { updateGroupChildren } from '../transform/systems/TransformSystem'
-import { XRLeftHandComponent, XRRightHandComponent } from '../xr/XRComponents'
-import { getCameraMode, isMobileXRHeadset, XRState } from '../xr/XRState'
+import { getCameraMode, isMobileXRHeadset, XRAction, XRState } from '../xr/XRState'
 import { updateAnimationGraph } from './animation/AnimationGraph'
 import { solveHipHeight } from './animation/HipIKSolver'
 import { solveLookIK } from './animation/LookAtIKSolver'
 import { solveTwoBoneIK } from './animation/TwoBoneIKSolver'
 import { AnimationManager } from './AnimationManager'
 import { AnimationComponent } from './components/AnimationComponent'
 import { AvatarAnimationComponent, AvatarRigComponent } from './components/AvatarAnimationComponent'
-import { AvatarArmsTwistCorrectionComponent } from './components/AvatarArmsTwistCorrectionComponent'
-import { AvatarControllerComponent } from './components/AvatarControllerComponent'
 import {
-  AvatarIKTargetsComponent,
-  AvatarLeftArmIKComponent,
-  AvatarRightArmIKComponent
+  AvatarIKTargetComponent,
+  xrTargetHeadSuffix,
+  xrTargetLeftHandSuffix,
+  xrTargetRightHandSuffix
 } from './components/AvatarIKComponents'
-import { AvatarHeadIKComponent } from './components/AvatarIKComponents'
 import { LoopAnimationComponent } from './components/LoopAnimationComponent'
 import { applyInputSourcePoseToIKTargets } from './functions/applyInputSourcePoseToIKTargets'
 
@@ -73,13 +71,6 @@ const _vec = new Vector3()
 //   RightArmTwistAmount: 0.6
 // })
 
-const leftArmQuery = defineQuery([VisibleComponent, AvatarLeftArmIKComponent, AvatarRigComponent])
-const rightArmQuery = defineQuery([VisibleComponent, AvatarRightArmIKComponent, AvatarRigComponent])
-const leftHandQuery = defineQuery([VisibleComponent, XRLeftHandComponent, AvatarRigComponent])
-const rightHandQuery = defineQuery([VisibleComponent, XRRightHandComponent, AvatarRigComponent])
-const headIKQuery = defineQuery([VisibleComponent, AvatarHeadIKComponent, AvatarRigComponent])
-const localHeadIKQuery = defineQuery([VisibleComponent, AvatarHeadIKComponent, AvatarControllerComponent])
-const armsTwistCorrectionQuery = defineQuery([VisibleComponent, AvatarArmsTwistCorrectionComponent, AvatarRigComponent])
 const loopAnimationQuery = defineQuery([
   VisibleComponent,
   LoopAnimationComponent,
@@ -88,6 +79,10 @@ const loopAnimationQuery = defineQuery([
   AvatarRigComponent
 ])
 const avatarAnimationQuery = defineQuery([AnimationComponent, AvatarAnimationComponent, AvatarRigComponent])
+const ikTargetSpawnQueue = defineActionQueue(XRAction.spawnIKTarget.matches)
+const sessionChangedQueue = defineActionQueue(XRAction.sessionChanged.matches)
+
+const ikTargetQuery = defineQuery([AvatarIKTargetComponent])
 
 const minimumFrustumCullDistanceSqr = 5 * 5 // 5 units
 
@@ -107,24 +102,54 @@ const execute = () => {
   const { priorityQueue, sortedTransformEntities } = getState(AvatarAnimationState)
   const { elapsedSeconds, deltaSeconds, localClientEntity, inputSources } = Engine.instance
 
-  if (xrState.sessionActive && localClientEntity && hasComponent(localClientEntity, AvatarIKTargetsComponent)) {
-    const ikTargets = getComponent(localClientEntity, AvatarIKTargetsComponent)
+  for (const action of sessionChangedQueue()) {
+    if (!localClientEntity) continue
+
+    const headUUID = (Engine.instance.userId + xrTargetHeadSuffix) as EntityUUID
+    const leftHandUUID = (Engine.instance.userId + xrTargetLeftHandSuffix) as EntityUUID
+    const rightHandUUID = (Engine.instance.userId + xrTargetRightHandSuffix) as EntityUUID
+
+    const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID]
+    const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID]
+    const ikTargetRightHand = UUIDComponent.entitiesByUUID[rightHandUUID]
+
+    if (ikTargetHead) removeEntity(ikTargetHead)
+    if (ikTargetLeftHand) removeEntity(ikTargetLeftHand)
+    if (ikTargetRightHand) removeEntity(ikTargetRightHand)
+  }
+
+  for (const action of ikTargetSpawnQueue()) {
+    const entity = Engine.instance.getNetworkObject(action.$from, action.networkId)!
+    setComponent(entity, NameComponent, action.$from + '_' + action.handedness)
+    setComponent(entity, AvatarIKTargetComponent, { handedness: action.handedness })
+    addObjectToGroup(entity, new AxesHelper(0.5))
+    setComponent(entity, VisibleComponent)
+  }
+
+  // todo - remove ik targets when session ends
+  if (xrState.sessionActive && localClientEntity) {
     const sources = Array.from(inputSources.values())
     const head = getCameraMode() === 'attached'
     const leftHand = !!sources.find((s) => s.handedness === 'left')
     const rightHand = !!sources.find((s) => s.handedness === 'right')
 
-    if (!head && ikTargets.head) removeComponent(localClientEntity, AvatarHeadIKComponent)
-    if (!leftHand && ikTargets.leftHand) removeComponent(localClientEntity, AvatarLeftArmIKComponent)
-    if (!rightHand && ikTargets.rightHand) removeComponent(localClientEntity, AvatarRightArmIKComponent)
+    const headUUID = (Engine.instance.userId + xrTargetHeadSuffix) as EntityUUID
+    const leftHandUUID = (Engine.instance.userId + xrTargetLeftHandSuffix) as EntityUUID
+    const rightHandUUID = (Engine.instance.userId + xrTargetRightHandSuffix) as EntityUUID
+
+    const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID]
+    const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID]
+    const ikTargetRightHand = UUIDComponent.entitiesByUUID[rightHandUUID]
 
-    if (head && !ikTargets.head) setComponent(localClientEntity, AvatarHeadIKComponent)
-    if (leftHand && !ikTargets.leftHand) setComponent(localClientEntity, AvatarLeftArmIKComponent)
-    if (rightHand && !ikTargets.rightHand) setComponent(localClientEntity, AvatarRightArmIKComponent)
+    if (!head && ikTargetHead) removeEntity(ikTargetHead)
+    if (!leftHand && ikTargetLeftHand) removeEntity(ikTargetLeftHand)
+    if (!rightHand && ikTargetRightHand) removeEntity(ikTargetRightHand)
 
-    ikTargets.head = head
-    ikTargets.leftHand = leftHand
-    ikTargets.rightHand = rightHand
+    if (head && !ikTargetHead) dispatchAction(XRAction.spawnIKTarget({ handedness: 'none', uuid: headUUID }))
+    if (leftHand && !ikTargetLeftHand)
+      dispatchAction(XRAction.spawnIKTarget({ handedness: 'left', uuid: leftHandUUID }))
+    if (rightHand && !ikTargetRightHand)
+      dispatchAction(XRAction.spawnIKTarget({ handedness: 'right', uuid: rightHandUUID }))
   }
 
   /**
@@ -169,10 +194,8 @@ const execute = () => {
    */
 
   const avatarAnimationEntities = avatarAnimationQuery().filter(filterPriorityEntities)
-  const headIKEntities = headIKQuery().filter(filterPriorityEntities)
-  const leftArmEntities = leftArmQuery().filter(filterPriorityEntities)
-  const rightArmEntities = rightArmQuery().filter(filterPriorityEntities)
   const loopAnimationEntities = loopAnimationQuery().filter(filterPriorityEntities)
+  const ikEntities = ikTargetQuery()
 
   for (const entity of avatarAnimationEntities) {
     /**
@@ -235,93 +258,81 @@ const execute = () => {
   }
 
   /**
-   * 3 - Apply avatar IK
+   * 3 - Get IK target pose from WebXR
    */
 
   applyInputSourcePoseToIKTargets()
 
   /**
-   * Apply head IK
+   * 4 - Apply avatar IK
    */
-  for (const entity of headIKEntities) {
-    const ik = getComponent(entity, AvatarHeadIKComponent)
-    if (!ik.target.position.equals(V_000)) {
-      ik.target.updateMatrixWorld(true)
-      const rig = getComponent(entity, AvatarRigComponent).rig
-      ik.target.getWorldDirection(_vec).multiplyScalar(-1)
-      solveHipHeight(entity, ik.target)
-      solveLookIK(rig.Head, _vec, ik.rotationClamp)
-    }
-  }
-
-  /**
-   * Apply left hand IK
-   */
-  for (const entity of leftArmEntities) {
-    const { rig } = getComponent(entity, AvatarRigComponent)
-
-    const ik = getComponent(entity, AvatarLeftArmIKComponent)
+  for (const entity of ikEntities) {
+    /** Filter by priority queue */
+    const networkObject = getComponent(entity, NetworkObjectComponent)
+    const ownerEntity = Engine.instance.getUserAvatarEntity(networkObject.ownerId)
+    if (!Engine.instance.priorityAvatarEntities.has(ownerEntity)) continue
 
+    const transformComponent = getComponent(entity, TransformComponent)
     // If data is zeroed out, assume there is no input and do not run IK
-    if (!ik.target.position.equals(V_000)) {
-      ik.target.updateMatrixWorld(true)
+    if (transformComponent.position.equals(V_000)) continue
+
+    const { rig } = getComponent(ownerEntity, AvatarRigComponent)
+
+    const ikComponent = getComponent(entity, AvatarIKTargetComponent)
+    if (ikComponent.handedness === 'none') {
+      _vec
+        .set(
+          transformComponent.matrix.elements[8],
+          transformComponent.matrix.elements[9],
+          transformComponent.matrix.elements[10]
+        )
+        .normalize() // equivalent to Object3D.getWorldDirection
+      solveHipHeight(ownerEntity, transformComponent.position)
+      solveLookIK(rig.Head, _vec)
+    } else if (ikComponent.handedness === 'left') {
       rig.LeftForeArm.quaternion.setFromAxisAngle(Axis.X, Math.PI * -0.25)
       /** @todo see if this is still necessary */
       rig.LeftForeArm.updateWorldMatrix(false, true)
       solveTwoBoneIK(
         rig.LeftArm,
         rig.LeftForeArm,
         rig.LeftHand,
-        ik.target,
-        ik.hint,
-        ik.targetOffset,
-        ik.targetPosWeight,
-        ik.targetRotWeight,
-        ik.hintWeight
+        transformComponent.position,
+        transformComponent.rotation
       )
-    }
-  }
-
-  /**
-   * Apply right hand IK
-   */
-  for (const entity of rightArmEntities) {
-    const { rig } = getComponent(entity, AvatarRigComponent)
-
-    const ik = getComponent(entity, AvatarRightArmIKComponent)
-
-    if (!ik.target.position.equals(V_000)) {
-      ik.target.updateMatrixWorld(true)
+    } else if (ikComponent.handedness === 'right') {
       rig.RightForeArm.quaternion.setFromAxisAngle(Axis.X, Math.PI * 0.25)
       /** @todo see if this is still necessary */
       rig.RightForeArm.updateWorldMatrix(false, true)
       solveTwoBoneIK(
         rig.RightArm,
         rig.RightForeArm,
         rig.RightHand,
-        ik.target,
-        ik.hint,
-        ik.targetOffset,
-        ik.targetPosWeight,
-        ik.targetRotWeight,
-        ik.hintWeight
+        transformComponent.position,
+        transformComponent.rotation
       )
     }
   }
 
   /**
-   * Since the scene does not automatically update the matricies for all objects,which updates bones,
+   * Since the scene does not automatically update the matricies for all objects, which updates bones,
    * we need to manually do it for Loop Animation Entities
    */
   for (const entity of loopAnimationEntities) updateGroupChildren(entity)
 
+  /** Run debug */
   for (const entity of Engine.instance.priorityAvatarEntities) {
     const avatarRig = getComponent(entity, AvatarRigComponent)
-    if (avatarRig) {
+    if (avatarRig?.helper) {
       avatarRig.rig.Hips.updateWorldMatrix(true, true)
       avatarRig.helper?.updateMatrixWorld(true)
     }
   }
+
+  /** We don't need to ever calculate the matrices for ik targets, so mark them not dirty */
+  for (const entity of ikEntities) {
+    delete TransformComponent.dirtyTransforms[entity]
+  }
 }
 
 const reactor = () => {