fix(stage-ui-three): fix the issue VRM model goes too far #642 (#662)

* bug-fix model goes too far

* bug-fix model goes too far gemini-review

* Update packages/stage-ui-three/src/composables/vrm/core.ts

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* bug-fix model goes too far gemini-review

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: Lilia-Chen

packages/stage-ui-three/src/components/Controls/OrbitControls.vue

@@ -27,6 +27,7 @@ import { onMounted, onUnmounted, shallowRef, toRefs, watch } from 'vue'
   * - camera distance: camera position - camera target
 */
 const props = defineProps<{
+  controlEnable: boolean
   modelLoaded: boolean
   modelSize: Vec3
   cameraPosition: Vec3
@@ -48,6 +49,7 @@ const emit = defineEmits<{
 }>()
 
 const {
+  controlEnable,
   modelLoaded,
   modelSize,
   cameraPosition,
@@ -119,6 +121,12 @@ function registerInfoFlow() {
     camera.value.updateProjectionMatrix()
     controls.value.update()
   })
+  watch(controlEnable, (newEnable) => {
+    if (!camera.value || !controls.value)
+      return
+    controls.value.enableRotate = newEnable
+    controls.value.enableZoom = newEnable
+  }, { immediate: true })
 
   /*
     * Upward info flow
@@ -160,6 +168,9 @@ onMounted(async () => {
   camera.value = cameraTres.value as PerspectiveCamera
   // Obtain orbitControl instance
   controls.value = new OrbitControls(camera.value, renderer.domElement)
+  controls.value.enablePan = false
+  controls.value.enableZoom = false
+  controls.value.enableRotate = false
   // Align to tresjs conventions
   controls.value.mouseButtons = {
     LEFT: MOUSE.ROTATE,
@@ -170,7 +181,7 @@ onMounted(async () => {
     ONE: TOUCH.ROTATE,
     TWO: TOUCH.DOLLY_PAN,
   }
-  controls.value.enablePan = false
+
   // define watch props and emit
   registerInfoFlow()
   controls.value.update()

packages/stage-ui-three/src/components/Model/VRMModel.vue

@@ -6,7 +6,7 @@
   * - Load & initialise animation
 */
 
-import type { VRMCore } from '@pixiv/three-vrm-core'
+import type { VRM } from '@pixiv/three-vrm'
 import type {
   Group,
   Object3D,
@@ -29,7 +29,6 @@ import {
   MeshPhysicalMaterial,
   MeshStandardMaterial,
   Plane,
-  Quaternion,
   Raycaster,
 
   SRGBColorSpace,
@@ -146,7 +145,7 @@ const {
 
 // Model and scene ref
 const { scene } = useTresContext()
-const vrm = shallowRef<VRMCore>()
+const vrm = shallowRef<VRM>()
 const vrmGroup = shallowRef<Group>()
 const modelLoaded = ref<boolean>(false)
 // for eye tracking modes
@@ -315,19 +314,7 @@ async function loadModel() {
       }
 
       // Set model facing direction
-      const targetDirection = new Vector3(0, 0, -1) // Default facing direction
-      const lookAt = _vrm.lookAt
-      const quaternion = new Quaternion()
-      if (lookAt) {
-        const facingDirection = lookAt.faceFront
-        quaternion.setFromUnitVectors(facingDirection.normalize(), targetDirection.normalize())
-        _vrmGroup.quaternion.premultiply(quaternion)
-        _vrmGroup.updateMatrixWorld(true)
-      }
-      else {
-        console.warn('No look-at target found in VRM model')
-      }
-
+      // Lilia: I brought forward the rotation to the core.ts, so that any ad-hoc rotation will not impact the model centre position.
       if (isFirstLoad) {
         // Reset model rotation Y
         emit('modelRotationY', 0)
@@ -436,6 +423,7 @@ async function loadModel() {
         idleEyeSaccades.update(vrm.value, lookAtTarget, delta)
         vrmEmote.value?.update(delta)
         vrmLipSync.update(vrm.value)
+        vrm.value?.springBoneManager?.update(delta)
       }).off
 
       // update the 'last model src'

packages/stage-ui-three/src/components/ThreeScene.vue

@@ -132,8 +132,10 @@ function onOrbitControlsReady() {
 
 //  === VRMModel ===
 const modelLoaded = ref<boolean>(false)
+const controlEnable = ref<boolean>(false)
 function onVRMModelLoadStart() {
   modelLoaded.value = false
+  controlEnable.value = false
 }
 function onVRMModelCameraPosition(value: Vec3) {
   cameraPosition.value.x = value.x
@@ -164,6 +166,7 @@ function onVRMModelLookAtTarget(value: Vec3) {
 function onVRMModelLoaded(value: string) {
   lastModelSrc.value = value
   modelLoaded.value = true
+  controlEnable.value = true
 }
 
 // === sky box ===
@@ -289,6 +292,7 @@ defineExpose({
     >
       <OrbitControls
         ref="controlsRef"
+        :control-enable="controlEnable"
         :model-loaded="modelLoaded"
         :model-size="modelSize"
         :camera-position="cameraPosition"

packages/stage-ui-three/src/composables/vrm/animation.ts

@@ -61,7 +61,8 @@ export function reAnchorRootPositionTrack(clip: AnimationClip, _vrm: VRMCore) {
 
   // Calculate the offset from the hips node to the hips's first frame position
   const hipsTrack = clip.tracks.find(track =>
-    track.name.endsWith('Hips.position'),
+    track instanceof VectorKeyframeTrack
+    && track.name === `${hipNode.name}.position`,
   )
   if (!(hipsTrack instanceof VectorKeyframeTrack)) {
     console.warn('No Hips.position track of type VectorKeyframeTrack found in animation.')

packages/stage-ui-three/src/composables/vrm/core.ts

@@ -1,9 +1,9 @@
-import type { VRMCore } from '@pixiv/three-vrm'
-import type { Object3D, Scene } from 'three'
+import type { VRM, VRMCore } from '@pixiv/three-vrm'
+import type { Mesh, Object3D, Scene } from 'three'
 
 import { VRMUtils } from '@pixiv/three-vrm'
 import { VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation'
-import { Box3, Group, Vector3 } from 'three'
+import { Box3, Group, Quaternion, Vector3 } from 'three'
 
 import { useVRMLoader } from './loader'
 
@@ -16,7 +16,7 @@ export async function loadVrm(model: string, options?: {
   lookAt?: boolean
   onProgress?: (progress: ProgressEvent<EventTarget>) => void | Promise<void>
 }): Promise<{
-  _vrm: VRMCore
+  _vrm: VRM
   _vrmGroup: Group
   modelCenter: Vector3
   modelSize: Vector3
@@ -55,7 +55,58 @@ export async function loadVrm(model: string, options?: {
     options.scene.add(_vrmGroup)
   }
 
-  const box = new Box3().setFromObject(_vrm.scene)
+  // Preset the facing direction
+  const targetDirection = new Vector3(0, 0, -1) // Default facing direction
+  const lookAt = _vrm.lookAt
+  const quaternion = new Quaternion()
+  if (lookAt) {
+    const facingDirection = lookAt.faceFront
+    quaternion.setFromUnitVectors(facingDirection.normalize(), targetDirection.normalize())
+    _vrmGroup.quaternion.premultiply(quaternion)
+    _vrmGroup.updateMatrixWorld(true)
+  }
+  else {
+    console.warn('No look-at target found in VRM model')
+  }
+  (_vrm as VRM).springBoneManager?.reset()
+  _vrmGroup.updateMatrixWorld(true)
+
+  function computeBoundingBox(vrm: Object3D) {
+    const box = new Box3()
+    const childBox = new Box3()
+
+    vrm.updateMatrixWorld(true)
+
+    vrm.traverse((obj) => {
+      if (!obj.visible)
+        return
+      const mesh = obj as Mesh
+      if (!mesh.isMesh)
+        return
+      if (!mesh.geometry)
+        return
+      // This traverse mesh console print will be important for future debugging
+      // console.debug("mesh node: ", mesh)
+
+      // Selectively filter out VRM spring bone colliders
+      if (mesh.name.startsWith('VRMC_springBone_collider'))
+        return
+
+      const geometry = mesh.geometry
+      if (!geometry.boundingBox) {
+        geometry.computeBoundingBox()
+      }
+
+      childBox.copy(geometry.boundingBox!)
+      childBox.applyMatrix4(mesh.matrixWorld)
+
+      box.union(childBox)
+    })
+
+    return box
+  }
+
+  const box = computeBoundingBox(_vrm.scene)
   const modelSize = new Vector3()
   const modelCenter = new Vector3()
   box.getSize(modelSize)
@@ -68,7 +119,7 @@ export async function loadVrm(model: string, options?: {
   const radians = (fov / 2 * Math.PI) / 180
   const initialCameraOffset = new Vector3(
     modelSize.x / 16,
-    modelSize.y / 6, // default y value
+    modelSize.y / 8, // default y value
     -(modelSize.y / 3) / Math.tan(radians), // default z value
   )