Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a multiplayer example #8

Open
msub2 opened this issue Mar 7, 2023 · 1 comment
Open

Add a multiplayer example #8

msub2 opened this issue Mar 7, 2023 · 1 comment
Labels
documentation Improvements or additions to documentation

Comments

@msub2
Copy link
Owner

msub2 commented Mar 7, 2023

Could maybe leverage networked-aframe for it

@msub2 msub2 added the documentation Improvements or additions to documentation label Mar 7, 2023
@vincentfretin
Copy link

You would need to create a component to sync all bones position and quaternion, and interpolate the position and rotation between two received states.
I did that for a proof of concept with mediapipe and the old AvatarSDK FitPerson (SMPL skeleton, not mixamo), only interpolating neck and arms with https://github.com/InfiniteLee/buffered-interpolation the same dependency that networked-aframe has. I don't have a runnable example anymore, and I don't have the bandwidth to create one, but here is a component that I did, if you want some inspiration.

AFRAME.registerComponent("avatar-bones", {
  schema: {
    neckQuat: {type: "vec4"},
    spine2Quat: {type: "vec4"},
    spine3Quat: {type: "vec4"},
    lShoulderQuat: {type: "vec4"},
    rShoulderQuat: {type: "vec4"},
    lElbowQuat: {type: "vec4"},
    rElbowQuat: {type: "vec4"},
    lWristQuat: {type: "vec4"},
    rWristQuat: {type: "vec4"},
    lindex0: {type: "vec4"},
    lindex1: {type: "vec4"},
    lindex2: {type: "vec4"},
    lmiddle0: {type: "vec4"},
    lmiddle1: {type: "vec4"},
    lmiddle2: {type: "vec4"},
    lpinky0: {type: "vec4"},
    lpinky1: {type: "vec4"},
    lpinky2: {type: "vec4"},
    lring0: {type: "vec4"},
    lring1: {type: "vec4"},
    lring2: {type: "vec4"},
    lthumb0: {type: "vec4"},
    lthumb1: {type: "vec4"},
    lthumb2: {type: "vec4"},
    rindex0: {type: "vec4"},
    rindex1: {type: "vec4"},
    rindex2: {type: "vec4"},
    rmiddle0: {type: "vec4"},
    rmiddle1: {type: "vec4"},
    rmiddle2: {type: "vec4"},
    rpinky0: {type: "vec4"},
    rpinky1: {type: "vec4"},
    rpinky2: {type: "vec4"},
    rring0: {type: "vec4"},
    rring1: {type: "vec4"},
    rring2: {type: "vec4"},
    rthumb0: {type: "vec4"},
    rthumb1: {type: "vec4"},
    rthumb2: {type: "vec4"},
  },

  init: function() {
    if (this.el.id !== "cameraRig") {
      this.neckBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.spine2Buffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.spine3Buffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.lShoulderBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.rShoulderBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.lElbowBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.rElbowBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.lWristBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
      this.rWristBuffer = new InterpolationBuffer(InterpolationBuffer.MODE_LERP, 0.6);
    }
    this.tempQuaternion = new THREE.Quaternion();
  },

  update: function(oldData) {
    if (!this.mesh) {
      this.mesh = this.el.object3D.getObjectByName('mesh_1');
      if (this.mesh) {
        const bones = this.mesh.skeleton.bones;
        const getBoneByName = THREE.SkeletonUtils.getBoneByName;
        this.neck = getBoneByName("Neck", bones);
        this.spine2 = getBoneByName("Spine2", bones);
        this.spine3 = getBoneByName("Spine3", bones);
        this.lShoulder = getBoneByName("L_Shoulder", bones);
        this.rShoulder = getBoneByName("R_Shoulder", bones);
        this.lElbow = getBoneByName("L_Elbow", bones);
        this.rElbow = getBoneByName("R_Elbow", bones);
        this.lWrist = getBoneByName("L_Wrist", bones);
        this.rWrist = getBoneByName("R_Wrist", bones);
        this.lindex0 = getBoneByName("lindex0", bones);
        this.lindex1 = getBoneByName("lindex1", bones);
        this.lindex2 = getBoneByName("lindex2", bones);
        this.lmiddle0 = getBoneByName("lmiddle0", bones);
        this.lmiddle1 = getBoneByName("lmiddle1", bones);
        this.lmiddle2 = getBoneByName("lmiddle2", bones);
        this.lpinky0 = getBoneByName("lpinky0", bones);
        this.lpinky1 = getBoneByName("lpinky1", bones);
        this.lpinky2 = getBoneByName("lpinky2", bones);
        this.lring0 = getBoneByName("lring0", bones);
        this.lring1 = getBoneByName("lring1", bones);
        this.lring2 = getBoneByName("lring2", bones);
        this.lthumb0 = getBoneByName("lthumb0", bones);
        this.lthumb1 = getBoneByName("lthumb1", bones);
        this.lthumb2 = getBoneByName("lthumb2", bones);
        this.rindex0 = getBoneByName("lindex0", bones);
        this.rindex1 = getBoneByName("rindex1", bones);
        this.rindex2 = getBoneByName("rindex2", bones);
        this.rmiddle0 = getBoneByName("rmiddle0", bones);
        this.rmiddle1 = getBoneByName("rmiddle1", bones);
        this.rmiddle2 = getBoneByName("rmiddle2", bones);
        this.rpinky0 = getBoneByName("rpinky0", bones);
        this.rpinky1 = getBoneByName("rpinky1", bones);
        this.rpinky2 = getBoneByName("rpinky2", bones);
        this.rring0 = getBoneByName("rring0", bones);
        this.rring1 = getBoneByName("rring1", bones);
        this.rring2 = getBoneByName("rring2", bones);
        this.rthumb0 = getBoneByName("rthumb0", bones);
        this.rthumb1 = getBoneByName("rthumb1", bones);
        this.rthumb2 = getBoneByName("rthumb2", bones);
      }
    }

    //if (NAF.utils.isMine(this.el)) {
    if (this.el.id === "cameraRig") {
//      console.log(this.data.neckQuat);
//      if (this.neck) this.neck.quaternion.copy(this.data.neckQuat);
    } else {
      this.neckBuffer.setQuaternion(this.data.neckQuat);
      this.spine2Buffer.setQuaternion(this.data.spine2Quat);
      this.spine3Buffer.setQuaternion(this.data.spine3Quat);
      this.lShoulderBuffer.setQuaternion(this.data.lShoulderQuat);
      this.rShoulderBuffer.setQuaternion(this.data.rShoulderQuat);
      this.lElbowBuffer.setQuaternion(this.data.lElbowQuat);
      this.rElbowBuffer.setQuaternion(this.data.rElbowQuat);
      this.lWristBuffer.setQuaternion(this.data.lWristQuat);
      this.rWristBuffer.setQuaternion(this.data.rWristQuat);
      if (this.lindex0) {
        this.lindex0.quaternion.set(this.data.lindex0.x, this.data.lindex0.y, this.data.lindex0.z, this.data.lindex0.w);
        this.lindex1.quaternion.set(this.data.lindex1.x, this.data.lindex1.y, this.data.lindex1.z, this.data.lindex1.w);
        this.lindex2.quaternion.set(this.data.lindex2.x, this.data.lindex2.y, this.data.lindex2.z, this.data.lindex2.w);
        this.lmiddle0.quaternion.set(this.data.lmiddle0.x, this.data.lmiddle0.y, this.data.lmiddle0.z, this.data.lmiddle0.w);
        this.lmiddle1.quaternion.set(this.data.lmiddle1.x, this.data.lmiddle1.y, this.data.lmiddle1.z, this.data.lmiddle1.w);
        this.lmiddle2.quaternion.set(this.data.lmiddle2.x, this.data.lmiddle2.y, this.data.lmiddle2.z, this.data.lmiddle2.w);
        this.lpinky0.quaternion.set(this.data.lpinky0.x, this.data.lpinky0.y, this.data.lpinky0.z, this.data.lpinky0.w);
        this.lpinky1.quaternion.set(this.data.lpinky1.x, this.data.lpinky1.y, this.data.lpinky1.z, this.data.lpinky1.w);
        this.lpinky2.quaternion.set(this.data.lpinky2.x, this.data.lpinky2.y, this.data.lpinky2.z, this.data.lpinky2.w);
        this.lring0.quaternion.set(this.data.lring0.x, this.data.lring0.y, this.data.lring0.z, this.data.lring0.w);
        this.lring1.quaternion.set(this.data.lring1.x, this.data.lring1.y, this.data.lring1.z, this.data.lring1.w);
        this.lring2.quaternion.set(this.data.lring2.x, this.data.lring2.y, this.data.lring2.z, this.data.lring2.w);
        this.lthumb0.quaternion.set(this.data.lthumb0.x, this.data.lthumb0.y, this.data.lthumb0.z, this.data.lthumb0.w);
        this.lthumb1.quaternion.set(this.data.lthumb1.x, this.data.lthumb1.y, this.data.lthumb1.z, this.data.lthumb1.w);
        this.lthumb2.quaternion.set(this.data.lthumb2.x, this.data.lthumb2.y, this.data.lthumb2.z, this.data.lthumb2.w);
        this.rindex0.quaternion.set(this.data.rindex0.x, this.data.rindex0.y, this.data.rindex0.z, this.data.rindex0.w);
        this.rindex1.quaternion.set(this.data.rindex1.x, this.data.rindex1.y, this.data.rindex1.z, this.data.rindex1.w);
        this.rindex2.quaternion.set(this.data.rindex2.x, this.data.rindex2.y, this.data.rindex2.z, this.data.rindex2.w);
        this.rmiddle0.quaternion.set(this.data.rmiddle0.x, this.data.rmiddle0.y, this.data.rmiddle0.z, this.data.rmiddle0.w);
        this.rmiddle1.quaternion.set(this.data.rmiddle1.x, this.data.rmiddle1.y, this.data.rmiddle1.z, this.data.rmiddle1.w);
        this.rmiddle2.quaternion.set(this.data.rmiddle2.x, this.data.rmiddle2.y, this.data.rmiddle2.z, this.data.rmiddle2.w);
        this.rpinky0.quaternion.set(this.data.rpinky0.x, this.data.rpinky0.y, this.data.rpinky0.z, this.data.rpinky0.w);
        this.rpinky1.quaternion.set(this.data.rpinky1.x, this.data.rpinky1.y, this.data.rpinky1.z, this.data.rpinky1.w);
        this.rpinky2.quaternion.set(this.data.rpinky2.x, this.data.rpinky2.y, this.data.rpinky2.z, this.data.rpinky2.w);
        this.rring0.quaternion.set(this.data.rring0.x, this.data.rring0.y, this.data.rring0.z, this.data.rring0.w);
        this.rring1.quaternion.set(this.data.rring1.x, this.data.rring1.y, this.data.rring1.z, this.data.rring1.w);
        this.rring2.quaternion.set(this.data.rring2.x, this.data.rring2.y, this.data.rring2.z, this.data.rring2.w);
        this.rthumb0.quaternion.set(this.data.rthumb0.x, this.data.rthumb0.y, this.data.rthumb0.z, this.data.rthumb0.w);
        this.rthumb1.quaternion.set(this.data.rthumb1.x, this.data.rthumb1.y, this.data.rthumb1.z, this.data.rthumb1.w);
        this.rthumb2.quaternion.set(this.data.rthumb2.x, this.data.rthumb2.y, this.data.rthumb2.z, this.data.rthumb2.w);
      }
    }
  },

  tick: function(time, dt) {
    //if (!NAF.utils.isMine(this.el)) {
    if (this.el.id !== "cameraRig") {
      this.neckBuffer.update(dt);
      this.spine2Buffer.update(dt);
      this.spine3Buffer.update(dt);
      this.lShoulderBuffer.update(dt);
      this.rShoulderBuffer.update(dt);
      this.lElbowBuffer.update(dt);
      this.rElbowBuffer.update(dt);
      this.lWristBuffer.update(dt);
      this.rWristBuffer.update(dt);
      if (this.neck) {
        const quaternion = this.neckBuffer.getQuaternion();
        this.neck.quaternion.copy(quaternion);
      }
      if (this.spine2) {
        const quaternion = this.spine2Buffer.getQuaternion();
        this.spine2.quaternion.copy(quaternion);
      }
      if (this.spine3) {
        const quaternion = this.spine3Buffer.getQuaternion();
        this.spine3.quaternion.copy(quaternion);
      }
      if (this.lShoulder) {
        const quaternion = this.lShoulderBuffer.getQuaternion();
        this.lShoulder.quaternion.copy(quaternion);
      }
      if (this.rShoulder) {
        const quaternion = this.rShoulderBuffer.getQuaternion();
        this.rShoulder.quaternion.copy(quaternion);
      }
      if (this.lElbow) {
        const quaternion = this.lElbowBuffer.getQuaternion();
        this.lElbow.quaternion.copy(quaternion);
      }
      if (this.rElbow) {
        const quaternion = this.rElbowBuffer.getQuaternion();
        this.rElbow.quaternion.copy(quaternion);
      }
      if (this.lWrist) {
        const quaternion = this.lWristBuffer.getQuaternion();
        this.lWrist.quaternion.copy(quaternion);
      }
      if (this.rWrist) {
        const quaternion = this.rWristBuffer.getQuaternion();
        this.rWrist.quaternion.copy(quaternion);
      }
    }
  },
});

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
documentation Improvements or additions to documentation
Projects
None yet
Development

No branches or pull requests

2 participants