Skip to content

Commit

Permalink
Merge pull request #671 from ctrlaltdavid/fix/gltf-blendshapes
Browse files Browse the repository at this point in the history 
Fix and extend glTF blendshapes support
  • Loading branch information
@two-one-five
two-one-five authored Sep 3, 2020
2 parents b9e72b8 + 9fc9b20 commit 7a686d9
Show file tree
Hide file tree
Showing 10 changed files with 209 additions and 210 deletions.
2 changes: 1 addition & 1 deletion interface/src/ModelPackager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ void ModelPackager::populateBasicMapping(QVariantHash& mapping, QString filename
mapping.insert(JOINT_FIELD, joints);

// If there are no blendshape mappings, and we detect that this is likely a mixamo file,
// then we can add the default mixamo to "faceshift" mappings
// then we can add the default mixamo to blendshape mappings.
if (!mapping.contains(BLENDSHAPE_FIELD) && likelyMixamoFile) {
QVariantHash blendshapes;
blendshapes.insertMulti("BrowsD_L", QVariantList() << "BrowsDown_Left" << 1.0);
Expand Down
2 changes: 1 addition & 1 deletion libraries/avatars-renderer/src/avatars-renderer/Head.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ void Head::simulate(float deltaTime) {
updateEyeLookAt();
}

// use data to update fake Faceshift blendshape coefficients
// Use data to update fake blendshape coefficients.
if (getProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation) &&
!getSuppressProceduralAnimationFlag(HeadData::AudioProceduralBlendshapeAnimation)) {

Expand Down
8 changes: 6 additions & 2 deletions libraries/fbx/src/FBXSerializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ HFMModel* FBXSerializer::extractHFMModel(const hifi::VariantHash& mapping, const

QMultiHash<hifi::ByteArray, WeightedIndex> blendshapeIndices;
for (int i = 0;; i++) {
hifi::ByteArray blendshapeName = FACESHIFT_BLENDSHAPES[i];
hifi::ByteArray blendshapeName = BLENDSHAPE_NAMES[i];
if (blendshapeName.isEmpty()) {
break;
}
Expand Down Expand Up @@ -1675,5 +1675,9 @@ HFMModel::Pointer FBXSerializer::read(const hifi::ByteArray& data, const hifi::V

// FBXSerializer's mapping parameter supports the bool "deduplicateIndices," which is passed into FBXSerializer::extractMesh as "deduplicate"

return HFMModel::Pointer(extractHFMModel(mapping, url.toString()));
auto hfmModel = extractHFMModel(mapping, url.toString());

//hfmModel->debugDump();

return HFMModel::Pointer(hfmModel);
}
2 changes: 1 addition & 1 deletion libraries/fbx/src/FST.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ FST* FST::createFSTFromModel(const QString& fstPath, const QString& modelFilePat


// If there are no blendshape mappings, and we detect that this is likely a mixamo file,
// then we can add the default mixamo to "faceshift" mappings
// then we can add the default mixamo to blendshape mappings.
if (likelyMixamoFile) {
QVariantHash blendshapes;
blendshapes.insertMulti("BrowsD_L", QVariantList() << "BrowsDown_Left" << 1.0);
Expand Down
255 changes: 56 additions & 199 deletions libraries/fbx/src/GLTFSerializer.cpp
View file

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions libraries/fbx/src/GLTFSerializer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -851,11 +851,10 @@ class GLTFSerializer : public QObject, public HFMSerializer {
QNetworkReply* request(hifi::URL& url, bool isTest);
bool doesResourceExist(const QString& url);


void setHFMMaterial(HFMMaterial& hfmMat, const GLTFMaterial& material);
HFMTexture getHFMTexture(const GLTFTexture& texture);

void glTFDebugDump();
void hfmDebugDump(const HFMModel& hfmModel);
};

#endif // hifi_GLTFSerializer_h
135 changes: 135 additions & 0 deletions libraries/hfm/src/hfm/HFM.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,3 +208,138 @@ void HFMModel::computeKdops() {
}
}
}

void HFMModel::debugDump() {
qCDebug(modelformat) << "---------------- hfmModel ----------------";
qCDebug(modelformat) << " originalURL =" << originalURL;

qCDebug(modelformat) << " hasSkeletonJoints =" << hasSkeletonJoints;
qCDebug(modelformat) << " offset =" << offset;

qCDebug(modelformat) << " neckPivot = " << neckPivot;

qCDebug(modelformat) << " bindExtents.size() = " << bindExtents.size();
qCDebug(modelformat) << " meshExtents.size() = " << meshExtents.size();

qCDebug(modelformat) << " jointIndices.size() =" << jointIndices.size();
qCDebug(modelformat) << " joints.count() =" << joints.count();

qCDebug(modelformat) << "---------------- Meshes ----------------";
qCDebug(modelformat) << " meshes.count() =" << meshes.count();
qCDebug(modelformat) << " blendshapeChannelNames = " << blendshapeChannelNames;
foreach(HFMMesh mesh, meshes) {
qCDebug(modelformat) << " meshpointer =" << mesh._mesh.get();
qCDebug(modelformat) << " meshindex =" << mesh.meshIndex;
qCDebug(modelformat) << " vertices.count() =" << mesh.vertices.size();
qCDebug(modelformat) << " colors.count() =" << mesh.colors.count();
qCDebug(modelformat) << " normals.count() =" << mesh.normals.size();
qCDebug(modelformat) << " tangents.count() =" << mesh.tangents.size();
qCDebug(modelformat) << " colors.count() =" << mesh.colors.count();
qCDebug(modelformat) << " texCoords.count() =" << mesh.texCoords.count();
qCDebug(modelformat) << " texCoords1.count() =" << mesh.texCoords1.count();
qCDebug(modelformat) << " clusterIndices.count() =" << mesh.clusterIndices.count();
qCDebug(modelformat) << " clusterWeights.count() =" << mesh.clusterWeights.count();
qCDebug(modelformat) << " modelTransform =" << mesh.modelTransform;
qCDebug(modelformat) << " parts.count() =" << mesh.parts.count();

qCDebug(modelformat) << "---------------- Meshes (blendshapes)--------";
foreach(HFMBlendshape bshape, mesh.blendshapes) {
qCDebug(modelformat) << " bshape.indices.count() =" << bshape.indices.count();
qCDebug(modelformat) << " bshape.vertices.count() =" << bshape.vertices.count();
qCDebug(modelformat) << " bshape.normals.count() =" << bshape.normals.count();
}

qCDebug(modelformat) << "---------------- Meshes (meshparts)--------";
foreach(HFMMeshPart meshPart, mesh.parts) {
qCDebug(modelformat) << " quadIndices.count() =" << meshPart.quadIndices.count();
qCDebug(modelformat) << " triangleIndices.count() =" << meshPart.triangleIndices.count();
qCDebug(modelformat) << " materialID =" << meshPart.materialID;
}

qCDebug(modelformat) << "---------------- Meshes (clusters)--------";
qCDebug(modelformat) << " clusters.count() =" << mesh.clusters.count();
foreach(HFMCluster cluster, mesh.clusters) {
qCDebug(modelformat) << " jointIndex =" << cluster.jointIndex;
qCDebug(modelformat) << " inverseBindMatrix =" << cluster.inverseBindMatrix;
}
}

qCDebug(modelformat) << "---------------- AnimationFrames ----------------";
foreach(HFMAnimationFrame anim, animationFrames) {
qCDebug(modelformat) << " anim.translations = " << anim.translations;
qCDebug(modelformat) << " anim.rotations = " << anim.rotations;
}

qCDebug(modelformat) << "---------------- Mesh model names ----------------";
QList<int> mitomona_keys = meshIndicesToModelNames.keys();
foreach(int key, mitomona_keys) {
qCDebug(modelformat) << " meshIndicesToModelNames key =" << key << " val =" << meshIndicesToModelNames[key];
}

qCDebug(modelformat) << "---------------- Materials ----------------";
foreach(HFMMaterial mat, materials) {
qCDebug(modelformat) << " mat.materialID =" << mat.materialID;
qCDebug(modelformat) << " diffuseColor =" << mat.diffuseColor;
qCDebug(modelformat) << " diffuseFactor =" << mat.diffuseFactor;
qCDebug(modelformat) << " specularColor =" << mat.specularColor;
qCDebug(modelformat) << " specularFactor =" << mat.specularFactor;
qCDebug(modelformat) << " emissiveColor =" << mat.emissiveColor;
qCDebug(modelformat) << " emissiveFactor =" << mat.emissiveFactor;
qCDebug(modelformat) << " shininess =" << mat.shininess;
qCDebug(modelformat) << " opacity =" << mat.opacity;
qCDebug(modelformat) << " metallic =" << mat.metallic;
qCDebug(modelformat) << " roughness =" << mat.roughness;
qCDebug(modelformat) << " emissiveIntensity =" << mat.emissiveIntensity;
qCDebug(modelformat) << " ambientFactor =" << mat.ambientFactor;

qCDebug(modelformat) << " materialID =" << mat.materialID;
qCDebug(modelformat) << " name =" << mat.name;
qCDebug(modelformat) << " shadingModel =" << mat.shadingModel;
qCDebug(modelformat) << " _material =" << mat._material.get();

qCDebug(modelformat) << " normalTexture =" << mat.normalTexture.filename;
qCDebug(modelformat) << " albedoTexture =" << mat.albedoTexture.filename;
qCDebug(modelformat) << " opacityTexture =" << mat.opacityTexture.filename;

qCDebug(modelformat) << " lightmapParams =" << mat.lightmapParams;

qCDebug(modelformat) << " isPBSMaterial =" << mat.isPBSMaterial;
qCDebug(modelformat) << " useNormalMap =" << mat.useNormalMap;
qCDebug(modelformat) << " useAlbedoMap =" << mat.useAlbedoMap;
qCDebug(modelformat) << " useOpacityMap =" << mat.useOpacityMap;
qCDebug(modelformat) << " useRoughnessMap =" << mat.useRoughnessMap;
qCDebug(modelformat) << " useSpecularMap =" << mat.useSpecularMap;
qCDebug(modelformat) << " useMetallicMap =" << mat.useMetallicMap;
qCDebug(modelformat) << " useEmissiveMap =" << mat.useEmissiveMap;
qCDebug(modelformat) << " useOcclusionMap =" << mat.useOcclusionMap;
}

qCDebug(modelformat) << "---------------- Joints ----------------";
foreach(HFMJoint joint, joints) {
qCDebug(modelformat) << " shapeInfo.avgPoint =" << joint.shapeInfo.avgPoint;
qCDebug(modelformat) << " shapeInfo.debugLines =" << joint.shapeInfo.debugLines;
qCDebug(modelformat) << " shapeInfo.dots =" << joint.shapeInfo.dots;
qCDebug(modelformat) << " shapeInfo.points =" << joint.shapeInfo.points;

qCDebug(modelformat) << " parentIndex" << joint.parentIndex;
qCDebug(modelformat) << " distanceToParent" << joint.distanceToParent;
qCDebug(modelformat) << " translation" << joint.translation;
qCDebug(modelformat) << " preTransform" << joint.preTransform;
qCDebug(modelformat) << " preRotation" << joint.preRotation;
qCDebug(modelformat) << " rotation" << joint.rotation;
qCDebug(modelformat) << " postRotation" << joint.postRotation;
qCDebug(modelformat) << " postTransform" << joint.postTransform;
qCDebug(modelformat) << " transform" << joint.transform;
qCDebug(modelformat) << " rotationMin" << joint.rotationMin;
qCDebug(modelformat) << " rotationMax" << joint.rotationMax;
qCDebug(modelformat) << " inverseDefaultRotation" << joint.inverseDefaultRotation;
qCDebug(modelformat) << " inverseBindRotation" << joint.inverseBindRotation;
qCDebug(modelformat) << " bindTransform" << joint.bindTransform;
qCDebug(modelformat) << " name" << joint.name;
qCDebug(modelformat) << " isSkeletonJoint" << joint.isSkeletonJoint;
qCDebug(modelformat) << " bindTransformFoundInCluster" << joint.hasGeometricOffset;
qCDebug(modelformat) << " bindTransformFoundInCluster" << joint.geometricTranslation;
qCDebug(modelformat) << " bindTransformFoundInCluster" << joint.geometricRotation;
qCDebug(modelformat) << " bindTransformFoundInCluster" << joint.geometricScaling;
}
}
2 changes: 2 additions & 0 deletions libraries/hfm/src/hfm/HFM.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,8 @@ class Model {
QMap<int, glm::quat> jointRotationOffsets;
std::vector<ShapeVertices> shapeVertices;
FlowData flowData;

void debugDump();
};

};
Expand Down
4 changes: 2 additions & 2 deletions libraries/shared/src/BlendshapeConstants.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

#include "BlendshapeConstants.h"

const char* FACESHIFT_BLENDSHAPES[] = {
const char* BLENDSHAPE_NAMES[] = {
"EyeBlink_L",
"EyeBlink_R",
"EyeSquint_L",
Expand Down Expand Up @@ -80,7 +80,7 @@ const char* FACESHIFT_BLENDSHAPES[] = {
const QMap<QString, int> BLENDSHAPE_LOOKUP_MAP = [] {
QMap<QString, int> toReturn;
for (int i = 0; i < (int)Blendshapes::BlendshapeCount; i++) {
toReturn[FACESHIFT_BLENDSHAPES[i]] = i;
toReturn[BLENDSHAPE_NAMES[i]] = i;
}
return toReturn;
}();
6 changes: 4 additions & 2 deletions libraries/shared/src/BlendshapeConstants.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

#include <glm/glm.hpp>

/// The names of the blendshapes expected by Faceshift, terminated with an empty string.
extern const char* FACESHIFT_BLENDSHAPES[];
/// The names of the supported blendshapes, terminated with an empty string.
extern const char* BLENDSHAPE_NAMES[];
extern const QMap<QString, int> BLENDSHAPE_LOOKUP_MAP;

enum class Blendshapes : int {
Expand Down Expand Up @@ -97,6 +97,8 @@ enum class LegacyBlendshpaes : int {
LegacyBlendshapeCount
};

// Original blendshapes were per Faceshift.

// NEW in ARKit
// * MouthClose
// * MouthUpperUp_L
Expand Down

0 comments on commit 7a686d9

Please sign in to comment.