UPBGE: Fix modifier deformer normal recalculation.

Before if an object was using a modifier and armature deformation
the deformer BL_ModifierDeformer was created and during an Update
the armature mesh was first updated. But the function responsible
to update the armature mesh is also recalculating the normals. But
the derived mesh can have an higher number of vertices and so the
update of normals was based on an incorrect number of vertices which
induced memory overflow.

When updating a derived mesh, then normals don't need to be updated.
In this case BL_SkinDeformer::UpdateInternal should informs
BGEDeformVerts or BlenderDeformVerts to not update normals.
This is achieve by introducing a "recalcNormal" argument to functions
UpdateInternal in BL_SkinDeformer and BL_ShapeDeformer levels.
When the update is directly proceeded from one of these classes,
recalcNormal is to true, if they are updated through top BL_ModifierDeformer
recalcNormal is to false.

Fix issue: #862.

source/gameengine/Converter/BL_ModifierDeformer.cpp

@@ -117,7 +117,7 @@ bool BL_ModifierDeformer::HasArmatureDeformer(Object *ob)
 
 bool BL_ModifierDeformer::Update(void)
 {
-	bool bShapeUpdate = BL_ShapeDeformer::Update();
+	bool bShapeUpdate = BL_ShapeDeformer::UpdateInternal(false);
 
 	if (bShapeUpdate || m_lastModifierUpdate != m_lastFrame) {
 		// static derived mesh are not updated

source/gameengine/Converter/BL_ShapeDeformer.cpp

@@ -132,7 +132,7 @@ bool BL_ShapeDeformer::ExecuteShapeDrivers()
 	return false;
 }
 
-bool BL_ShapeDeformer::Update()
+bool BL_ShapeDeformer::UpdateInternal(bool recalcNormal)
 {
 	bool bShapeUpdate = false;
 	bool bSkinUpdate = false;
@@ -173,7 +173,7 @@ bool BL_ShapeDeformer::Update()
 		bShapeUpdate = true;
 	}
 	// check for armature deform
-	bSkinUpdate = BL_SkinDeformer::UpdateInternal(bShapeUpdate && m_bDynamic);
+	bSkinUpdate = BL_SkinDeformer::UpdateInternal(bShapeUpdate && m_bDynamic, recalcNormal);
 
 	// non dynamic deformer = Modifer without armature and shape keys, no need to create storage
 	if (!bSkinUpdate && bShapeUpdate && m_bDynamic) {
@@ -185,6 +185,11 @@ bool BL_ShapeDeformer::Update()
 	return bSkinUpdate;
 }
 
+bool BL_ShapeDeformer::Update()
+{
+	return UpdateInternal(true);
+}
+
 Key *BL_ShapeDeformer::GetKey()
 {
 	return m_key;

source/gameengine/Converter/BL_ShapeDeformer.h

@@ -54,7 +54,8 @@ class BL_ShapeDeformer : public BL_SkinDeformer
 
 	virtual ~BL_ShapeDeformer();
 
-	bool Update();
+	bool UpdateInternal(bool recalcNormal);
+	virtual bool Update();
 	bool LoadShapeDrivers(KX_GameObject *parent);
 	bool ExecuteShapeDrivers();
 

source/gameengine/Converter/BL_SkinDeformer.cpp

@@ -116,7 +116,7 @@ void BL_SkinDeformer::Apply(RAS_DisplayArray *array)
 	}
 }
 
-void BL_SkinDeformer::BlenderDeformVerts()
+void BL_SkinDeformer::BlenderDeformVerts(bool recalcNormal)
 {
 	float obmat[4][4];  // the original object matrix
 	Object *par_arma = m_armobj->GetArmatureObject();
@@ -131,10 +131,12 @@ void BL_SkinDeformer::BlenderDeformVerts()
 	// restore matrix
 	copy_m4_m4(m_objMesh->obmat, obmat);
 
-	RecalcNormals();
+	if (recalcNormal) {
+		RecalcNormals();
+	}
 }
 
-void BL_SkinDeformer::BGEDeformVerts()
+void BL_SkinDeformer::BGEDeformVerts(bool recalcNormal)
 {
 	Object *par_arma = m_armobj->GetArmatureObject();
 	MDeformVert *dverts = m_bmesh->dvert;
@@ -166,20 +168,18 @@ void BL_SkinDeformer::BGEDeformVerts()
 	MDeformWeight *dw;
 
 	for (int i = 0; i < m_bmesh->totvert; ++i, dv++) {
+		if (!dv->totweight) {
+			continue;
+		}
+
 		float contrib = 0.0f, weight, max_weight = -1.0f;
 		bPoseChannel *pchan = nullptr;
-		Eigen::Vector3f normorg(m_bmesh->mvert[i].no[0], m_bmesh->mvert[i].no[1], m_bmesh->mvert[i].no[2]);
-		Eigen::Map<Eigen::Vector3f> norm = Eigen::Vector3f::Map(m_transnors[i].data);
 		Eigen::Vector4f vec(0.0f, 0.0f, 0.0f, 1.0f);
 		Eigen::Vector4f co(m_transverts[i].x,
 		                   m_transverts[i].y,
 		                   m_transverts[i].z,
 		                   1.0f);
 
-		if (!dv->totweight) {
-			continue;
-		}
-
 		co = pre_mat * co;
 
 		dw = dv->dw;
@@ -207,8 +207,12 @@ void BL_SkinDeformer::BGEDeformVerts()
 			}
 		}
 
-		// Update Vertex Normal
-		norm = norm_chan_mat.topLeftCorner<3, 3>() * normorg;
+		if (recalcNormal) {
+			const Eigen::Vector3f normorg(m_bmesh->mvert[i].no[0], m_bmesh->mvert[i].no[1], m_bmesh->mvert[i].no[2]);
+			Eigen::Map<Eigen::Vector3f> norm = Eigen::Vector3f::Map(m_transnors[i].data);
+			// Update Vertex Normal
+			norm = norm_chan_mat.topLeftCorner<3, 3>() * normorg;
+		}
 
 		co.noalias() += vec / contrib;
 		co[3] = 1.0f; // Make sure we have a 1 for the w component!
@@ -266,7 +270,7 @@ void BL_SkinDeformer::UpdateTransverts()
 	}
 }
 
-bool BL_SkinDeformer::UpdateInternal(bool shape_applied)
+bool BL_SkinDeformer::UpdateInternal(bool shape_applied, bool recalcNormal)
 {
 	/* See if the armature has been updated for this frame */
 	if (PoseUpdated()) {
@@ -278,10 +282,10 @@ bool BL_SkinDeformer::UpdateInternal(bool shape_applied)
 		m_armobj->ApplyPose();
 
 		if (m_armobj->GetVertDeformType() == ARM_VDEF_BGE_CPU) {
-			BGEDeformVerts();
+			BGEDeformVerts(recalcNormal);
 		}
 		else {
-			BlenderDeformVerts();
+			BlenderDeformVerts(recalcNormal);
 		}
 
 		/* Update the current frame */
@@ -301,5 +305,5 @@ bool BL_SkinDeformer::UpdateInternal(bool shape_applied)
 
 bool BL_SkinDeformer::Update(void)
 {
-	return UpdateInternal(false);
+	return UpdateInternal(false, true);
 }

source/gameengine/Converter/BL_SkinDeformer.h

@@ -59,8 +59,8 @@ class BL_SkinDeformer : public BL_MeshDeformer
 					BL_ArmatureObject *arma);
 
 	virtual ~BL_SkinDeformer();
-	bool Update();
-	bool UpdateInternal(bool shape_applied);
+	virtual bool Update();
+	bool UpdateInternal(bool shape_applied, bool recalcNormal);
 	virtual void Apply(RAS_DisplayArray *array);
 	virtual void UpdateBuckets()
 	{
@@ -88,8 +88,8 @@ class BL_SkinDeformer : public BL_MeshDeformer
 	std::vector<bPoseChannel *> m_dfnrToPC;
 	short m_deformflags;
 
-	void BlenderDeformVerts();
-	void BGEDeformVerts();
+	void BlenderDeformVerts(bool recalcNormal);
+	void BGEDeformVerts(bool recalcNormal);
 
 	virtual void UpdateTransverts();
 };