Implementing Derek's comments for performance improvement (idaholab#5913

)

framework/include/base/Assembly.h

@@ -347,9 +347,10 @@ class Assembly
    * @param var The variable that needs to have it's datastructures prepared
    */
   void prepareVariable(MooseVariable * var);
+  void prepareVariableNonlocal(MooseVariable * var);
   void prepareNeighbor();
   void prepareBlock(unsigned int ivar, unsigned jvar, const std::vector<dof_id_type> & dof_indices);
-  void prepareBlockNonlocal(unsigned int ivar, unsigned jvar, const std::vector<dof_id_type> & dof_indices);
+  void prepareBlockNonlocal(unsigned int ivar, unsigned jvar, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices);
   void prepareScalar();
   void prepareOffDiagScalar();
 
@@ -398,7 +399,7 @@ class Assembly
   void addJacobian(SparseMatrix<Number> & jacobian);
   void addJacobianNonlocal(SparseMatrix<Number> & jacobian);
   void addJacobianBlock(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices);
-  void addJacobianBlockNonlocal(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices);
+  void addJacobianBlockNonlocal(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices);
   void addJacobianNeighbor(SparseMatrix<Number> & jacobian);
   void addJacobianNeighbor(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices, std::vector<dof_id_type> & neighbor_dof_indices);
   void addJacobianScalar(SparseMatrix<Number> & jacobian);
@@ -434,7 +435,7 @@ class Assembly
   DenseMatrix<Number> & jacobianBlockNonlocal(unsigned int ivar, unsigned int jvar);
   DenseMatrix<Number> & jacobianBlockNeighbor(Moose::DGJacobianType type, unsigned int ivar, unsigned int jvar);
   void cacheJacobianBlock(DenseMatrix<Number> & jac_block, std::vector<dof_id_type> & idof_indices, std::vector<dof_id_type> & jdof_indices, Real scaling_factor);
-  void cacheJacobianBlockNonlocal(DenseMatrix<Number> & jac_block, std::vector<dof_id_type> & idof_indices, std::vector<dof_id_type> & jdof_indices, Real scaling_factor);
+  void cacheJacobianBlockNonlocal(DenseMatrix<Number> & jac_block, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices, Real scaling_factor);
 
   std::vector<std::pair<MooseVariable *, MooseVariable *> > & couplingEntries() { return _cm_entry; }
 

framework/include/base/ComputeUserObjectsThread.h

@@ -60,8 +60,6 @@ class ComputeUserObjectsThread : public ThreadedElementLoop<ConstElemRange>
   const MooseObjectWarehouse<SideUserObject> & _side_user_objects;
   const MooseObjectWarehouse<InternalSideUserObject> & _internal_side_user_objects;
   ///@}
-
-  std::vector<MooseVariable *> _jacobian_moose_vars;
 };
 
 #endif //COMPUTEUSEROBJECTSTHREAD_H

framework/include/base/DisplacedProblem.h

@@ -125,7 +125,7 @@ class DisplacedProblem : public SubProblem
   virtual void prepareNonlocal(THREAD_ID tid);
   virtual void prepareFace(const Elem * elem, THREAD_ID tid) override;
   virtual void prepare(const Elem * elem, unsigned int ivar, unsigned int jvar, const std::vector<dof_id_type> & dof_indices, THREAD_ID tid) override;
-  virtual void prepareBlockNonlocal(unsigned int ivar, unsigned int jvar, const std::vector<dof_id_type> & dof_indices, THREAD_ID tid);
+  virtual void prepareBlockNonlocal(unsigned int ivar, unsigned int jvar, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices, THREAD_ID tid);
   virtual void prepareAssembly(THREAD_ID tid) override;
   virtual void prepareAssemblyNeighbor(THREAD_ID tid);
 
@@ -163,7 +163,7 @@ class DisplacedProblem : public SubProblem
   virtual void addJacobianNonlocal(SparseMatrix<Number> & jacobian, THREAD_ID tid);
   virtual void addJacobianNeighbor(SparseMatrix<Number> & jacobian, THREAD_ID tid) override;
   virtual void addJacobianBlock(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices, THREAD_ID tid) override;
-  virtual void addJacobianBlockNonlocal(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices, THREAD_ID tid);
+  virtual void addJacobianBlockNonlocal(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices, THREAD_ID tid);
   virtual void addJacobianNeighbor(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices, std::vector<dof_id_type> & neighbor_dof_indices, THREAD_ID tid) override;
 
   virtual void cacheJacobian(THREAD_ID tid) override;

framework/include/base/FEProblem.h

@@ -38,6 +38,7 @@
 class DisplacedProblem;
 class FEProblem;
 class MooseMesh;
+class SystemBase;
 class NonlinearSystem;
 class RandomInterface;
 class RandomData;
@@ -260,6 +261,10 @@ class FEProblem :
    * @return Flag indicating nonlocal coupling exists or not.
    */
   void checkNonlocalCoupling();
+  void checkUserObjectJacobianRequirement(THREAD_ID tid);
+  void setVariableAllDoFMap(const std::vector<MooseVariable *> moose_vars);
+
+  const std::vector<MooseVariable *> & getUserObjectJacobianVariables(THREAD_ID tid) const { return _uo_jacobian_moose_vars[tid]; }
 
   virtual Assembly & assembly(THREAD_ID tid) override { return *_assembly[tid]; }
 
@@ -1202,8 +1207,10 @@ class FEProblem :
   bool _has_jacobian;
 
   /// Indicates if nonlocal coupling is required/exists
-  bool _requires_nonlocal_coupling;
   bool _has_nonlocal_coupling;
+  bool _calculate_jacobian_in_uo;
+
+  std::vector<std::vector<MooseVariable *> > _uo_jacobian_moose_vars;
 
   SolverParams _solver_params;
 

framework/include/base/MooseVariableBase.h

@@ -77,8 +77,8 @@ class MooseVariableBase
   /**
    * Get all global dofindices for the variable
    */
-  std::vector<dof_id_type> & allDofIndices();
-  unsigned int totalVarDofs() { return _all_dof_indices.size(); }
+  const std::vector<dof_id_type> & allDofIndices() const;
+  unsigned int totalVarDofs() { return allDofIndices().size(); }
 
   /**
    * Kind of the variable (Nonlinear, Auxiliary, ...)
@@ -138,8 +138,6 @@ class MooseVariableBase
   const DofMap & _dof_map;
   /// DOF indices
   std::vector<dof_id_type> _dof_indices;
-
-  std::vector<dof_id_type> _all_dof_indices;
   /// mesh the variable is active in
   MooseMesh & _mesh;
   /// scaling factor for this variable

framework/include/base/NonlinearSystem.h

@@ -230,6 +230,8 @@ class NonlinearSystem : public SystemTempl<TransientNonlinearImplicitSystem>,
    */
   void constraintJacobians(SparseMatrix<Number> & jacobian, bool displaced);
 
+  const std::vector<dof_id_type> & getVariableGlobalDoFs(const std::string & var_name);
+
   /**
    * Computes Jacobian
    * @param jacobian Jacobian is formed in here
@@ -624,6 +626,8 @@ class NonlinearSystem : public SystemTempl<TransientNonlinearImplicitSystem>,
   bool _has_nodalbc_diag_save_in;
 
   void getNodeDofs(unsigned int node_id, std::vector<dof_id_type> & dofs);
+
+  std::vector<dof_id_type> _var_all_dof_indices;
 };
 
 #endif /* NONLINEARSYSTEM_H */

framework/include/base/SubProblem.h

@@ -54,6 +54,8 @@ class SubProblem : public Problem
   virtual EquationSystems & es() = 0;
   virtual MooseMesh & mesh() = 0;
 
+  virtual bool checkNonlocalCouplingRequirement() { return _requires_nonlocal_coupling; }
+
   /**
    * Whether or not this problem should utilize FE shape function caching.
    *
@@ -317,6 +319,8 @@ class SubProblem : public Problem
    */
   virtual void registerRestartableData(std::string name, RestartableDataValue * data, THREAD_ID tid);
 
+  std::map<std::string, std::vector<dof_id_type> > _var_dof_map;
+
 protected:
   /// The Factory for building objects
   Factory & _factory;
@@ -356,6 +360,9 @@ class SubProblem : public Problem
   /* This needs to remain <unsigned int> for threading purposes */
   std::vector<unsigned int> _has_active_elemental_moose_variables;
 
+  /// nonlocal coupling requirement flag
+  bool _requires_nonlocal_coupling;
+
   /// Elements that should have Dofs ghosted to the local processor
   std::set<dof_id_type> _ghosted_elems;
 

framework/include/kernels/NonlocalKernel.h

@@ -36,7 +36,6 @@ class NonlocalKernel :
   virtual Real computeQpNonlocalJacobian(dof_id_type dof_index);
   virtual Real computeQpNonlocalOffDiagJacobian(unsigned int jvar, dof_id_type dof_index);
 
-  DenseMatrix<Number> _nonlocal_ke;
   unsigned int _k;
 };
 

framework/include/userobject/ShapeElementUserObject.h

@@ -36,10 +36,13 @@ class ShapeElementUserObject : public ElementUserObject
 public:
   ShapeElementUserObject(const InputParameters & parameters);
 
+  /// check if jacobian is to be computed in user objects
+  const bool & computeJacobianFlag() const { return _compute_jacobians; }
+
   /**
    * Returns the set of variables a Jacobian has been requested for
    */
-  const std::set<MooseVariable *> & jacobianMooseVariables() { return _jacobian_moose_variables; }
+  const std::set<MooseVariable *> & jacobianMooseVariables() const { return _jacobian_moose_variables; }
 
   /**
    * This function will be called with the shape functions for jvar initialized. It
@@ -74,9 +77,6 @@ class ShapeElementUserObject : public ElementUserObject
   /// global DOF ID corresponding to _j
   dof_id_type _j_global;
 
-  /// set to true iff the current call of the user object is for the purpose of calculating Jacobians
-  // const bool _currently_computing_jacobian;
-
 private:
   const bool _compute_jacobians;
   std::set<MooseVariable *> _jacobian_moose_variables;

framework/src/base/Assembly.C

@@ -988,12 +988,16 @@ Assembly::prepareNonlocal()
     MooseVariable & ivar = *(it.first);
     MooseVariable & jvar = *(it.second);
 
-    unsigned int vi = ivar.number();
-    unsigned int vj = jvar.number();
+    const auto map_it = _sys.subproblem()._var_dof_map.find(jvar.name());
+    if (map_it != _sys.subproblem()._var_dof_map.end())
+    {
+      unsigned int vi = ivar.number();
+      unsigned int vj = jvar.number();
 
-    jacobianBlockNonlocal(vi,vj).resize(ivar.dofIndices().size(), _dof_map.n_dofs());
-    jacobianBlockNonlocal(vi,vj).zero();
-    _jacobian_block_nonlocal_used[vi][vj] = 0;
+      jacobianBlockNonlocal(vi,vj).resize(ivar.dofIndices().size(), jvar.allDofIndices().size());
+      jacobianBlockNonlocal(vi,vj).zero();
+      _jacobian_block_nonlocal_used[vi][vj] = 0;
+    }
   }
 }
 
@@ -1009,9 +1013,32 @@ Assembly::prepareVariable(MooseVariable * var)
     unsigned int vj = jvar.number();
 
     if (vi == var->number() || vj == var->number())
-    {
       jacobianBlock(vi,vj).resize(ivar.dofIndices().size(), jvar.dofIndices().size());
-      jacobianBlockNonlocal(vi,vj).resize(ivar.dofIndices().size(), _dof_map.n_dofs());
+  }
+
+  for (unsigned int i = 0; i < _sub_Re.size(); i++)
+  {
+    _sub_Re[i][var->number()].resize(var->dofIndices().size());
+    _sub_Re[i][var->number()].zero();
+  }
+}
+
+void
+Assembly::prepareVariableNonlocal(MooseVariable * var)
+{
+  for (const auto & it : _cm_entry)
+  {
+    MooseVariable & ivar = *(it.first);
+    MooseVariable & jvar = *(it.second);
+
+    const auto map_it = _sys.subproblem()._var_dof_map.find(jvar.name());
+    if (map_it != _sys.subproblem()._var_dof_map.end())
+    {
+      unsigned int vi = ivar.number();
+      unsigned int vj = jvar.number();
+
+      if (vi == var->number() || vj == var->number())
+        jacobianBlockNonlocal(vi,vj).resize(ivar.dofIndices().size(), jvar.allDofIndices().size());
     }
   }
 
@@ -1069,9 +1096,9 @@ Assembly::prepareBlock(unsigned int ivar, unsigned int jvar, const std::vector<d
 }
 
 void
-Assembly::prepareBlockNonlocal(unsigned int ivar, unsigned int jvar, const std::vector<dof_id_type> & dof_indices)
+Assembly::prepareBlockNonlocal(unsigned int ivar, unsigned int jvar, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices)
 {
-  jacobianBlockNonlocal(ivar,jvar).resize(dof_indices.size(), _dof_map.n_dofs());
+  jacobianBlockNonlocal(ivar,jvar).resize(idof_indices.size(), jdof_indices.size());
   jacobianBlockNonlocal(ivar,jvar).zero();
   _jacobian_block_nonlocal_used[ivar][jvar] = 0;
 }
@@ -1399,7 +1426,7 @@ Assembly::cacheJacobianBlock(DenseMatrix<Number> & jac_block, std::vector<dof_id
 }
 
 void
-Assembly::cacheJacobianBlockNonlocal(DenseMatrix<Number> & jac_block, std::vector<dof_id_type> & idof_indices, std::vector<dof_id_type> & jdof_indices, Real scaling_factor)
+Assembly::cacheJacobianBlockNonlocal(DenseMatrix<Number> & jac_block, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices, Real scaling_factor)
 {
   if ((idof_indices.size() > 0) && (jdof_indices.size() > 0) && jac_block.n() && jac_block.m())
   {
@@ -1477,14 +1504,15 @@ Assembly::addJacobian(SparseMatrix<Number> & jacobian)
 void
 Assembly::addJacobianNonlocal(SparseMatrix<Number> & jacobian)
 {
-  std::vector<dof_id_type> jdof_indices(_dof_map.n_dofs());
-  for (unsigned int i = 0; i < _dof_map.n_dofs(); ++i)
-    jdof_indices[i] = i;
   const std::vector<MooseVariable *> & vars = _sys.getVariables(_tid);
   for (const auto & ivar : vars)
     for (const auto & jvar : vars)
-      if ((*_cm)(ivar->number(), jvar->number()) != 0 && _jacobian_block_nonlocal_used[ivar->number()][jvar->number()])
-        addJacobianBlock(jacobian, jacobianBlockNonlocal(ivar->number(), jvar->number()), ivar->dofIndices(), jdof_indices, ivar->scalingFactor());
+    {
+      const auto it = _sys.subproblem()._var_dof_map.find(jvar->name());
+      if (it != _sys.subproblem()._var_dof_map.end())
+        if ((*_cm)(ivar->number(), jvar->number()) != 0 && _jacobian_block_nonlocal_used[ivar->number()][jvar->number()])
+          addJacobianBlock(jacobian, jacobianBlockNonlocal(ivar->number(), jvar->number()), ivar->dofIndices(), jvar->allDofIndices(), ivar->scalingFactor());
+    }
 }
 
 void
@@ -1549,14 +1577,15 @@ Assembly::cacheJacobian()
 void
 Assembly::cacheJacobianNonlocal()
 {
-  std::vector<dof_id_type> jdof_indices(_dof_map.n_dofs());
-  for (unsigned int i = 0; i < _dof_map.n_dofs(); ++i)
-    jdof_indices[i] = i;
   const std::vector<MooseVariable *> & vars = _sys.getVariables(_tid);
   for (const auto & ivar : vars)
     for (const auto & jvar : vars)
-      if ((*_cm)(ivar->number(), jvar->number()) != 0 && _jacobian_block_nonlocal_used[ivar->number()][jvar->number()])
-        cacheJacobianBlockNonlocal(jacobianBlockNonlocal(ivar->number(), jvar->number()), ivar->dofIndices(), jdof_indices, ivar->scalingFactor());
+    {
+      const auto it = _sys.subproblem()._var_dof_map.find(jvar->name());
+      if (it != _sys.subproblem()._var_dof_map.end())
+        if ((*_cm)(ivar->number(), jvar->number()) != 0 && _jacobian_block_nonlocal_used[ivar->number()][jvar->number()])
+          cacheJacobianBlockNonlocal(jacobianBlockNonlocal(ivar->number(), jvar->number()), ivar->dofIndices(), jvar->allDofIndices(), ivar->scalingFactor());
+    }
 }
 
 void
@@ -1594,14 +1623,12 @@ Assembly::addJacobianBlock(SparseMatrix<Number> & jacobian, unsigned int ivar, u
 }
 
 void
-Assembly::addJacobianBlockNonlocal(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices)
+Assembly::addJacobianBlockNonlocal(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, const std::vector<dof_id_type> & idof_indices, const std::vector<dof_id_type> & jdof_indices)
 {
   DenseMatrix<Number> & keg = jacobianBlockNonlocal(ivar, jvar);
 
-  std::vector<dof_id_type> di(dof_indices);
-  std::vector<dof_id_type> dg(_dof_map.n_dofs());
-  for (unsigned int i = 0; i < _dof_map.n_dofs(); ++i)
-    dg[i] = i;
+  std::vector<dof_id_type> di(idof_indices);
+  std::vector<dof_id_type> dg(jdof_indices);
   dof_map.constrain_element_matrix(keg, di, dg, false);
 
   Real scaling_factor = _sys.getVariable(_tid, ivar).scalingFactor();

framework/src/base/ComputeUserObjectsThread.C

@@ -65,25 +65,6 @@ ComputeUserObjectsThread::subdomainChanged()
 
   _fe_problem.setActiveElementalMooseVariables(needed_moose_vars, _tid);
   _fe_problem.prepareMaterials(_subdomain, _tid);
-
-  // ShapeElementUserObject requested Jacobian variables
-  if (_elemental_user_objects.hasActiveBlockObjects(_subdomain, _tid))
-  {
-    std::set<MooseVariable *> jacobian_moose_vars;
-
-    const std::vector<MooseSharedPointer<ElementUserObject> > & e_objects = _elemental_user_objects.getActiveBlockObjects(_subdomain, _tid);
-    for (std::vector<MooseSharedPointer<ElementUserObject> >::const_iterator it = e_objects.begin(); it != e_objects.end(); ++it)
-    {
-      MooseSharedPointer<ShapeElementUserObject> shape_element_uo = MooseSharedNamespace::dynamic_pointer_cast<ShapeElementUserObject>(*it);
-      if (shape_element_uo)
-      {
-        const std::set<MooseVariable *> & mv_deps = shape_element_uo->jacobianMooseVariables();
-        jacobian_moose_vars.insert(mv_deps.begin(), mv_deps.end());
-      }
-
-      _jacobian_moose_vars.assign(jacobian_moose_vars.begin(), jacobian_moose_vars.end());
-    }
-  }
 }
 
 void
@@ -102,26 +83,28 @@ ComputeUserObjectsThread::onElement(const Elem * elem)
 
   // UserObject Jacobians
   if (_fe_problem.currentlyComputingJacobian())
-  {
-    // Prepare shape functions for ShapeElementUserObjects
-    for (std::vector<MooseVariable *>::const_iterator jvar_it = _jacobian_moose_vars.begin();
-         jvar_it != _jacobian_moose_vars.end();
-         ++jvar_it)
+    if (_elemental_user_objects.hasActiveBlockObjects(_subdomain, _tid))
     {
-      unsigned int jvar = (*jvar_it)->number();
-      std::vector<dof_id_type> & dof_indices = (*jvar_it)->dofIndices();
-
-      _fe_problem.prepareShapes(jvar, _tid);
-
-      const std::vector<MooseSharedPointer<ElementUserObject> > & e_objects = _elemental_user_objects.getActiveBlockObjects(_subdomain, _tid);
-      for (std::vector<MooseSharedPointer<ElementUserObject> >::const_iterator it = e_objects.begin(); it != e_objects.end(); ++it)
+      // Prepare shape functions for ShapeElementUserObjects
+      std::vector<MooseVariable *> jacobian_moose_vars = _fe_problem.getUserObjectJacobianVariables(_tid);
+      for (auto jvar_it = jacobian_moose_vars.begin();
+           jvar_it != jacobian_moose_vars.end();
+           ++jvar_it)
       {
-        MooseSharedPointer<ShapeElementUserObject> shape_element_uo = MooseSharedNamespace::dynamic_pointer_cast<ShapeElementUserObject>(*it);
-        if (shape_element_uo)
-          shape_element_uo->executeJacobianWrapper(jvar, dof_indices);
+        unsigned int jvar = (*jvar_it)->number();
+        std::vector<dof_id_type> & dof_indices = (*jvar_it)->dofIndices();
+
+        _fe_problem.prepareShapes(jvar, _tid);
+
+        const std::vector<MooseSharedPointer<ElementUserObject> > & e_objects = _elemental_user_objects.getActiveBlockObjects(_subdomain, _tid);
+        for (const auto & uo : e_objects)
+        {
+          MooseSharedPointer<ShapeElementUserObject> shape_element_uo = MooseSharedNamespace::dynamic_pointer_cast<ShapeElementUserObject>(uo);
+          if (shape_element_uo)
+            shape_element_uo->executeJacobianWrapper(jvar, dof_indices);
+        }
       }
     }
-  }
 
   _fe_problem.swapBackMaterials(_tid);
 }