fixup! Threaded general user objects

idaholab · Aug 7, 2018 · c50d6da · c50d6da
1 parent df06e41
commit c50d6da
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 24 deletions.
diff --git a/framework/include/loops/ComputeThreadedGeneralUserObjectsThread.h b/framework/include/loops/ComputeThreadedGeneralUserObjectsThread.h
@@ -15,7 +15,9 @@
 #include "MooseException.h"
 #include "GeneralUserObject.h"
 
-typedef StoredRange<std::vector<THREAD_ID>::iterator, THREAD_ID> ThreadIdRange;
+typedef StoredRange<std::vector<std::shared_ptr<GeneralUserObject>>::iterator,
+                    std::shared_ptr<GeneralUserObject>>
+    GeneralUserObjectRange;
 
 /**
  * This mutex is used to protect the creation of the strings used in the propogation
@@ -34,14 +36,13 @@ static Threads::spin_mutex threaded_general_user_objects_mutex;
 class ComputeThreadedGeneralUserObjectsThread
 {
 public:
-  ComputeThreadedGeneralUserObjectsThread(
-      FEProblemBase & fe_problem, const MooseObjectWarehouse<GeneralUserObject> & user_objects);
+  ComputeThreadedGeneralUserObjectsThread(FEProblemBase & fe_problem);
   ComputeThreadedGeneralUserObjectsThread(ComputeThreadedGeneralUserObjectsThread & x,
                                           Threads::split split);
 
   virtual ~ComputeThreadedGeneralUserObjectsThread();
 
-  void operator()(const ThreadIdRange & range);
+  void operator()(const GeneralUserObjectRange & range);
 
   void join(const ComputeThreadedGeneralUserObjectsThread & /*y*/) {}
 
@@ -54,8 +55,6 @@ class ComputeThreadedGeneralUserObjectsThread
 protected:
   /// FEProblem running this thread
   FEProblemBase & _fe_problem;
-  /// Storage for threaded GeneralUserObjects (see FEProblemBase::computeUserObjects)
-  const MooseObjectWarehouse<GeneralUserObject> & _user_objects;
 };
 
 #endif // COMPUTETHREADEDGENERALUSEROBJECTSTHREAD_H
diff --git a/framework/src/loops/ComputeThreadedGeneralUserObjectsThread.C b/framework/src/loops/ComputeThreadedGeneralUserObjectsThread.C
@@ -10,14 +10,14 @@
 #include "ComputeThreadedGeneralUserObjectsThread.h"
 
 ComputeThreadedGeneralUserObjectsThread::ComputeThreadedGeneralUserObjectsThread(
-    FEProblemBase & fe_problem, const MooseObjectWarehouse<GeneralUserObject> & user_objects)
-  : _fe_problem(fe_problem), _user_objects(user_objects)
+    FEProblemBase & fe_problem)
+  : _fe_problem(fe_problem)
 {
 }
 
 ComputeThreadedGeneralUserObjectsThread::ComputeThreadedGeneralUserObjectsThread(
     ComputeThreadedGeneralUserObjectsThread & x, Threads::split /*split*/)
-  : _fe_problem(x._fe_problem), _user_objects(x._user_objects)
+  : _fe_problem(x._fe_problem)
 {
 }
 
@@ -33,16 +33,14 @@ ComputeThreadedGeneralUserObjectsThread::caughtMooseException(MooseException & e
 }
 
 void
-ComputeThreadedGeneralUserObjectsThread::operator()(const ThreadIdRange & range)
+ComputeThreadedGeneralUserObjectsThread::operator()(const GeneralUserObjectRange & range)
 {
   try
   {
     for (auto it = range.begin(); it != range.end(); ++it)
     {
-      THREAD_ID tid = *it;
-      const auto & objects = _user_objects.getActiveObjects(tid);
-      for (const auto & obj : objects)
-        obj->execute();
+      auto & tguo = *it;
+      tguo->execute();
     }
   }
   catch (MooseException & e)

diff --git a/framework/src/problems/FEProblemBase.C b/framework/src/problems/FEProblemBase.C
@@ -2996,21 +2996,48 @@ FEProblemBase::computeUserObjects(const ExecFlagType & type, const Moose::AuxGro
   // Finalize, threadJoin, and update PP values of Nodal
   finalizeUserObjects<NodalUserObject>(nodal);
 
-  // Initialize threaded GeneralUserObjects
-  initializeUserObjects<GeneralUserObject>(threaded_general);
 
   if (threaded_general.hasActiveObjects())
   {
-    std::vector<THREAD_ID> thread_ids(libMesh::n_threads());
-    for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
-      thread_ids[tid] = tid;
+    for (std::size_t i = 0; i < threaded_general.getActiveObjects(0).size(); ++i)
+    {
+      std::vector<std::shared_ptr<GeneralUserObject>> tguos(libMesh::n_threads());
+      for (THREAD_ID tid = 0; tid < libMesh::n_threads(); ++tid)
+        tguos[tid] = threaded_general.getActiveObjects(tid)[i];
 
-    ComputeThreadedGeneralUserObjectsThread ctguot(*this, threaded_general);
-    Threads::parallel_reduce(ThreadIdRange(thread_ids.begin(), thread_ids.end()), ctguot);
-  }
+      initializeUserObjects<GeneralUserObject>(threaded_general);
+
+      for (auto & object : tguos)
+        object->initialize();
 
-  // Finalize threaded GeneralUserObjects
-  finalizeUserObjects<GeneralUserObject>(threaded_general);
+      ComputeThreadedGeneralUserObjectsThread ctguot(*this);
+      Threads::parallel_reduce(GeneralUserObjectRange(tguos.begin(), tguos.end()), ctguot);
+
+      // Join threaded user objects down to thread 0
+      const auto & object = tguos[0];
+      for (THREAD_ID tid = 1; tid < libMesh::n_threads(); ++tid)
+        object->threadJoin(*(tguos[tid]));
+
+      // Finalize them and save off PP values
+      std::set<std::string> vpps_finalized;
+      for (auto & object : tguos)
+      {
+        object->finalize();
+
+        auto pp = std::dynamic_pointer_cast<Postprocessor>(object);
+        if (pp)
+          _pps_data.storeValue(pp->PPName(), pp->getValue());
+
+        auto vpp = std::dynamic_pointer_cast<VectorPostprocessor>(object);
+        if (vpp)
+          vpps_finalized.insert(vpp->PPName());
+      }
+
+      // Broadcast/Scatter any VPPs that need it
+      for (auto & vpp_name : vpps_finalized)
+        _vpps_data.broadcastScatterVectors(vpp_name);
+    }
+  }
 
   // Execute GeneralUserObjects
   if (general.hasActiveObjects())