Initial implementation for parallel VPP output

.gitignore

@@ -16,6 +16,7 @@
 *.toc
 *.snm
 *.csv
+*.csv.*
 *.dylib
 *.so
 *.so.*

framework/doc/content/syntax/VectorPostprocessors/index.md

@@ -76,10 +76,10 @@ getScatterVectorPostprocessorValue('the_vpp_parameter_name', 'the_vector_name')
 `getScatterVectorPostprocessorValue()` returns a `const ScatterVectorPostprocessorValue &`... which is a single scalar value that you don't index into.  Each process receives the "correct" value and can just directly use it.
 
 
-If the data in a VPP is naturally replicated on all processors a VectorPostprocessor should set `_is_broadcast = true` in its `validParams()` like so:
+If the data in a VPP is naturally replicated on all processors a VectorPostprocessor should set `_auto_broadcast = false` in its `validParams()` like so:
 
 ```c++
-params.set<bool>("_is_broadcast") = true;
+params.set<MooseEnum>("_auto_broadcast") = "false";
 ```
 
 This tells MOOSE that the data is already replicated and there is no need to broadcast it if another object is asking for it to be broadcast.

framework/include/outputs/CSV.h

@@ -75,7 +75,9 @@ class CSV : public TableOutput
    * Generates a filename pattern for Vectorpostprocessors
    * filebase + VPP name + time step + ".csv"
    */
-  std::string getVectorPostprocessorFileName(const std::string & vpp_name, bool include_time_step);
+  std::string getVectorPostprocessorFileName(const std::string & vpp_name,
+                                             bool include_time_step,
+                                             bool is_distributed);
 
 private:
   /// Flag for aligning data in .csv file
@@ -106,13 +108,12 @@ class CSV : public TableOutput
   bool _create_latest_symlink;
 
   /// Current list of VPP filenames for creating _LATEST/_FINAL symlinks
-  // The pair is composed of the complete filename (foo_variable_0001.csv) and the incomplete name
-  // (foo_variable) to which the _FINAL or _LATEST is to be applied.
-  std::vector<std::pair<std::string, std::string>> _latest_vpp_filenames;
+  // The pair is composed of the complete filename (foo_variable_0001.csv), the incomplete name
+  // (foo_variable) to which the _FINAL or _LATEST is to be applied, and the "is_distributed" flag
+  std::vector<std::tuple<std::string, std::string, bool>> _latest_vpp_filenames;
 
   /**
    * Returns the filename without the time/timestep information.
    */
   std::string getVectorPostprocessorFilePrefix(const std::string & vpp_name);
 };
-

framework/include/problems/FEProblemBase.h

@@ -914,7 +914,8 @@ class FEProblemBase : public SubProblem, public Restartable
   VectorPostprocessorValue & declareVectorPostprocessorVector(const VectorPostprocessorName & name,
                                                               const std::string & vector_name,
                                                               bool contains_complete_history,
-                                                              bool is_broadcast);
+                                                              bool is_broadcast,
+                                                              bool is_distributed);
 
   /**
    * Whether or not the specified VectorPostprocessor has declared any vectors

framework/include/vectorpostprocessors/VectorPostprocessor.h

@@ -12,6 +12,7 @@
 // MOOSE includes
 #include "MooseTypes.h"
 #include "OutputInterface.h"
+#include "MooseEnum.h"
 
 // libMesh
 #include "libmesh/parallel.h"
@@ -56,6 +57,11 @@ class VectorPostprocessor : public OutputInterface
    */
   bool containsCompleteHistory() const { return _contains_complete_history; }
 
+  /**
+   * Return true if the VPP is operating in distributed mode.
+   */
+  bool isDistributed() const { return _is_distributed; }
+
 protected:
   /**
    * Register a new vector to fill up.
@@ -75,8 +81,11 @@ class VectorPostprocessor : public OutputInterface
 
   const bool _contains_complete_history;
 
+  const MooseEnum & _parallel_type;
+
+  const bool _is_distributed;
+
   const bool _is_broadcast;
 
   std::map<std::string, VectorPostprocessorValue> _thread_local_vectors;
 };
-

framework/include/vectorpostprocessors/VectorPostprocessorData.h

@@ -66,7 +66,8 @@ class VectorPostprocessorData : public Restartable, public libMesh::ParallelObje
   VectorPostprocessorValue & declareVector(const std::string & vpp_name,
                                            const std::string & vector_name,
                                            bool contains_complete_history,
-                                           bool is_broadcast);
+                                           bool is_broadcast,
+                                           bool is_distributed);
 
   /**
    * Returns a true value if the VectorPostprocessor exists
@@ -126,6 +127,11 @@ class VectorPostprocessorData : public Restartable, public libMesh::ParallelObje
    */
   bool containsCompleteHistory(const std::string & name) const;
 
+  /**
+   * Returns a Boolean indicating whether the specified VPP vectors are distributed
+   */
+  bool isDistributed(const std::string & name) const;
+
   /**
    * Get the map of vectors for a particular VectorPostprocessor
    * @param vpp_name The name of the VectorPostprocessor
@@ -151,6 +157,7 @@ class VectorPostprocessorData : public Restartable, public libMesh::ParallelObje
                                                           bool get_current = true,
                                                           bool contains_complete_history = false,
                                                           bool is_broadcast = false,
+                                                          bool is_distributed = false,
                                                           bool needs_broadcast = false,
                                                           bool needs_scatter = false);
   /**
@@ -177,6 +184,9 @@ class VectorPostprocessorData : public Restartable, public libMesh::ParallelObje
 
     /// Boolean indicating whether any old vectors have been requested.
     bool _needs_old;
+
+    /// Flag if data is distributed
+    bool _is_distributed;
   };
 
   /// The VPP data store in a map: VPP Name to vector storage
@@ -185,4 +195,3 @@ class VectorPostprocessorData : public Restartable, public libMesh::ParallelObje
   std::set<std::string> _requested_items;
   std::set<std::string> _supplied_items;
 };
-

framework/src/outputs/CSV.C

@@ -120,7 +120,9 @@ CSV::outputVectorPostprocessors()
 }
 
 std::string
-CSV::getVectorPostprocessorFileName(const std::string & vpp_name, bool include_time_step)
+CSV::getVectorPostprocessorFileName(const std::string & vpp_name,
+                                    bool include_time_step,
+                                    bool is_distributed)
 {
   std::ostringstream file_name;
   file_name << _file_base;
@@ -132,8 +134,15 @@ CSV::getVectorPostprocessorFileName(const std::string & vpp_name, bool include_t
   if (include_time_step)
     file_name << '_' << std::setw(_padding) << std::setprecision(0) << std::setfill('0')
               << std::right << timeStep();
+
   file_name << ".csv";
 
+  if (is_distributed)
+  {
+    int digits = MooseUtils::numDigits(n_processors());
+    file_name << "." << std::setw(digits) << std::setfill('0') << processor_id();
+  }
+
   return file_name.str();
 }
 
@@ -154,7 +163,7 @@ CSV::output(const ExecFlagType & type)
   const auto & vpp_data = _problem_ptr->getVectorPostprocessorData();
 
   // Output each VectorPostprocessor's data to a file
-  if (_write_vector_table && processor_id() == 0)
+  if (_write_vector_table)
   {
 
     // The VPP table will not write the same data twice, so to get the symlinks correct
@@ -172,28 +181,50 @@ CSV::output(const ExecFlagType & type)
         it.second.sortColumns();
 
       auto include_time_suffix = !vpp_data.containsCompleteHistory(vpp_name);
-      std::string fname = getVectorPostprocessorFileName(vpp_name, include_time_suffix);
-      std::string fprefix = getVectorPostprocessorFilePrefix(vpp_name);
-      _latest_vpp_filenames.emplace_back(fname, fprefix);
-      it.second.printCSV(fname, 1, _align);
+      auto is_distributed = vpp_data.isDistributed(vpp_name);
 
-      if (_create_latest_symlink)
+      if (is_distributed || processor_id() == 0)
       {
-        std::string out_latest = fprefix + "_LATEST.csv";
-        MooseUtils::createSymlink(fname, out_latest);
+        std::string fname =
+            getVectorPostprocessorFileName(vpp_name, include_time_suffix, is_distributed);
+        std::string fprefix = getVectorPostprocessorFilePrefix(vpp_name);
+
+        _latest_vpp_filenames.emplace_back(fname, fprefix, is_distributed);
+
+        it.second.printCSV(fname, 1, _align);
+
+        if (_create_latest_symlink)
+        {
+          std::ostringstream out_latest;
+          out_latest << fprefix << "_LATEST.csv";
+          if (is_distributed)
+          {
+            int digits = MooseUtils::numDigits(n_processors());
+            out_latest << "." << std::setw(digits) << std::setfill('0') << processor_id();
+          }
+          MooseUtils::createSymlink(fname, out_latest.str());
+        }
+
+        if (_time_data)
+          _vector_postprocessor_time_tables[vpp_name].printCSV(fprefix + "_time.csv");
       }
-
-      if (_time_data)
-        _vector_postprocessor_time_tables[vpp_name].printCSV(fprefix + "_time.csv");
     }
   }
 
-  if (type == EXEC_FINAL && _create_final_symlink && processor_id() == 0)
+  if (type == EXEC_FINAL && _create_final_symlink)
   {
-    for (const auto & name_pair : _latest_vpp_filenames)
+    for (const auto & name_tuple : _latest_vpp_filenames)
     {
-      std::string out_final = name_pair.second + "_FINAL.csv";
-      MooseUtils::createSymlink(name_pair.first, out_final);
+      std::ostringstream out_final;
+      out_final << std::get<1>(name_tuple) << "_FINAL.csv";
+      if (std::get<2>(name_tuple))
+      {
+        int digits = MooseUtils::numDigits(n_processors());
+        out_final << "." << std::setw(digits) << std::setfill('0') << processor_id();
+        MooseUtils::createSymlink(std::get<0>(name_tuple), out_final.str());
+      }
+      else if (processor_id() == 0)
+        MooseUtils::createSymlink(std::get<0>(name_tuple), out_final.str());
     }
   }
 

framework/src/problems/FEProblemBase.C

@@ -3306,9 +3306,11 @@ VectorPostprocessorValue &
 FEProblemBase::declareVectorPostprocessorVector(const VectorPostprocessorName & name,
                                                 const std::string & vector_name,
                                                 bool contains_complete_history,
-                                                bool is_broadcast)
+                                                bool is_broadcast,
+                                                bool is_distributed)
 {
-  return _vpps_data.declareVector(name, vector_name, contains_complete_history, is_broadcast);
+  return _vpps_data.declareVector(
+      name, vector_name, contains_complete_history, is_broadcast, is_distributed);
 }
 
 const std::vector<std::pair<std::string, VectorPostprocessorData::VectorPostprocessorState>> &

framework/src/vectorpostprocessors/CSVReader.C

@@ -45,7 +45,7 @@ CSVReader::validParams()
 
   // The value from this VPP is naturally already on every processor
   // TODO: Make this not the case!  See #11415
-  params.set<bool>("_is_broadcast") = true;
+  params.set<bool>("_auto_broadcast") = false;
 
   return params;
 }

framework/src/vectorpostprocessors/ElementVariablesDifferenceMax.C

@@ -35,7 +35,7 @@ ElementVariablesDifferenceMax::validParams()
 
   // The value from this VPP is naturally already on every processor
   // TODO: Make this not the case!  See #11415
-  params.set<bool>("_is_broadcast") = true;
+  params.set<bool>("_auto_broadcast") = false;
 
   return params;
 }

framework/src/vectorpostprocessors/SamplerBase.C

@@ -30,7 +30,7 @@ SamplerBase::validParams()
 
   // The value from this VPP is naturally already on every processor
   // TODO: Make this not the case!  See #11415
-  params.set<bool>("_is_broadcast") = true;
+  params.set<bool>("_auto_broadcast") = false;
 
   return params;
 }

framework/src/vectorpostprocessors/VectorOfPostprocessors.C

@@ -26,7 +26,7 @@ VectorOfPostprocessors::validParams()
 
   // The value from this VPP is naturally already on every processor
   // TODO: Make this not the case!  See #11415
-  params.set<bool>("_is_broadcast") = true;
+  params.set<bool>("_auto_broadcast") = false;
 
   return params;
 }

framework/src/vectorpostprocessors/VectorPostprocessor.C

@@ -30,8 +30,21 @@ VectorPostprocessor::validParams()
                         "only a single file is output and updated with each invocation");
 
   // VPPs can set this to true if their resulting vectors are naturally replicated in parallel
-  // setting this to true will keep MOOSE from unnecesarily broadcasting those vectors
-  params.addPrivateParam<bool>("_is_broadcast", false);
+  // setting this to false will keep MOOSE from unnecessarily broadcasting those vectors
+  params.addPrivateParam<bool>("_auto_broadcast", true);
+
+  // VPPs can operate in "distributed" mode, which disables automatic the automatic broadcasting
+  // and results in an individual file per processor if CSV output is enabled
+  MooseEnum parallel_type("DISTRIBUTED REPLICATED", "REPLICATED");
+  params.addParam<MooseEnum>(
+      "parallel_type",
+      parallel_type,
+      "Set how the data is represented within the VectorPostprocessor (VPP); 'distributed' "
+      "indicates that data within the VPP is distributed and no auto communication is preformed, "
+      "this setting will result in parallel output within the CSV output; 'replicated' indicates "
+      "that the data within the VPP is correct on processor 0, the data will automatically be "
+      "broadcast to all processors unless the '_auto_broadcast' param is set to false within the "
+      "validParams function.");
 
   params.addParamNamesToGroup("outputs", "Advanced");
   params.registerBase("VectorPostprocessor");
@@ -44,7 +57,9 @@ VectorPostprocessor::VectorPostprocessor(const InputParameters & parameters)
     _vpp_fe_problem(parameters.getCheckedPointerParam<FEProblemBase *>("_fe_problem_base")),
     _vpp_tid(parameters.isParamValid("_tid") ? parameters.get<THREAD_ID>("_tid") : 0),
     _contains_complete_history(parameters.get<bool>("contains_complete_history")),
-    _is_broadcast(parameters.get<bool>("_is_broadcast"))
+    _parallel_type(parameters.get<MooseEnum>("parallel_type")),
+    _is_distributed(_parallel_type == "DISTRIBUTED"),
+    _is_broadcast(_is_distributed || !parameters.get<bool>("_auto_broadcast"))
 {
 }
 
@@ -61,5 +76,5 @@ VectorPostprocessor::declareVector(const std::string & vector_name)
     return _thread_local_vectors.emplace(vector_name, VectorPostprocessorValue()).first->second;
   else
     return _vpp_fe_problem->declareVectorPostprocessorVector(
-        _vpp_name, vector_name, _contains_complete_history, _is_broadcast);
+        _vpp_name, vector_name, _contains_complete_history, _is_broadcast, _is_distributed);
 }