Merge pull request #10623 from rwcarlsen/split-mesh

implement convenience workflow for using distributed mesh

docs/content/documentation/systems/Mesh/index.md

@@ -48,6 +48,27 @@ processor, it can be split prior to the simulation.
     element assembly and solve. In both types the solution data is distributed, which is the portion of the simulation
     that usually dominates memory demands.
 
+## Mesh splitting
+
+For large meshes, MOOSE provides the ability to pre-split a mesh for use in the the "distributed"
+format/mode.  To split and use a mesh for distributed runs:
+
+```
+// For input files with a file-based mesh:
+$ moose-app-opt -i your_input-file.i --split-mesh 500,1000,2000 // comma-separated list of split configurations
+Splitting 500 ways...
+    - writing 500 files per process...
+Splitting 1000 ways...
+    - writing 1000 files per process...
+...
+
+// MOOSE automatically selects the pre-split mesh configuration based on MPI procs
+$ mpiexec -n 1000 moose-app-opt -i your_input-file.i --use-split
+```
+
+For more details see "[Mesh Splitting](/Mesh/splitting.md)".
+
+
 ## Displaced Mesh
 
 Calculations can take place in either the initial mesh configuration or, when requested, the "displaced"

docs/content/documentation/systems/Mesh/splitting.md

@@ -0,0 +1,69 @@
+
+# Mesh Splitting
+
+MOOSE provides the ability to pre-split a mesh into several chunks for use with
+[distributed mesh](/Mesh/index.md#Replicated-and-Distributed-Mesh).
+This can be useful if the whole mesh is too big to fit into memory. A split-mesh
+workflow involves generating the mesh split configuration(s) and then telling moose
+to use them when you run simulations.  This is described in the sections below.
+
+## Generating Split Configurations
+
+You can run MOOSE with the `--split-mesh <split1>[,<split2>]...` flag which takes a
+comma-separated list of describing each split with an integer number of chunks to split the mesh
+into.
+
+```
+$ moose-app-opt -i your_input.i --split-mesh 13,42,1000
+Splitting 13 ways...
+    - writing 13 files per process...
+Splitting 42 ways...
+    - writing 42 files per process...
+...
+```
+
+This will create three split configurations for your mesh that include 13, 42, and 1000 chunks
+each.  If your input file uses the mesh file `foo.e`, this process will generate a directory
+named `foo.cpr`.  In general, you should neither rename this directory nor modify its
+contents.  All split configurations for the mesh will be stored in this directory (even across
+independent splitting operations).
+
+For non-file-based mesh cases (e.g. GeneratedMesh), you will need to tell MOOSE what name to use
+for the split mesh files it will generate using the `--split-file <file_name>` flag; in general,
+you should use the `.cpr` file extension here. For example:
+
+```
+$ moose-app-opt -i your_input.i --split-mesh 42 --split-file foo.cpr
+```
+
+You can also run the splitting operation itself in parallel on several processors to speed up the
+operation:
+
+```
+$ mpiexec -n 9 -i moose-app-opt your_input.i --split-mesh 42
+Splitting 42 ways...
+    - writing 5 files per process...
+```
+
+This will generate the same split configuration as the 42 chunk split generated by the first
+command but just generates it in parallel.
+
+## Using Split Meshes
+
+To use a mesh split configuration use the `--use-split` flag (which takes no arguments):
+
+```
+$ mpiexec -n 42 moose-app-opt your_input.i --use-split
+```
+
+This will cause MOOSE to look for a mesh split configuration with 42 chunks.  If one exists, MOOSE
+will use it running in a distributed mesh mode, otherwise an error will occur.  Note that you do
+not need to modify your input file - MOOSE automatically switches to using the pre-split
+distributed mesh regardless of the mesh type specified in the input file.  If your input file does
+not use a file-based mesh, you will need to specify a split mesh file name using the
+`--split-file <file_name>` flag just as you did when splitting the mesh:
+
+```
+$ mpiexec -n 42 moose-app-opt your_input.i --use-split --split-file foo.cpr
+```
+

framework/include/actions/SplitMeshAction.h

@@ -0,0 +1,30 @@
+//* This file is part of the MOOSE framework
+//* https://www.mooseframework.org
+//*
+//* All rights reserved, see COPYRIGHT for full restrictions
+//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
+//*
+//* Licensed under LGPL 2.1, please see LICENSE for details
+//* https://www.gnu.org/licenses/lgpl-2.1.html
+
+#ifndef SPLITMESHACTION_H
+#define SPLITMESHACTION_H
+
+#include "Action.h"
+
+#include <string>
+
+class SplitMeshAction;
+
+template <>
+InputParameters validParams<SplitMeshAction>();
+
+class SplitMeshAction : public Action
+{
+public:
+  SplitMeshAction(InputParameters params);
+
+  virtual void act() override;
+};
+
+#endif // SPLITMESHACTION_H

framework/include/base/MooseApp.h

@@ -252,6 +252,8 @@ class MooseApp : public ConsoleStreamInterface, public libMesh::ParallelObject
    */
   Factory & getFactory() { return _factory; }
 
+  processor_id_type processor_id() { return cast_int<processor_id_type>(_comm->rank()); }
+
   /**
    * Retrieve the ActionFactory associated with this App.
    */

framework/include/mesh/FileMesh.h

@@ -34,7 +34,7 @@ class FileMesh : public MooseMesh
 
   // Get/Set Filename (for meshes read from a file)
   void setFileName(const std::string & file_name) { _file_name = file_name; }
-  const std::string & getFileName() const { return _file_name; }
+  virtual std::string getFileName() const override { return _file_name; }
 
 protected:
   /// the file_name from whence this mesh came

framework/include/mesh/MooseMesh.h

@@ -812,6 +812,12 @@ class MooseMesh : public MooseObject, public Restartable
    */
   virtual std::unique_ptr<PointLocatorBase> getPointLocator() const;
 
+  /**
+   * Returns the name of the mesh file read to produce this mesh if any or an empty string
+   * otherwise.
+   */
+  virtual std::string getFileName() const { return ""; }
+
 protected:
   /// Deprecated (DO NOT USE)
   std::vector<std::unique_ptr<GhostingFunctor>> _ghosting_functors;
@@ -966,9 +972,6 @@ class MooseMesh : public MooseObject, public Restartable
   /// The patch update strategy
   Moose::PatchUpdateType _patch_update_strategy;
 
-  /// file_name iff this mesh was read from a file
-  std::string _file_name;
-
   /// Vector of all the Nodes in the mesh for determining when to add a new point
   std::vector<Node *> _node_map;
 

framework/include/mesh/TiledMesh.h

@@ -27,6 +27,8 @@ class TiledMesh : public MooseMesh
 
   virtual void buildMesh() override;
 
+  virtual std::string getFileName() const override;
+
 protected:
   const Real _x_width;
   const Real _y_width;

framework/include/utils/MooseUtils.h

@@ -110,6 +110,12 @@ void serialEnd(const libMesh::Parallel::Communicator & comm);
  */
 bool hasExtension(const std::string & filename, std::string ext, bool strip_exodus_ext = false);
 
+/**
+ * Removes any file extension from the fiven string s (i.e. any ".[extension]" suffix of s) and
+ * returns the result.
+ */
+std::string stripExtension(const std::string & s);
+
 /**
  * Function for splitting path and filename
  * @param full_file A complete filename and path

framework/src/actions/SetupMeshAction.C

@@ -10,6 +10,7 @@
 #include "SetupMeshAction.h"
 #include "MooseApp.h"
 #include "MooseMesh.h"
+#include "FileMesh.h"
 #include "FEProblem.h"
 #include "ActionWarehouse.h"
 #include "Factory.h"
@@ -155,6 +156,42 @@ SetupMeshAction::act()
   // Create the mesh object and tell it to build itself
   if (_current_task == "setup_mesh")
   {
+    // switch non-file meshes to be a file-mesh if using a pre-split mesh configuration.
+    if (_app.parameters().get<bool>("use_split"))
+    {
+      auto split_file = _app.parameters().get<std::string>("split_file");
+      if (split_file != "")
+        split_file = MooseUtils::stripExtension(split_file) + ".cpr";
+
+      if (_type != "FileMesh")
+      {
+        if (split_file == "")
+        {
+          if (_app.processor_id() == 0)
+            mooseError(
+                "Cannot use split mesh for a non-file mesh without specifying --split-file on "
+                "command line");
+          MOOSE_ABORT;
+        }
+
+        _type = "FileMesh";
+        auto new_pars = validParams<FileMesh>();
+        new_pars.set<MeshFileName>("file") = split_file;
+        new_pars.set<MooseApp *>("_moose_app") = _moose_object_pars.get<MooseApp *>("_moose_app");
+        new_pars.set<MooseEnum>("parallel_type") = "distributed";
+        _moose_object_pars = new_pars;
+      }
+      else
+      {
+        _moose_object_pars.set<MooseEnum>("parallel_type") = "distributed";
+        if (split_file != "")
+          _moose_object_pars.set<MeshFileName>("file") = split_file;
+        else
+          _moose_object_pars.set<MeshFileName>("file") =
+              MooseUtils::stripExtension(_moose_object_pars.get<MeshFileName>("file")) + ".cpr";
+      }
+    }
+
     _mesh = _factory.create<MooseMesh>(_type, "mesh", _moose_object_pars);
     if (isParamValid("displacements"))
       _displaced_mesh = _factory.create<MooseMesh>(_type, "displaced_mesh", _moose_object_pars);

framework/src/actions/SplitMeshAction.C

@@ -0,0 +1,56 @@
+//* This file is part of the MOOSE framework
+//* https://www.mooseframework.org
+//*
+//* All rights reserved, see COPYRIGHT for full restrictions
+//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
+//*
+//* Licensed under LGPL 2.1, please see LICENSE for details
+//* https://www.gnu.org/licenses/lgpl-2.1.html
+
+#include "SplitMeshAction.h"
+
+#include "MooseApp.h"
+#include "MooseUtils.h"
+#include "MooseMesh.h"
+#include "libmesh/checkpoint_io.h"
+
+template <>
+InputParameters
+validParams<SplitMeshAction>()
+{
+  return validParams<Action>();
+}
+
+SplitMeshAction::SplitMeshAction(InputParameters params) : Action(params) {}
+
+void
+SplitMeshAction::act()
+{
+  auto mesh = _app.actionWarehouse().mesh();
+
+  if (mesh->getFileName() == "" && _app.parameters().get<std::string>("split_file") == "")
+    mooseError("Output mesh file name must be specified (with --split-file) when splitting "
+               "non-file-based meshes");
+
+  auto splitstr = _app.parameters().get<std::string>("split_mesh");
+  std::vector<unsigned int> splits;
+  bool success = MooseUtils::tokenizeAndConvert(splitstr, splits, ", ");
+  if (!success)
+    mooseError("invalid argument for --split-mesh: '", splitstr, "'");
+
+  for (std::size_t i = 0; i < splits.size(); i++)
+  {
+    processor_id_type n = splits[i];
+    Moose::out << "Splitting " << n << " ways..." << std::endl;
+
+    auto cpr = libMesh::split_mesh(*mesh, n);
+    Moose::out << "    - writing " << cpr->current_processor_ids().size() << " files per process..."
+               << std::endl;
+    cpr->binary() = true;
+    auto fname = mesh->getFileName();
+    if (fname == "")
+      fname = _app.parameters().get<std::string>("split_file");
+    fname = MooseUtils::stripExtension(fname) + ".cpr";
+    cpr->write(fname);
+  }
+}

framework/src/base/Moose.C

@@ -449,6 +449,7 @@
 #include "MaterialDerivativeTestAction.h"
 #include "AddRelationshipManager.h"
 #include "MeshOnlyAction.h"
+#include "SplitMeshAction.h"
 
 // Outputs
 #ifdef LIBMESH_HAVE_EXODUS_API
@@ -1159,6 +1160,7 @@ registerActions(Syntax & syntax, ActionFactory & action_factory)
 
   registerAction(SetupPostprocessorDataAction, "setup_postprocessor_data");
 
+  registerAction(SplitMeshAction, "split_mesh");
   registerAction(MeshOnlyAction, "mesh_only");
   registerAction(SetupMeshAction, "setup_mesh");
   registerAction(SetupMeshAction, "init_mesh");

framework/src/base/MooseApp.C

@@ -40,6 +40,7 @@
 #include "libmesh/exodusII_io.h"
 #include "libmesh/mesh_refinement.h"
 #include "libmesh/string_to_enum.h"
+#include "libmesh/checkpoint_io.h"
 
 // System include for dynamic library methods
 #include <dlfcn.h>
@@ -131,6 +132,19 @@ validParams<MooseApp>()
       false,
       "The libMesh Mesh underlying MooseMesh should always be a DistributedMesh");
 
+  params.addCommandLineParam<std::string>(
+      "split_mesh",
+      "--split-mesh [splits]",
+      "comma-separated list of numbers of chunks to split the mesh into");
+
+  params.addCommandLineParam<std::string>("split_file",
+                                          "--split-file [filename]",
+                                          "",
+                                          "optional name of split mesh file(s) to write/read");
+
+  params.addCommandLineParam<bool>(
+      "use_split", "--use-split", false, "use split distributed mesh files");
+
   params.addCommandLineParam<unsigned int>(
       "refinements",
       "-r <n>",
@@ -511,6 +525,12 @@ MooseApp::setupOptions()
       _syntax.addDependency("mesh_only", "setup_mesh_complete");
       _action_warehouse.setFinalTask("mesh_only");
     }
+    else if (isParamValid("split_mesh"))
+    {
+      _syntax.registerTaskName("split_mesh", true);
+      _syntax.addDependency("split_mesh", "setup_mesh_complete");
+      _action_warehouse.setFinalTask("split_mesh");
+    }
     _action_warehouse.build();
   }
   else
@@ -547,17 +567,17 @@ MooseApp::runInputFile()
 
   _action_warehouse.executeAllActions();
 
-  if (isParamValid("mesh_only"))
+  if (isParamValid("mesh_only") || isParamValid("split_mesh"))
     _ready_to_exit = true;
   else if (getParam<bool>("list_constructed_objects"))
   {
     // TODO: ask multiapps for their constructed objects
+    _ready_to_exit = true;
     std::vector<std::string> obj_list = _factory.getConstructedObjects();
     Moose::out << "**START OBJECT DATA**\n";
     for (const auto & name : obj_list)
       Moose::out << name << "\n";
     Moose::out << "**END OBJECT DATA**\n" << std::endl;
-    _ready_to_exit = true;
   }
 }
 

framework/src/mesh/TiledMesh.C

@@ -81,6 +81,12 @@ TiledMesh::clone() const
   return *(new TiledMesh(*this));
 }
 
+std::string
+TiledMesh::getFileName() const
+{
+  return getParam<MeshFileName>("file");
+}
+
 void
 TiledMesh::buildMesh()
 {