Picking a better name for _is_parallel (it really means _is_nemesis).

The accessor is removed, but it could always be readded.

'nemesis' is a parameter on MooseMesh instead of FileMesh.

Reformatting long doc strings onto separate lines.

Adding new command line option to MOOSE: --parallel-mesh.

MooseMesh now uses the MooseApp object to determine if
--parallel-mesh has been specified on the command line.

MooseMesh reads an enumeration for the type of mesh "distribution" to use.

Note: if the user does not specify anything for 'distribution' in the
Mesh block, it is possible to set it to parallel via the
--parallel-mesh command line argument, otherwise it defaults to
serial.  This command line arg is not allowed to override the user if
he does specify 'distribution = serial' in the Mesh block.

Refs #2105.

r20832

framework/include/base/MooseApp.h

@@ -224,6 +224,12 @@ class MooseApp
    */
   virtual void executeExecutioner();
 
+  /**
+   * Returns true if the user specifed --parallel-mesh on the command line and false
+   * otherwise.
+   */
+  bool getParallelMeshOnCommandLine() const { return _parallel_mesh_on_command_line; }
+
 protected:
   MooseApp(const std::string & name, InputParameters parameters);
 
@@ -283,6 +289,9 @@ class MooseApp
 
   /// This variable indicates when a request has been made to restart from an Exodus file
   bool _initial_from_file;
+
+  /// This variable indicates that ParallelMesh should be used for the libMesh mesh underlying MooseMesh.
+  bool _parallel_mesh_on_command_line;
 };
 
 #endif /* MOOSEAPP_H */

framework/include/mesh/MooseMesh.h

@@ -193,11 +193,6 @@ class MooseMesh : public MooseObject
   bool prepared() const;
   void prepared(bool state);
 
-  /**
-   * Getter for the _is_parallel flag.
-   */
-  bool parallel();
-
   /**
    * Declares that the MooseMesh has changed, invalidates cached data
    * and rebuilds caches.  Sets a flag so that clients of the
@@ -530,6 +525,15 @@ class MooseMesh : public MooseObject
   bool isBoundaryNode(unsigned int node_id);
 
 protected:
+  /// Can be set to PARALLEL, SERIAL, or DEFAULT.  Determines whether
+  /// the underlying libMesh mesh is a SerialMesh or ParallelMesh.
+  MooseEnum _mesh_distribution_type;
+
+  /// False by default.  Final value is determined by several factors
+  /// including the 'distribution' setting in the input file, and whether
+  /// or not the Mesh file is a Nemesis file.
+  bool _use_parallel_mesh;
+
   /// Pointer to underlying libMesh mesh object
   libMesh::MeshBase* _mesh;
 
@@ -550,8 +554,8 @@ class MooseMesh : public MooseObject
   /// true if mesh is changed (i.e. after adaptivity step)
   bool _is_changed;
 
-  /// True if using a TRUE parallel mesh (i.e. Nemesis)
-  bool _is_parallel;
+  /// True if a Nemesis Mesh was read in
+  bool _is_nemesis;
 
   /// True if prepare has been called on the mesh
   bool _is_prepared;

framework/src/base/MooseApp.C

@@ -44,6 +44,8 @@ InputParameters validParams<MooseApp>()
   params.addCommandLineParam<bool>("error_unused", "-e --error-unused", "Error when encounting unused input file options");
   params.addCommandLineParam<bool>("error_override", "-o --error-override", "Error when encountering overriden or parameters supplied multipled times");
 
+  params.addCommandLineParam<bool>("parallel_mesh", "--parallel-mesh", "The libMesh Mesh underlying MooseMesh should always be a ParallelMesh");
+
   params.addCommandLineParam<unsigned int>("refinements", "-r <n>", 0, "Specify additional initial uniform refinements for automatic scaling");
 
   params.addPrivateParam<int>("_argc");
@@ -69,7 +71,8 @@ MooseApp::MooseApp(const std::string & name, InputParameters parameters):
     _factory(*this),
     _error_overridden(false),
     _ready_to_exit(false),
-    _initial_from_file(false)
+    _initial_from_file(false),
+    _parallel_mesh_on_command_line(false)
 {
   if(isParamValid("_argc") && isParamValid("_argv"))
   {
@@ -101,6 +104,9 @@ MooseApp::setupOptions()
   if (isParamValid("error_override"))
     setErrorOverridden();
 
+  if (isParamValid("parallel_mesh"))
+    _parallel_mesh_on_command_line = true;
+
   if (isParamValid("help"))
   {
     _command_line->printUsage();

framework/src/mesh/FileMesh.C

@@ -29,11 +29,9 @@ InputParameters validParams<FileMesh>()
   InputParameters params = validParams<MooseMesh>();
 
   params.addRequiredParam<MeshFileName>("file", "The name of the mesh file to read");
-  params.addParam<bool>("nemesis", false, "If nemesis=true and file=foo.e, actually reads foo.e.N.0, foo.e.N.1, ... foo.e.N.N-1, where N = # CPUs, with NemesisIO.");
   params.addParam<bool>("skip_partitioning", false, "If true the mesh won't be partitioned.  Probably not a good idea to use it with a serial mesh!");
 
   // groups
-  params.addParamNamesToGroup("nemesis", "Advanced");
   params.addParamNamesToGroup("skip_partitioning", "Partitioning");
 
   return params;
@@ -46,7 +44,6 @@ FileMesh::FileMesh(const std::string & name, InputParameters parameters) :
     _exreader(NULL)
 {
   getMesh().set_mesh_dimension(getParam<MooseEnum>("dim"));
-  _is_parallel = getParam<bool>("nemesis");
 }
 
 FileMesh::FileMesh(const FileMesh & other_mesh) :
@@ -74,7 +71,7 @@ FileMesh::init()
   std::string _file_name = getParam<MeshFileName>("file");
 
   Moose::setup_perf_log.push("Read Mesh","Setup");
-  if (getParam<bool>("nemesis"))
+  if (_is_nemesis)
   {
     // Nemesis_IO only takes a reference to ParallelMesh, so we can't be quite so short here.
     ParallelMesh& pmesh = libmesh_cast_ref<ParallelMesh&>(getMesh());
@@ -105,8 +102,8 @@ FileMesh::init()
 void
 FileMesh::read(const std::string & file_name)
 {
-  if (dynamic_cast<ParallelMesh *>(&getMesh()) && !_is_parallel)
-    getMesh().read(file_name, NULL, false);
+  if (dynamic_cast<ParallelMesh *>(&getMesh()) && !_is_nemesis)
+    getMesh().read(file_name, /*mesh_data=*/NULL, /*skip_renumber=*/false);
   else
-    getMesh().read(file_name, NULL, true);
+    getMesh().read(file_name, /*mesh_data=*/NULL, /*skip_renumber=*/true);
 }

framework/src/mesh/MooseMesh.C

@@ -18,6 +18,7 @@
 #include "CacheChangedListsThread.h"
 #include "Assembly.h"
 #include "MooseUtils.h"
+#include "MooseApp.h"
 
 // libMesh
 #include "libmesh/boundary_info.h"
@@ -36,23 +37,42 @@ InputParameters validParams<MooseMesh>()
 {
   InputParameters params = validParams<MooseObject>();
 
+  MooseEnum mesh_distribution_type("PARALLEL=0, SERIAL, DEFAULT", "DEFAULT");
+  params.addParam<MooseEnum>("distribution", mesh_distribution_type,
+                             "PARALLEL: Always use libMesh::ParallelMesh "
+                             "SERIAL: Always use libMesh::SerialMesh "
+                             "DEFAULT: Use libMesh::SerialMesh unless --parallel-mesh is specified on the command line");
+
+  params.addParam<bool>("nemesis", false,
+                        "If nemesis=true and file=foo.e, actually reads "
+                        "foo.e.N.0, foo.e.N.1, ... foo.e.N.N-1, "
+                        "where N = # CPUs, with NemesisIO.");
+
   MooseEnum dims("1 = 1, 2, 3", "3");
-  params.addParam<MooseEnum>("dim", dims, "This is only required for certain mesh formats where the dimension of the mesh cannot be autodetected.  In particular you must supply this for GMSH meshes.  Note: This is completely ignored for ExodusII meshes!");
+  params.addParam<MooseEnum>("dim", dims,
+                             "This is only required for certain mesh formats where "
+                             "the dimension of the mesh cannot be autodetected.  "
+                             "In particular you must supply this for GMSH meshes.  "
+                             "Note: This is completely ignored for ExodusII meshes!");
 
   params.addPrivateParam<std::string>("built_by_action", "setup_mesh");
+
   // groups
   params.addParamNamesToGroup("dim", "Advanced");
+  params.addParamNamesToGroup("nemesis", "Advanced");
 
   return params;
 }
 
 
 MooseMesh::MooseMesh(const std::string & name, InputParameters parameters) :
     MooseObject(name, parameters),
-    _mesh(new libMesh::Mesh(getParam<MooseEnum>("dim"))),
+    _mesh_distribution_type(getParam<MooseEnum>("distribution")),
+    _use_parallel_mesh(false),
+    _mesh(NULL),
     _uniform_refine_level(0),
     _is_changed(false),
-    _is_parallel(false),
+    _is_nemesis(getParam<bool>("nemesis")),
     _is_prepared(false),
     _refined_elements(NULL),
     _coarsened_elements(NULL),
@@ -66,14 +86,48 @@ MooseMesh::MooseMesh(const std::string & name, InputParameters parameters) :
     _patch_size(40),
     _regular_orthogonal_mesh(false)
 {
+  switch (_mesh_distribution_type)
+  {
+  case 0: // PARALLEL
+    _use_parallel_mesh = true;
+    break;
+  case 1: // SERIAL
+    break;
+  case 2: // DEFAULT
+  {
+    // The user did not specify 'distribution = XYZ' in the input file,
+    // so we allow the --parallel-mesh command line arg to possibly turn
+    // on ParallelMesh.  If the command line arg is not present, we pick SerialMesh.
+    if (_app.getParallelMeshOnCommandLine())
+      _use_parallel_mesh = true;
+
+    break;
+  }
+  default:
+    mooseError("Unknown mesh distribution type!");
+  }
+
+  // If the user specifies 'nemesis = true' in the Mesh block, we
+  // must use ParallelMesh.
+  if (_is_nemesis)
+    _use_parallel_mesh = true;
+
+  unsigned dim = getParam<MooseEnum>("dim");
+
+  if (_use_parallel_mesh)
+    _mesh = new ParallelMesh(dim);
+  else
+    _mesh = new SerialMesh(dim);
 }
 
 MooseMesh::MooseMesh(const MooseMesh & other_mesh) :
     MooseObject(other_mesh._name, other_mesh._pars),
+    _mesh_distribution_type(other_mesh._mesh_distribution_type),
+    _use_parallel_mesh(other_mesh._use_parallel_mesh),
     _mesh(other_mesh.getMesh().clone().release()),
     _uniform_refine_level(0),
     _is_changed(false),
-    _is_parallel(false),
+    _is_nemesis(false),
     _is_prepared(false),
     _refined_elements(NULL),
     _coarsened_elements(NULL),
@@ -152,15 +206,17 @@ MooseMesh::freeBndElems()
 void
 MooseMesh::prepare()
 {
-  if (dynamic_cast<ParallelMesh *>(&getMesh()) && !_is_parallel)
+  if (dynamic_cast<ParallelMesh *>(&getMesh()) && !_is_nemesis)
   {
+    // Call prepare_for_use() and allow renumbering
     getMesh().allow_renumbering(true);
-    getMesh().prepare_for_use(/*false*/);
+    getMesh().prepare_for_use();
   }
   else
   {
+    // Call prepare_for_use() and DO NOT allow renumbering
     getMesh().allow_renumbering(false);
-    getMesh().prepare_for_use(/*true*/);
+    getMesh().prepare_for_use();
   }
 
   // Collect (local) subdomain IDs
@@ -1592,12 +1648,6 @@ MooseMesh::prepared(bool state)
   _is_prepared = state;
 }
 
-bool
-MooseMesh::parallel()
-{
-  return _is_parallel;
-}
-
 const std::set<SubdomainID> &
 MooseMesh::meshSubdomains() const
 {