Small number of cores for partition

The number of cores for the graph partition can be different from that used
for mesh generation and simulation. Partitioners often become inefficient in
compute time when the number of cores is large (around 10,0000).
A possible "fix" is to partition the graph using a small number of cores
when the mesh generation and the numerical simulation use a large number of processor cores.

Closes idaholab#15464

framework/include/meshgenerators/DistributedRectilinearMeshGenerator.h

@@ -303,6 +303,9 @@ class DistributedRectilinearMeshGenerator : public MeshGenerator
   /// The min/max values for x,y,z component
   Real &_xmin, &_xmax, &_ymin, &_ymax, &_zmin, &_zmax;
 
+  /// Number of cores for partitioning the graph
+  dof_id_type _num_cores_for_partition;
+
   /// The type of element to build
   ElemType _elem_type;
 

framework/src/meshgenerators/DistributedRectilinearMeshGenerator.C

@@ -51,6 +51,14 @@ DistributedRectilinearMeshGenerator::validParams()
   params.addParam<Real>("ymax", 1.0, "Upper Y Coordinate of the generated mesh");
   params.addParam<Real>("zmax", 1.0, "Upper Z Coordinate of the generated mesh");
 
+  // The number of cores for the graph partition can be different from that used
+  // for mesh generation and simulation. Partitioners often become inefficient in
+  // compute time when the number of cores is large (around 10,0000).
+  // A possible "fix" is to partition the graph using a small number of cores
+  // when the mesh generation and the numerical simulation use a large number of processor cores.
+  params.addParam<dof_id_type>(
+      "num_cores_for_partition", 0, "Number of cores for partitioning the graph");
+
   MooseEnum elem_types(
       "EDGE EDGE2 EDGE3 EDGE4 QUAD QUAD4 QUAD8 QUAD9 TRI3 TRI6 HEX HEX8 HEX20 HEX27 TET4 TET10 "
       "PRISM6 PRISM15 PRISM18 PYRAMID5 PYRAMID13 PYRAMID14"); // no default
@@ -97,6 +105,7 @@ DistributedRectilinearMeshGenerator::DistributedRectilinearMeshGenerator(
     _ymax(declareMeshProperty("ymax", getParam<Real>("ymax"))),
     _zmin(declareMeshProperty("zmin", getParam<Real>("zmin"))),
     _zmax(declareMeshProperty("zmax", getParam<Real>("zmax"))),
+    _num_cores_for_partition(getParam<dof_id_type>("num_cores_for_partition")),
     _bias_x(getParam<Real>("bias_x")),
     _bias_y(getParam<Real>("bias_y")),
     _bias_z(getParam<Real>("bias_z")),
@@ -840,6 +849,12 @@ DistributedRectilinearMeshGenerator::buildCube(UnstructuredMesh & mesh,
   // Current processor ID
   const auto pid = comm.rank();
 
+  if (_num_cores_for_partition > num_procs)
+    mooseError("Number of cores for partition is too large ", _num_cores_for_partition);
+
+  if (_num_cores_for_partition <= 0)
+    _num_cores_for_partition = num_procs;
+
   auto & boundary_info = mesh.get_boundary_info();
 
   std::unique_ptr<Elem> canonical_elem = libmesh_make_unique<T>();
@@ -853,8 +868,19 @@ DistributedRectilinearMeshGenerator::buildCube(UnstructuredMesh & mesh,
   dof_id_type num_local_elems;
   dof_id_type local_elems_begin;
   dof_id_type local_elems_end;
-  MooseUtils::linearPartitionItems(
-      num_elems, num_procs, pid, num_local_elems, local_elems_begin, local_elems_end);
+  if (pid < _num_cores_for_partition)
+    MooseUtils::linearPartitionItems(num_elems,
+                                     _num_cores_for_partition,
+                                     pid,
+                                     num_local_elems,
+                                     local_elems_begin,
+                                     local_elems_end);
+  else
+  {
+    num_local_elems = 0;
+    local_elems_begin = 0;
+    local_elems_end = 0;
+  }
 
   std::vector<std::vector<dof_id_type>> graph;
 
@@ -926,7 +952,7 @@ DistributedRectilinearMeshGenerator::buildCube(UnstructuredMesh & mesh,
   for (auto & ghost_id : ghost_elems)
   {
     // This is the processor ID the ghost_elem was originally assigned to
-    auto proc_id = MooseUtils::linearPartitionChunk(num_elems, num_procs, ghost_id);
+    auto proc_id = MooseUtils::linearPartitionChunk(num_elems, _num_cores_for_partition, ghost_id);
 
     // Using side-effect insertion on purpose
     ghost_elems_to_request[proc_id].push_back(ghost_id);

test/tests/meshgenerators/distributed_rectilinear_mesh_generator/tests

@@ -47,4 +47,23 @@
     requirement = 'MOOSE shall be able to generate 3D HEX8 mesh in parallel using hierarch partitioner'
   [../]
 
+  [./drmg_3d_scomm_out]
+    type = 'Exodiff'
+    input = 'distributed_rectilinear_mesh_generator.i'
+    exodiff = 'drmg_3d_scomm_out.e'
+    min_parallel = 2
+    issues = '#15464'
+    cli_args = 'Mesh/gmg/num_cores_for_partition=2 Mesh/gmg/dim=3 Mesh/gmg/nx=20 Mesh/gmg/ny=20 Mesh/gmg/nz=20 Outputs/file_base=drmg_3d_scomm_out Outputs/hide="pid npid" '
+    requirement = 'MOOSE shall be able to generate 3D HEX8 mesh in parallel using a small number of cores for partition.'
+  [../]
+
+  [./drmg_3d_scomm_10_out]
+    type = 'RunException'
+    input = 'distributed_rectilinear_mesh_generator.i'
+    expect_err = 'Number of cores for partition'
+    max_parallel = 8
+    issues = '#15464'
+    cli_args = 'Mesh/gmg/num_cores_for_partition=10 Mesh/gmg/dim=3 Mesh/gmg/nx=20 Mesh/gmg/ny=20 Mesh/gmg/nz=20 Outputs/hide="pid npid" '
+    requirement = 'MOOSE shall error out if the number of cores for partition is larger than the total number of core.'
+  [../]
 []