CheckpointIO DistributedMesh fixes

include/Makefile.in

@@ -693,6 +693,7 @@ include_HEADERS = \
         geom/cell_tet.h \
         geom/cell_tet10.h \
         geom/cell_tet4.h \
+        geom/compare_elems_by_level.h \
         geom/edge.h \
         geom/edge_edge2.h \
         geom/edge_edge3.h \

include/geom/compare_elems_by_level.h

@@ -0,0 +1,61 @@
+// The libMesh Finite Element Library.
+// Copyright (C) 2002-2017 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
+
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+
+#ifndef LIBMESH_COMPARE_ELEMS_BY_LEVEL_H
+#define LIBMESH_COMPARE_ELEMS_BY_LEVEL_H
+
+
+// Local Includes
+#include "libmesh/elem.h"
+
+// C++ Includes
+#include <numeric>
+#include <set>
+
+
+
+
+namespace libMesh {
+
+/**
+ * Specific weak ordering for Elem *'s to be used in a set.
+ * We use the id, but first sort by level.  This guarantees
+ * when traversing the set from beginning to end the lower
+ * level (parent) elements are encountered first.
+ */
+struct CompareElemIdsByLevel
+{
+  bool operator()(const Elem * a,
+                  const Elem * b) const
+  {
+    libmesh_assert (a);
+    libmesh_assert (b);
+    const unsigned int
+      al = a->level(), bl = b->level();
+    const dof_id_type
+      aid = a->id(),   bid = b->id();
+
+    return (al == bl) ? aid < bid : al < bl;
+  }
+};
+
+
+} // namespace libMesh
+
+#endif // LIBMESH_COMPARE_ELEMS_BY_LEVEL_H

include/include_HEADERS

@@ -125,6 +125,7 @@ include_HEADERS =  \
         geom/cell_tet.h \
         geom/cell_tet10.h \
         geom/cell_tet4.h \
+        geom/compare_elems_by_level.h \
         geom/edge.h \
         geom/edge_edge2.h \
         geom/edge_edge3.h \

include/libmesh/Makefile.am

@@ -114,6 +114,7 @@ BUILT_SOURCES = \
         cell_tet.h \
         cell_tet10.h \
         cell_tet4.h \
+        compare_elems_by_level.h \
         edge.h \
         edge_edge2.h \
         edge_edge3.h \
@@ -852,6 +853,9 @@ cell_tet10.h: $(top_srcdir)/include/geom/cell_tet10.h
 cell_tet4.h: $(top_srcdir)/include/geom/cell_tet4.h
 	$(AM_V_GEN)rm -f $@ && $(LN_S) -f $< $@
 
+compare_elems_by_level.h: $(top_srcdir)/include/geom/compare_elems_by_level.h
+	$(AM_V_GEN)rm -f $@ && $(LN_S) -f $< $@
+
 edge.h: $(top_srcdir)/include/geom/edge.h
 	$(AM_V_GEN)rm -f $@ && $(LN_S) -f $< $@
 

include/libmesh/Makefile.in

@@ -522,17 +522,17 @@ BUILT_SOURCES = auto_ptr.h default_coupling.h dirichlet_boundaries.h \
 	cell_inf_hex8.h cell_inf_prism.h cell_inf_prism12.h \
 	cell_inf_prism6.h cell_prism.h cell_prism15.h cell_prism18.h \
 	cell_prism6.h cell_pyramid.h cell_pyramid13.h cell_pyramid14.h \
-	cell_pyramid5.h cell_tet.h cell_tet10.h cell_tet4.h edge.h \
-	edge_edge2.h edge_edge3.h edge_edge4.h edge_inf_edge2.h elem.h \
-	elem_cutter.h elem_hash.h elem_quality.h elem_range.h face.h \
-	face_inf_quad.h face_inf_quad4.h face_inf_quad6.h face_quad.h \
-	face_quad4.h face_quad4_shell.h face_quad8.h face_quad9.h \
-	face_tri.h face_tri3.h face_tri3_shell.h \
-	face_tri3_subdivision.h face_tri6.h node.h node_elem.h \
-	node_range.h plane.h point.h reference_elem.h remote_elem.h \
-	side.h sphere.h stored_range.h surface.h abaqus_io.h \
-	boundary_info.h boundary_mesh.h checkpoint_io.h \
-	distributed_mesh.h ensight_io.h exodusII_io.h \
+	cell_pyramid5.h cell_tet.h cell_tet10.h cell_tet4.h \
+	compare_elems_by_level.h edge.h edge_edge2.h edge_edge3.h \
+	edge_edge4.h edge_inf_edge2.h elem.h elem_cutter.h elem_hash.h \
+	elem_quality.h elem_range.h face.h face_inf_quad.h \
+	face_inf_quad4.h face_inf_quad6.h face_quad.h face_quad4.h \
+	face_quad4_shell.h face_quad8.h face_quad9.h face_tri.h \
+	face_tri3.h face_tri3_shell.h face_tri3_subdivision.h \
+	face_tri6.h node.h node_elem.h node_range.h plane.h point.h \
+	reference_elem.h remote_elem.h side.h sphere.h stored_range.h \
+	surface.h abaqus_io.h boundary_info.h boundary_mesh.h \
+	checkpoint_io.h distributed_mesh.h ensight_io.h exodusII_io.h \
 	exodusII_io_helper.h fro_io.h gmsh_io.h gmv_io.h gnuplot_io.h \
 	inf_elem_builder.h matlab_io.h medit_io.h mesh.h mesh_base.h \
 	mesh_communication.h mesh_function.h mesh_generation.h \
@@ -1198,6 +1198,9 @@ cell_tet10.h: $(top_srcdir)/include/geom/cell_tet10.h
 cell_tet4.h: $(top_srcdir)/include/geom/cell_tet4.h
 	$(AM_V_GEN)rm -f $@ && $(LN_S) -f $< $@
 
+compare_elems_by_level.h: $(top_srcdir)/include/geom/compare_elems_by_level.h
+	$(AM_V_GEN)rm -f $@ && $(LN_S) -f $< $@
+
 edge.h: $(top_srcdir)/include/geom/edge.h
 	$(AM_V_GEN)rm -f $@ && $(LN_S) -f $< $@
 

include/mesh/checkpoint_io.h

@@ -22,7 +22,7 @@
 
 
 // Local includes
-#include "libmesh/libmesh.h"
+#include "libmesh/compare_elems_by_level.h"
 #include "libmesh/mesh_input.h"
 #include "libmesh/mesh_output.h"
 #include "libmesh/parallel_object.h"
@@ -39,13 +39,13 @@ class Xdr;
 /**
  * The CheckpointIO class can be used to write simplified restart
  * files that can be used to restart simulations that have
- * crashed. Only N-to-N (procs) restart is supported with CheckpointIO
- * files.
+ * crashed.
  *
  * \author Benjamin Kirk
  * \author John Peterson
  * \author Derek Gaston
- * \date 2013
+ * \author Roy Stogner
+ * \date 2017
  */
 class CheckpointIO : public MeshInput<MeshBase>,
                      public MeshOutput<MeshBase>,
@@ -112,18 +112,24 @@ class CheckpointIO : public MeshInput<MeshBase>,
   std::string &       version ()       { return _version; }
 
   /**
-   * Get/Set the processor_id to use
+   * Get/Set the processor id or processor ids to use.
+   *
+   * The default processor_id to use is the processor_id() of the
+   * mesh.
    *
    * This is used for m->n parallel checkpoint file writing:
-   * You can force CheckpointIO to view the world as if it is on a particular
-   * processor_id by setting it here
+   * You can force CheckpointIO to write out different partitions of a
+   * mesh by setting which partitions to write from each processor here.
    */
-  const processor_id_type & current_processor_id() const { return _my_processor_id; }
-  processor_id_type & current_processor_id() { return _my_processor_id; }
+  const std::vector<processor_id_type> & current_processor_ids() const { return _my_processor_ids; }
+  std::vector<processor_id_type> & current_processor_ids() { return _my_processor_ids; }
 
   /**
    * Get/Set the n_processors to use
    *
+   * The default n_processors to use is the n_processors() of the
+   * mesh.
+   *
    * This is used for m->n parallel checkpoint file writing:
    * You can force CheckpointIO to view the world as if it contains this number of
    * processors by setting it here
@@ -136,39 +142,39 @@ class CheckpointIO : public MeshInput<MeshBase>,
   // Write Implementation
 
   /**
-   * Build up the elem list
+   * Write subdomain name information - NEW in 0.9.2 format
    */
-  void build_elem_list();
+  void write_subdomain_names(Xdr & io) const;
 
   /**
-   * Build up the node list
+   * Write the connectivity for part of a mesh
    */
-  void build_node_list();
+  void write_connectivity (Xdr & io,
+                           const std::set<const Elem *, CompareElemIdsByLevel> & elements) const;
 
   /**
-   * Write subdomain name information - NEW in 0.9.2 format
+   * Write the remote_elem neighbor links for part of a mesh
    */
-  void write_subdomain_names(Xdr & io) const;
+  void write_remote_elem (Xdr & io,
+                          const std::set<const Elem *, CompareElemIdsByLevel> & elements) const;
 
   /**
-   * Write the connectivity for a parallel, distributed mesh
+   * Write the nodal locations for part of a mesh
    */
-  void write_connectivity (Xdr & io) const;
+  void write_nodes (Xdr & io,
+                    const std::set<const Node *> & nodeset) const;
 
   /**
-   * Write the nodal locations for a parallel, distributed mesh
+   * Write the side boundary conditions for part of a mesh
    */
-  void write_nodes (Xdr & io) const;
+  void write_bcs (Xdr & io,
+                  const std::set<const Elem *, CompareElemIdsByLevel> & elements) const;
 
   /**
-   * Write the boundary conditions for a parallel, distributed mesh
+   * Write the nodal boundary conditions for part of a mesh
    */
-  void write_bcs (Xdr & io) const;
-
-  /**
-   * Write the boundary conditions for a parallel, distributed mesh
-   */
-  void write_nodesets (Xdr & io) const;
+  void write_nodesets (Xdr & io,
+                       const std::set<const Node *> & nodeset) const;
 
   /**
    * Write boundary names information (sideset and nodeset) - NEW in 0.9.2 format
@@ -188,6 +194,11 @@ class CheckpointIO : public MeshInput<MeshBase>,
    */
   void read_connectivity (Xdr & io);
 
+  /**
+   * Read the remote_elem neighbor links for a parallel, distributed mesh
+   */
+  void read_remote_elem (Xdr & io);
+
   /**
    * Read the nodal locations for a parallel, distributed mesh
    */
@@ -214,21 +225,19 @@ class CheckpointIO : public MeshInput<MeshBase>,
    * Implemented by looping over all the active elements and finding
    * the maximum level.
    */
-  unsigned int n_active_levels_on_processor(const MeshBase & mesh) const;
+  unsigned int n_active_levels_in(MeshBase::const_element_iterator begin,
+                                  MeshBase::const_element_iterator end) const;
 
   bool _binary;
   bool _parallel;
   std::string _version;
   unsigned int _mesh_dimension;
+  unsigned int _n_active_levels;
 
-  /// These are sets of IDs to make the lookup for boundary conditions simpler
-  std::set<largest_id_type> _local_elements;
-  std::set<largest_id_type> _nodes_connected_to_local_elements;
-
-  /// The processor_id to use
-  processor_id_type _my_processor_id;
+  // The processor ids to write
+  std::vector<processor_id_type> _my_processor_ids;
 
-  /// The number of processors to use
+  // The largest processor id to write
   processor_id_type _my_n_processors;
 };
 

include/mesh/distributed_mesh.h

@@ -118,6 +118,13 @@ class DistributedMesh : public UnstructuredMesh
   virtual bool is_serial () const libmesh_override
   { return _is_serial; }
 
+  /**
+   * Asserts that not all elements and nodes of the mesh necessarily
+   * exist on the current processor.
+   */
+  virtual void set_distributed ()
+  { _is_serial = false; }
+
   /**
    * @returns \p true if new elements and nodes can and should be
    * created in synchronization on all processors, \p false otherwise

include/mesh/mesh_base.h

@@ -134,6 +134,14 @@ class MeshBase : public ParallelObject
   virtual bool is_serial () const
   { return true; }
 
+  /**
+   * Asserts that not all elements and nodes of the mesh necessarily
+   * exist on the current processor.  Only valid to call on classes
+   * which can be created in a distributed form.
+   */
+  virtual void set_distributed ()
+  { libmesh_error(); }
+
   /**
    * @returns \p true if new elements and nodes can and should be
    * created in synchronization on all processors, \p false otherwise

include/mesh/mesh_communication.h

@@ -21,6 +21,7 @@
 #define LIBMESH_MESH_COMMUNICATION_H
 
 // Local Includes
+#include "libmesh/compare_elems_by_level.h"
 #include "libmesh/libmesh_common.h"
 #include "libmesh/mesh_tools.h"
 
@@ -231,6 +232,31 @@ class MeshCommunication
 };
 
 
+// Related utilities
+
+// Ask a mesh's ghosting functors to insert into a set all elements
+// that are either on or connected to processor id \p pid.  Ask only
+// for newly-coarsened elements if \p newly_coarsened_only is true.
+void query_ghosting_functors(const MeshBase & mesh,
+                             processor_id_type pid,
+                             bool newly_coarsened_only,
+                             std::set<const Elem *, CompareElemIdsByLevel> & connected_elements);
+
+// Take a set of elements and insert all elements' immediate
+// children as well.
+void connect_children(const MeshBase & mesh,
+                      processor_id_type pid,
+                      std::set<const Elem *, CompareElemIdsByLevel> & connected_elements);
+
+// Take a set of elements and insert all elements' ancestors and
+// subactive descendants as well.
+void connect_families(std::set<const Elem *, CompareElemIdsByLevel> & connected_elements);
+
+// Take a set of elements and create a set of connected nodes.
+void reconnect_nodes (const std::set<const Elem *, CompareElemIdsByLevel> & connected_elements,
+                      std::set<const Node *> & connected_nodes);
+
+
 
 //--------------------------------------------------------------
 // MeshCommunication inline members