generate_partition for index of multi-domain meshes (#696)

* Add generate_partition for partition map of multi-domain meshes.

* Add partition map to generate_index

* Add test that exercises generate_domain

* Remove generate_partition from generate_index

* Change generate_partition to generate_domain_to_rank_map

* Handle case where a subset of the ranks hold a single domain, others
hold none.

* Add value checks in t_blueprint_mpi_mesh_verify

* fix `generate_domain_to_rank_map` for mixed-domain environments (#731)

* Add call to generate_domain_to_rank_map with some ranks empty.

* * Fixed the logic for sparse uni-domain inputs to 'blueprint::mpi::mesh::generate_domain_to_rank_map.'

* * Fixed an issue with building on Windows.

* * Replaced 'std::max' with ternary because 'std::min/max' cause issues on Windows.

Co-authored-by: Joseph Ciurej <ciurej1@llnl.gov>

src/libs/blueprint/conduit_blueprint_mesh.cpp

@@ -769,19 +769,26 @@ verify_multi_domain(const Node &n,
     bool res = true;
     info.reset();
 
-    if(!n.dtype().is_object() && !n.dtype().is_list())
+    if(!n.dtype().is_object() && !n.dtype().is_list() && !n.dtype().is_empty())
     {
-        log::error(info, protocol, "not an object or a list");
+        log::error(info, protocol, "not an object, a list, or empty");
         res = false;
     }
     else
     {
-        NodeConstIterator itr = n.children();
-        while(itr.has_next())
+        if(n.dtype().is_empty() || n.number_of_children() == 0)
         {
-            const Node &chld = itr.next();
-            const std::string chld_name = itr.name();
-            res &= verify_single_domain(chld, info[chld_name]);
+            log::info(info, protocol, "is an empty mesh");
+        }
+        else
+        {
+            NodeConstIterator itr = n.children();
+            while(itr.has_next())
+            {
+                const Node &chld = itr.next();
+                const std::string chld_name = itr.name();
+                res &= verify_single_domain(chld, info[chld_name]);
+            }
         }
 
         log::info(info, protocol, "is a multi domain mesh");
@@ -1429,6 +1436,7 @@ bool mesh::is_multi_domain(const conduit::Node &n)
     return !n.has_child("coordsets");
 }
 
+
 //-------------------------------------------------------------------------
 index_t
 mesh::number_of_domains(const conduit::Node &n)
@@ -1448,6 +1456,32 @@ mesh::number_of_domains(const conduit::Node &n)
 }
 
 
+//-------------------------------------------------------------------------
+std::vector<const conduit::Node *>
+mesh::domains(const conduit::Node &n)
+{
+    // this is a blueprint property, we can assume it will be called
+    // only when mesh verify is true. Given that - it is easy to
+    // aggregate all of the domains into a list
+
+    std::vector<const conduit::Node *> doms;
+
+    if(!mesh::is_multi_domain(n))
+    {
+        doms.push_back(&n);
+    }
+    else if(!n.dtype().is_empty())
+    {
+        NodeConstIterator nitr = n.children();
+        while(nitr.has_next())
+        {
+            doms.push_back(&nitr.next());
+        }
+    }
+
+    return std::vector<const conduit::Node *>(std::move(doms));
+}
+
 
 //-------------------------------------------------------------------------
 void mesh::to_multi_domain(const conduit::Node &n,

src/libs/blueprint/conduit_blueprint_mesh.hpp

@@ -69,13 +69,15 @@ bool CONDUIT_BLUEPRINT_API verify(const conduit::Node &n,
 //-------------------------------------------------------------------------
 bool CONDUIT_BLUEPRINT_API is_multi_domain(const conduit::Node &n);
 
-//
-/// Note: to_multi_domain uses Node::set_external to avoid copying data.
-/// If you need a copy of the data unlinked from the input, set into 
-/// another node.
 //-------------------------------------------------------------------------
 index_t CONDUIT_BLUEPRINT_API number_of_domains(const conduit::Node &n);
 
+//-----------------------------------------------------------------------------
+std::vector<const conduit::Node *> CONDUIT_BLUEPRINT_API domains(const Node &n);
+
+/// Note: to_multi_domain uses Node::set_external to avoid copying data.
+/// If you need a copy of the data unlinked from the input, set into
+/// another node.
 //-------------------------------------------------------------------------
 void CONDUIT_BLUEPRINT_API to_multi_domain(const conduit::Node &n,
                                            conduit::Node &dest);

src/libs/blueprint/conduit_blueprint_mpi_mesh.cpp

@@ -51,19 +51,11 @@ verify(const conduit::Node &n,
        conduit::Node &info,
        MPI_Comm comm)
 {
-    // some MPI tasks may not have data, that is fine
-    // but blueprint verify will fail, so if the
-    // input node is empty skip verify
-    int local_verify_ok = 0;
-    if(!n.dtype().is_empty())
-    {
-        if(conduit::blueprint::mesh::verify(n,
-                                            info))
-        {
-            local_verify_ok = 1;
-        }
-    }
+    int par_size = relay::mpi::size(comm);
 
+    // NOTE(JRC): MPI tasks without any domains should use a multi-domain
+    // format with empty contents (i.e. an empty object or list node).
+    int local_verify_ok = conduit::blueprint::mesh::verify(n, info) ? 1 : 0;
     int global_verify_ok = 0;
 
     Node n_snd, n_reduce;
@@ -72,7 +64,7 @@ verify(const conduit::Node &n,
     n_reduce.set_external(&global_verify_ok,1);
 
     relay::mpi::sum_all_reduce(n_snd, n_reduce, comm);
-    return global_verify_ok > 0;
+    return global_verify_ok == par_size;
 }
 
 //-------------------------------------------------------------------------
@@ -128,6 +120,47 @@ generate_index(const conduit::Node &mesh,
 }
 
 
+//-----------------------------------------------------------------------------
+void generate_domain_to_rank_map(const conduit::Node &mesh,
+                                 Node &domain_to_rank_map,
+                                 MPI_Comm comm)
+{
+    int64 par_rank = relay::mpi::rank(comm);
+    int64 max_local_id = -1;
+
+    std::vector<const Node *> domains = ::conduit::blueprint::mesh::domains(mesh);
+    std::vector<int64> local_domains;
+    for(index_t di = 0; di < (index_t)domains.size(); di++)
+    {
+        const conduit::Node &domain = *domains[di];
+
+        int64 domain_id = par_rank;
+        if(domain.has_child("state") && domain["state"].has_child("domain_id"))
+        {
+            domain_id = domain["state/domain_id"].as_int64();
+        }
+        local_domains.push_back(domain_id);
+
+        max_local_id = (domain_id > max_local_id) ? domain_id : max_local_id;
+    }
+
+    Node max_local, max_global;
+    max_local.set_int64(max_local_id);
+    max_global.set_int64(-1);
+    relay::mpi::max_all_reduce(max_local, max_global, comm);
+
+    std::vector<int64> local_map(max_global.as_int64() + 1, -1);
+    for(auto m_itr = local_domains.begin(); m_itr != local_domains.end(); ++m_itr)
+    {
+        local_map[*m_itr] = par_rank;
+    }
+
+    Node local_par;
+    local_par.set_external(&local_map[0], local_map.size());
+
+    relay::mpi::max_all_reduce(local_par, domain_to_rank_map, comm);
+}
+
 //-----------------------------------------------------------------------------
 index_t
 number_of_domains(const conduit::Node &n,

src/libs/blueprint/conduit_blueprint_mpi_mesh.hpp

@@ -66,6 +66,12 @@ void CONDUIT_BLUEPRINT_API generate_index(const conduit::Node &mesh,
                                           Node &index_out,
                                           MPI_Comm comm);
 
+//-------------------------------------------------------------------------
+void CONDUIT_BLUEPRINT_API generate_domain_to_rank_map(
+                                              const conduit::Node &mesh,
+                                              Node &domain_to_rank_map,
+                                              MPI_Comm comm);
+
 //-------------------------------------------------------------------------
 index_t CONDUIT_BLUEPRINT_API number_of_domains(const conduit::Node &mesh,
                                                 MPI_Comm comm);

src/libs/blueprint/conduit_blueprint_mpi_mesh_examples.cpp

@@ -91,6 +91,7 @@ spiral_round_robin(conduit::index_t ndomains,
                    MPI_Comm comm)
 {
     res.reset();
+    res.set(DataType::list());
 
     int par_rank = relay::mpi::rank(comm);
     int par_size = relay::mpi::size(comm);

src/tests/blueprint/c/t_c_blueprint_mesh.cpp

@@ -22,7 +22,7 @@ TEST(c_conduit_blueprint_mesh, create_and_verify)
     conduit_node *nempty = conduit_node_create();
     conduit_node *info   = conduit_node_create();
 
-    EXPECT_FALSE(conduit_blueprint_mesh_verify(nempty,info));
+    EXPECT_TRUE(conduit_blueprint_mesh_verify(nempty,info));
 
 
     conduit_blueprint_mesh_examples_braid("hexs",3,3,3,n);

src/tests/blueprint/fortran/t_f_blueprint_mesh.f90

@@ -44,7 +44,7 @@ subroutine t_blueprint_mesh_create_and_verify
         nindex = conduit_node_create()
         info = conduit_node_create()
 
-        call assert_true( conduit_blueprint_mesh_verify(nempty,info) .eqv. .false. , "verify false on empty")
+        call assert_true( conduit_blueprint_mesh_verify(nempty,info) .eqv. .true. , "verify true on empty")
         call conduit_blueprint_mesh_examples_braid("hexs",3_8,3_8,3_8,n)
         call assert_true( conduit_blueprint_mesh_verify(n,info) .eqv. .true., "verify true on braid hexs")
 

src/tests/blueprint/python/t_python_blueprint_mesh.py

@@ -2,8 +2,8 @@
 # Project developers. See top-level LICENSE AND COPYRIGHT files for dates and
 # other details. No copyright assignment is required to contribute to Conduit.
 """
- file: t_python_blueprint_mcarray.py
- description: Simple unit test for the conduit blueprint mcarray python module.
+ file: t_python_blueprint_mesh.py
+ description: Simple unit test for the conduit blueprint mesh python module.
 
 """
 
@@ -19,13 +19,29 @@
 
 class Test_Blueprint_Mesh(unittest.TestCase):
 
+    def has_empty_warning(self, info):
+        res = False
+
+        if info.dtype().is_object() and info.has_child("info"):
+            iinfo = info.fetch('info')
+            if iinfo.dtype().is_object() or iinfo.dtype().is_list():
+                iitr = iinfo.children()
+                for ival in iitr:
+                    inode = ival.node()
+                    res = res or (inode.dtype().is_string() and "is an empty mesh" in inode.to_string())
+
+        return res
+
     def test_basic_and_verify(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         blueprint.mesh.examples.basic("hexs",2,2,2,n);
         self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertFalse(self.has_empty_warning(info))
         n_idx = Node()
         blueprint.mesh.generate_index(n,"",1,n_idx)
         self.assertTrue(blueprint.verify("mesh/index",n_idx,info))
@@ -34,10 +50,13 @@ def test_basic_and_verify(self):
     def test_braid_and_verify(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         blueprint.mesh.examples.braid("hexs",2,2,2,n);
         self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertFalse(self.has_empty_warning(info))
         n_idx = Node()
         blueprint.mesh.generate_index(n,"",1,n_idx)
         self.assertTrue(blueprint.verify("mesh/index",n_idx,info))
@@ -46,14 +65,17 @@ def test_braid_and_verify(self):
     def test_julia_and_verify(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         blueprint.mesh.examples.julia(200,200,
                                       -2.0, 2.0,
                                       -2.0, 2.0,
                                       0.285, 0.01,
                                       n);
         self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertFalse(self.has_empty_warning(info))
         n_idx = Node()
         blueprint.mesh.generate_index(n,"",1,n_idx)
         self.assertTrue(blueprint.verify("mesh/index",n_idx,info))
@@ -62,10 +84,13 @@ def test_julia_and_verify(self):
     def test_spiral_and_verify(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         blueprint.mesh.examples.spiral(4,n);
         self.assertTrue(blueprint.mesh.verify(n["domain_000000"],info))
+        self.assertFalse(self.has_empty_warning(info))
         n_idx = Node()
         blueprint.mesh.generate_index(n["domain_000000"],"",4,n_idx)
         self.assertTrue(blueprint.verify("mesh/index",n_idx,info))
@@ -74,14 +99,17 @@ def test_spiral_and_verify(self):
     def test_julia_nestsets(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         # simple case
         blueprint.mesh.examples.julia_nestsets_simple(-2.0, 2.0,
                                                       -2.0, 2.0,
                                                       0.285, 0.01,
                                                       n);
         self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertFalse(self.has_empty_warning(info))
         n_idx = Node()
         blueprint.mesh.generate_index(n["domain_000000"],"",1,n_idx)
         self.assertTrue(blueprint.verify("mesh/index",n_idx,info))
@@ -102,8 +130,10 @@ def test_julia_nestsets(self):
     def test_venn(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         for matset_type in ['full', 
                             'sparse_by_material',
                             'sparse_by_element' ]:
@@ -112,6 +142,7 @@ def test_venn(self):
                                          .5,
                                          n);
             self.assertTrue(blueprint.mesh.verify(n,info))
+            self.assertFalse(self.has_empty_warning(info))
             n_idx = Node()
             blueprint.mesh.generate_index(n,"",1,n_idx)
             self.assertTrue(blueprint.verify("mesh/index",n_idx,info))
@@ -122,11 +153,14 @@ def test_venn(self):
     def test_polytess(self):
         n = Node()
         info = Node()
-        self.assertFalse(blueprint.verify("mesh",n,info))
-        self.assertFalse(blueprint.mesh.verify(n,info))
+        self.assertTrue(blueprint.verify("mesh",n,info))
+        self.assertTrue(self.has_empty_warning(info))
+        self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertTrue(self.has_empty_warning(info))
         # simple case
         blueprint.mesh.examples.polytess(3,n);
         self.assertTrue(blueprint.mesh.verify(n,info))
+        self.assertFalse(self.has_empty_warning(info))
         n_idx = Node()
         blueprint.mesh.generate_index(n,"",1,n_idx)
         self.assertTrue(blueprint.verify("mesh/index",n_idx,info))