Skip to content

Commit

Permalink
fix generate_domain_to_rank_map for mixed-domain environments (#731)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xjrc
xjrc committed Apr 7, 2021
1 parent 2db4c31 commit d9160ae
Show file tree
Hide file tree
Showing 10 changed files with 280 additions and 136 deletions.
48 changes: 41 additions & 7 deletions src/libs/blueprint/conduit_blueprint_mesh.cpp
View file
Expand Up @@ -769,19 +769,26 @@ verify_multi_domain(const Node &n,
bool res = true;
info.reset();

if(!n.dtype().is_object() && !n.dtype().is_list())
if(!n.dtype().is_object() && !n.dtype().is_list() && !n.dtype().is_empty())
{
log::error(info, protocol, "not an object or a list");
log::error(info, protocol, "not an object, a list, or empty");
res = false;
}
else
{
NodeConstIterator itr = n.children();
while(itr.has_next())
if(n.dtype().is_empty() || n.number_of_children() == 0)
{
const Node &chld = itr.next();
const std::string chld_name = itr.name();
res &= verify_single_domain(chld, info[chld_name]);
log::info(info, protocol, "is an empty mesh");
}
else
{
NodeConstIterator itr = n.children();
while(itr.has_next())
{
const Node &chld = itr.next();
const std::string chld_name = itr.name();
res &= verify_single_domain(chld, info[chld_name]);
}
}

log::info(info, protocol, "is a multi domain mesh");
Expand Down Expand Up @@ -1429,6 +1436,7 @@ bool mesh::is_multi_domain(const conduit::Node &n)
return !n.has_child("coordsets");
}


//-------------------------------------------------------------------------
index_t
mesh::number_of_domains(const conduit::Node &n)
Expand All @@ -1448,6 +1456,32 @@ mesh::number_of_domains(const conduit::Node &n)
}


//-------------------------------------------------------------------------
std::vector<const conduit::Node *>
mesh::domains(const conduit::Node &n)
{
// this is a blueprint property, we can assume it will be called
// only when mesh verify is true. Given that - it is easy to
// aggregate all of the domains into a list

std::vector<const conduit::Node *> doms;

if(!mesh::is_multi_domain(n))
{
doms.push_back(&n);
}
else if(!n.dtype().is_empty())
{
NodeConstIterator nitr = n.children();
while(nitr.has_next())
{
doms.push_back(&nitr.next());
}
}

return std::vector<const conduit::Node *>(std::move(doms));
}


//-------------------------------------------------------------------------
void mesh::to_multi_domain(const conduit::Node &n,
Expand Down
10 changes: 6 additions & 4 deletions src/libs/blueprint/conduit_blueprint_mesh.hpp
View file
Expand Up @@ -69,13 +69,15 @@ bool CONDUIT_BLUEPRINT_API verify(const conduit::Node &n,
//-------------------------------------------------------------------------
bool CONDUIT_BLUEPRINT_API is_multi_domain(const conduit::Node &n);

//
/// Note: to_multi_domain uses Node::set_external to avoid copying data.
/// If you need a copy of the data unlinked from the input, set into
/// another node.
//-------------------------------------------------------------------------
index_t CONDUIT_BLUEPRINT_API number_of_domains(const conduit::Node &n);

//-----------------------------------------------------------------------------
std::vector<const conduit::Node *> CONDUIT_BLUEPRINT_API domains(const Node &n);

/// Note: to_multi_domain uses Node::set_external to avoid copying data.
/// If you need a copy of the data unlinked from the input, set into
/// another node.
//-------------------------------------------------------------------------
void CONDUIT_BLUEPRINT_API to_multi_domain(const conduit::Node &n,
conduit::Node &dest);
Expand Down
99 changes: 25 additions & 74 deletions src/libs/blueprint/conduit_blueprint_mpi_mesh.cpp
View file
Expand Up @@ -51,19 +51,11 @@ verify(const conduit::Node &n,
conduit::Node &info,
MPI_Comm comm)
{
// some MPI tasks may not have data, that is fine
// but blueprint verify will fail, so if the
// input node is empty skip verify
int local_verify_ok = 0;
if(!n.dtype().is_empty())
{
if(conduit::blueprint::mesh::verify(n,
info))
{
local_verify_ok = 1;
}
}
int par_size = relay::mpi::size(comm);

// NOTE(JRC): MPI tasks without any domains should use a multi-domain
// format with empty contents (i.e. an empty object or list node).
int local_verify_ok = conduit::blueprint::mesh::verify(n, info) ? 1 : 0;
int global_verify_ok = 0;

Node n_snd, n_reduce;
Expand All @@ -72,7 +64,7 @@ verify(const conduit::Node &n,
n_reduce.set_external(&global_verify_ok,1);

relay::mpi::sum_all_reduce(n_snd, n_reduce, comm);
return global_verify_ok > 0;
return global_verify_ok == par_size;
}

//-------------------------------------------------------------------------
Expand Down Expand Up @@ -135,76 +127,35 @@ void generate_domain_to_rank_map(const conduit::Node &mesh,
{
int64 par_rank = relay::mpi::rank(comm);

if (!mesh.dtype().is_empty() &&
::conduit::blueprint::mesh::is_multi_domain(mesh))
std::vector<const Node *> domains = ::conduit::blueprint::mesh::domains(mesh);
std::vector<int64> local_domains;
for(index_t di = 0; di < (index_t)domains.size(); di++)
{
const conduit::Node &domain = *domains[di];

std::vector<conduit::int64> local_domains;

conduit::NodeConstIterator itr = mesh.children();
while (itr.has_next())
{
const conduit::Node &chld = itr.next();
if (chld.has_child("state"))
{
const conduit::Node &state = chld["state"];
if (state.has_child("domain_id"))
{
conduit::int64 dom_id = state["domain_id"].as_int64();
local_domains.push_back(dom_id);
}
}
}

Node num_local, num_global;
num_local.set_int64(local_domains.size());
num_global.set_int64(0);
relay::mpi::sum_all_reduce(num_local, num_global, comm);

std::vector<int64> local_map(num_global.as_int64(), 0);
for (auto m_itr = local_domains.begin(); m_itr != local_domains.end();
++m_itr)
int64 domain_id = par_rank;
if(domain.has_child("state") && domain["state"].has_child("domain_id"))
{
local_map[*m_itr] = par_rank;
domain_id = domain["state/domain_id"].as_int64();
}

Node local_par;
local_par.set_external(&local_map[0], local_map.size());

relay::mpi::max_all_reduce(local_par, domain_to_rank_map, comm);
local_domains.push_back(domain_id);
}
else
{
int64 par_size = relay::mpi::size(comm);

std::vector<int64> local_map(par_size, 0);
if (mesh.dtype().is_empty())
{
local_map[par_rank] = -1;
}
else
{
local_map[par_rank] = par_rank;
}

Node local_full;
local_full.set_external(&local_map[0], local_map.size());
Node num_local, num_global;
num_local.set_int64(local_domains.size());
num_global.set_int64(0);
relay::mpi::sum_all_reduce(num_local, num_global, comm);

Node global_full;
relay::mpi::sum_all_reduce(local_full, global_full, comm);
std::vector<int64> local_map(num_global.as_int64(), 0);
for(auto m_itr = local_domains.begin(); m_itr != local_domains.end(); ++m_itr)
{
local_map[*m_itr] = par_rank;
}

conduit::int64_array global_array = global_full.as_int64_array();
std::vector<int64> compact_map;
for (int i = 0; i < par_size; ++i)
{
if (global_array[i] != -1)
{
compact_map.push_back(i);
}
}
Node local_par;
local_par.set_external(&local_map[0], local_map.size());

domain_to_rank_map.set(compact_map);
}
relay::mpi::max_all_reduce(local_par, domain_to_rank_map, comm);
}

//-----------------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions src/libs/blueprint/conduit_blueprint_mpi_mesh_examples.cpp
View file
Expand Up @@ -91,6 +91,7 @@ spiral_round_robin(conduit::index_t ndomains,
MPI_Comm comm)
{
res.reset();
res.set(DataType::list());

int par_rank = relay::mpi::rank(comm);
int par_size = relay::mpi::size(comm);
Expand Down
2 changes: 1 addition & 1 deletion src/tests/blueprint/c/t_c_blueprint_mesh.cpp
View file
Expand Up @@ -22,7 +22,7 @@ TEST(c_conduit_blueprint_mesh, create_and_verify)
conduit_node *nempty = conduit_node_create();
conduit_node *info = conduit_node_create();

EXPECT_FALSE(conduit_blueprint_mesh_verify(nempty,info));
EXPECT_TRUE(conduit_blueprint_mesh_verify(nempty,info));


conduit_blueprint_mesh_examples_braid("hexs",3,3,3,n);
Expand Down
2 changes: 1 addition & 1 deletion src/tests/blueprint/fortran/t_f_blueprint_mesh.f90
View file
Expand Up @@ -44,7 +44,7 @@ subroutine t_blueprint_mesh_create_and_verify
nindex = conduit_node_create()
info = conduit_node_create()

call assert_true( conduit_blueprint_mesh_verify(nempty,info) .eqv. .false. , "verify false on empty")
call assert_true( conduit_blueprint_mesh_verify(nempty,info) .eqv. .true. , "verify true on empty")
call conduit_blueprint_mesh_examples_braid("hexs",3_8,3_8,3_8,n)
call assert_true( conduit_blueprint_mesh_verify(n,info) .eqv. .true., "verify true on braid hexs")

Expand Down

0 comments on commit d9160ae

Please sign in to comment.