Skip to content

Commit

Permalink
Merge pull request #13326 from bangerth/ca-interface-2
Browse files Browse the repository at this point in the history 
Use the new CA interface instead of using AnonymousProcess.
  • Loading branch information
@drwells
drwells committed Feb 10, 2022
2 parents b22f442 + ede78d0 commit 7e5b33a
Show file tree
Hide file tree
Showing 5 changed files with 191 additions and 193 deletions.
35 changes: 14 additions & 21 deletions include/deal.II/fe/fe_tools_extrapolate.templates.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1139,57 +1139,50 @@ namespace FETools
// communication of information in cases where we do not know up
// front which processes (and from how many processes) we have to
// expect information from.
const auto get_destinations = [&destinations]() { return destinations; };

const auto create_request =
[&cells_to_send](const types::subdomain_id other_rank,
std::vector<char> & send_buffer) {
[&cells_to_send](const types::subdomain_id other_rank) {
std::vector<CellData> cells_for_this_destination;
for (const auto &cell : cells_to_send)
if (cell.receiver == other_rank)
cells_for_this_destination.emplace_back(cell);

send_buffer = Utilities::pack(cells_for_this_destination, false);
return Utilities::pack(cells_for_this_destination, false);
};

const auto answer_request =
[&received_cells](const unsigned int other_rank,
const std::vector<char> &buffer_recv,
std::vector<char> & request_buffer) {
const std::vector<char> &request) {
// We got a message from 'other_rank', so let us decode the
// message in the same way as we have assembled it above.
// Note that the cells just received do not contain
// information where they came from, and we have to add that
// ourselves for later use.
for (CellData &cell_data :
Utilities::unpack<std::vector<CellData>>(buffer_recv, false))
Utilities::unpack<std::vector<CellData>>(request, false))
{
cell_data.receiver = other_rank;
received_cells.emplace_back(std::move(cell_data));
}

// Nothing left to do here, we don't actually need to provide an
// answer:
request_buffer.clear();
return std::vector<char>();
};

const auto read_answer = [](const unsigned int /*other_rank*/,
const std::vector<char> &recv_buffer) {
const std::vector<char> &answer) {
// We don't put anything into the answers, so nothing should
// have been coming out at this end either:
(void)recv_buffer;
Assert(recv_buffer.size() == 0, ExcInternalError());
(void)answer;
Assert(answer.size() == 0, ExcInternalError());
};

Utilities::MPI::ConsensusAlgorithms::AnonymousProcess<char, char>
operations(get_destinations,
create_request,
answer_request,
read_answer);

Utilities::MPI::ConsensusAlgorithms::Selector<char, char>(operations,
communicator)
.run();
Utilities::MPI::ConsensusAlgorithms::Selector<char, char>().run(
destinations,
create_request,
answer_request,
read_answer,
communicator);
}


Expand Down
111 changes: 56 additions & 55 deletions include/deal.II/matrix_free/matrix_free.templates.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2094,65 +2094,66 @@ MatrixFree<dim, Number, VectorizedArrayType>::initialize_indices(
});

// get offsets of shared cells
Utilities::MPI::ConsensusAlgorithms::
AnonymousProcess<dealii::types::global_dof_index, unsigned int>
process(
[&]() {
std::vector<unsigned int> targets;
for (unsigned int i = 0; i < cells_shared_ghosts.size(); ++i)
if (cells_shared_ghosts[i].size() > 0)
targets.push_back(i);
return targets;
},
[&](const auto other_rank, auto &send_buffer) {
auto &source = cells_shared_ghosts[other_rank];
std::sort(source.begin(),
source.end(),
[](const auto &a, const auto &b) {
return a.second < b.second;
});

send_buffer.reserve(source.size());
for (const auto &i : source)
send_buffer.push_back(i.second);
},
[&](const auto &other_rank,
const auto &buffer_recv,
auto & request_buffer) {
(void)other_rank;

request_buffer.reserve(buffer_recv.size());

unsigned int j = 0;

for (unsigned int i = 0; i < buffer_recv.size(); ++i)
{
for (; j < cells_locally_owned.size(); ++j)
if (cells_locally_owned[j].second == buffer_recv[i])
{
request_buffer.push_back(
cells_locally_owned[j].first);
break;
}
}
std::vector<unsigned int> targets;
for (unsigned int i = 0; i < cells_shared_ghosts.size(); ++i)
if (cells_shared_ghosts[i].size() > 0)
targets.push_back(i);

// Then set up the callbacks the consensus algorithm needs:
const auto create_request = [&](const auto other_rank) {
auto &source = cells_shared_ghosts[other_rank];
std::sort(source.begin(),
source.end(),
[](const auto &a, const auto &b) {
return a.second < b.second;
});

std::vector<dealii::types::global_dof_index> send_buffer;
send_buffer.reserve(source.size());
for (const auto &i : source)
send_buffer.push_back(i.second);

return send_buffer;
};

const auto answer_request = [&](const auto, const auto &request) {
std::vector<unsigned int> answer;
answer.reserve(request.size());

AssertDimension(request_buffer.size(), buffer_recv.size());
},
[&](const auto other_rank, const auto &recv_buffer) {
Assert(recv_buffer.size() ==
cells_shared_ghosts[other_rank].size(),
ExcInternalError());
for (unsigned int i = 0; i < recv_buffer.size(); ++i)
unsigned int j = 0;

for (unsigned int i = 0; i < request.size(); ++i)
{
for (; j < cells_locally_owned.size(); ++j)
if (cells_locally_owned[j].second == request[i])
{
cells[cells_shared_ghosts[other_rank][i].first] = {
other_rank, recv_buffer[i]};
answer.push_back(cells_locally_owned[j].first);
break;
}
});
}

AssertDimension(answer.size(), request.size());
return answer;
};

Utilities::MPI::ConsensusAlgorithms::Selector<
dealii::types::global_dof_index,
unsigned int>(process, communicator_sm)
.run();
const auto process_answer = [&](const auto other_rank,
const auto &answer) {
Assert(answer.size() == cells_shared_ghosts[other_rank].size(),
ExcInternalError());
for (unsigned int i = 0; i < answer.size(); ++i)
{
cells[cells_shared_ghosts[other_rank][i].first] = {other_rank,
answer[i]};
}
};

Utilities::MPI::ConsensusAlgorithms::
Selector<dealii::types::global_dof_index, unsigned int>()
.run(targets,
create_request,
answer_request,
process_answer,
communicator_sm);

return cells;
}();
Expand Down
15 changes: 5 additions & 10 deletions source/base/mpi.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,11 +432,8 @@ namespace Utilities
if (Utilities::MPI::min((my_destinations_are_unique ? 1 : 0), mpi_comm) ==
1)
{
ConsensusAlgorithms::AnonymousProcess<char, char> process(
[&]() { return destinations; });
ConsensusAlgorithms::NBX<char, char> consensus_algorithm(process,
mpi_comm);
return consensus_algorithm.run();
return ConsensusAlgorithms::NBX<char, char>().run(
destinations, {}, {}, {}, mpi_comm);
}
// If that was not the case, we need to use the remainder of the code
// below, i.e., just fall through the if condition above.
Expand Down Expand Up @@ -591,11 +588,9 @@ namespace Utilities
if (Utilities::MPI::min((my_destinations_are_unique ? 1 : 0), mpi_comm) ==
1)
{
ConsensusAlgorithms::AnonymousProcess<char, char> process(
[&]() { return destinations; });
ConsensusAlgorithms::NBX<char, char> consensus_algorithm(process,
mpi_comm);
return consensus_algorithm.run().size();
return ConsensusAlgorithms::NBX<char, char>()
.run(destinations, {}, {}, {}, mpi_comm)
.size();
}
else
# endif
Expand Down
156 changes: 80 additions & 76 deletions source/grid/grid_tools.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5933,94 +5933,98 @@ namespace GridTools
(std::find(marked_vertices.begin(), marked_vertices.end(), true) !=
marked_vertices.end());

Utilities::MPI::ConsensusAlgorithms::AnonymousProcess<char, char> process(
[&]() { return potential_owners_ranks; },
[&](const unsigned int other_rank, std::vector<char> &send_buffer) {
const auto other_rank_index = translate(other_rank);
const auto create_request = [&](const unsigned int other_rank) {
const auto other_rank_index = translate(other_rank);

std::vector<std::pair<unsigned int, Point<spacedim>>> temp;
temp.reserve(potential_owners_ptrs[other_rank_index + 1] -
potential_owners_ptrs[other_rank_index]);
std::vector<std::pair<unsigned int, Point<spacedim>>> temp;
temp.reserve(potential_owners_ptrs[other_rank_index + 1] -
potential_owners_ptrs[other_rank_index]);

for (unsigned int i = potential_owners_ptrs[other_rank_index];
i < potential_owners_ptrs[other_rank_index + 1];
++i)
temp.emplace_back(potential_owners_indices[i],
points[potential_owners_indices[i]]);
for (unsigned int i = potential_owners_ptrs[other_rank_index];
i < potential_owners_ptrs[other_rank_index + 1];
++i)
temp.emplace_back(potential_owners_indices[i],
points[potential_owners_indices[i]]);

send_buffer = Utilities::pack(temp, false);
},
[&](const unsigned int & other_rank,
const std::vector<char> &recv_buffer,
std::vector<char> & request_buffer) {
const auto recv_buffer_unpacked = Utilities::unpack<
std::vector<std::pair<unsigned int, Point<spacedim>>>>(recv_buffer,
false);
return Utilities::pack(temp, false);
};

std::vector<unsigned int> request_buffer_temp(
recv_buffer_unpacked.size(), 0);
const auto answer_request =
[&](const unsigned int & other_rank,
const std::vector<char> &request) -> std::vector<char> {
const auto recv_buffer_unpacked = Utilities::unpack<
std::vector<std::pair<unsigned int, Point<spacedim>>>>(request,
false);

if (has_relevant_vertices)
{
cell_hint = cache.get_triangulation().begin_active();
std::vector<unsigned int> request_buffer_temp(
recv_buffer_unpacked.size(), 0);

for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i)
{
const auto &index_and_point = recv_buffer_unpacked[i];
if (has_relevant_vertices)
{
cell_hint = cache.get_triangulation().begin_active();

const auto cells_and_reference_positions =
find_all_locally_owned_active_cells_around_point(
cache,
for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i)
{
const auto &index_and_point = recv_buffer_unpacked[i];

const auto cells_and_reference_positions =
find_all_locally_owned_active_cells_around_point(
cache,
index_and_point.second,
cell_hint,
marked_vertices,
tolerance,
enforce_unique_mapping);

for (const auto &cell_and_reference_position :
cells_and_reference_positions)
{
send_components.emplace_back(
std::pair<int, int>(
cell_and_reference_position.first->level(),
cell_and_reference_position.first->index()),
other_rank,
index_and_point.first,
cell_and_reference_position.second,
index_and_point.second,
cell_hint,
marked_vertices,
tolerance,
enforce_unique_mapping);
numbers::invalid_unsigned_int);
}

for (const auto &cell_and_reference_position :
cells_and_reference_positions)
{
send_components.emplace_back(
std::pair<int, int>(
cell_and_reference_position.first->level(),
cell_and_reference_position.first->index()),
other_rank,
index_and_point.first,
cell_and_reference_position.second,
index_and_point.second,
numbers::invalid_unsigned_int);
}
request_buffer_temp[i] = cells_and_reference_positions.size();
}
}

request_buffer_temp[i] = cells_and_reference_positions.size();
}
}
if (perform_handshake)
return Utilities::pack(request_buffer_temp, false);
else
return {};
};

if (perform_handshake)
request_buffer = Utilities::pack(request_buffer_temp, false);
},
[&](const unsigned int other_rank,
const std::vector<char> &recv_buffer) {
if (perform_handshake)
{
const auto recv_buffer_unpacked =
Utilities::unpack<std::vector<unsigned int>>(recv_buffer,
false);

const auto other_rank_index = translate(other_rank);

for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i)
for (unsigned int j = 0; j < recv_buffer_unpacked[i]; ++j)
recv_components.emplace_back(
other_rank,
potential_owners_indices
[i + potential_owners_ptrs[other_rank_index]],
numbers::invalid_unsigned_int);
}
});
const auto process_answer = [&](const unsigned int other_rank,
const std::vector<char> &answer) {
if (perform_handshake)
{
const auto recv_buffer_unpacked =
Utilities::unpack<std::vector<unsigned int>>(answer, false);

const auto other_rank_index = translate(other_rank);

for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i)
for (unsigned int j = 0; j < recv_buffer_unpacked[i]; ++j)
recv_components.emplace_back(
other_rank,
potential_owners_indices
[i + potential_owners_ptrs[other_rank_index]],
numbers::invalid_unsigned_int);
}
};

Utilities::MPI::ConsensusAlgorithms::Selector<char, char>(
process, cache.get_triangulation().get_communicator())
.run();
Utilities::MPI::ConsensusAlgorithms::Selector<char, char>().run(
potential_owners_ranks,
create_request,
answer_request,
process_answer,
cache.get_triangulation().get_communicator());

if (true)
{
Expand Down

0 comments on commit 7e5b33a

Please sign in to comment.