Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid double packing/unpacking for CA algorithms. #13733

Merged
merged 1 commit into from
May 14, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
61 changes: 31 additions & 30 deletions include/deal.II/fe/fe_tools_extrapolate.templates.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <deal.II/base/config.h>

#include <deal.II/base/mpi_consensus_algorithms.h>
#include <deal.II/base/mpi_consensus_algorithms.templates.h>

#include <deal.II/distributed/p4est_wrappers.h>
#include <deal.II/distributed/tria.h>
Expand Down Expand Up @@ -1140,45 +1141,45 @@ namespace FETools
// front which processes (and from how many processes) we have to
// expect information from.
const auto create_request =
[&cells_to_send](const types::subdomain_id other_rank) {
std::vector<CellData> cells_for_this_destination;
for (const auto &cell : cells_to_send)
if (cell.receiver == other_rank)
cells_for_this_destination.emplace_back(cell);

return Utilities::pack(cells_for_this_destination, false);
};
[&cells_to_send](
const types::subdomain_id other_rank) -> std::vector<CellData> {
std::vector<CellData> cells_for_this_destination;
for (const auto &cell : cells_to_send)
if (cell.receiver == other_rank)
cells_for_this_destination.emplace_back(cell);

return cells_for_this_destination;
};

const auto answer_request =
[&received_cells](const unsigned int other_rank,
const std::vector<char> &request) {
// We got a message from 'other_rank', so let us decode the
// message in the same way as we have assembled it above.
// Note that the cells just received do not contain
// information where they came from, and we have to add that
// ourselves for later use.
for (CellData &cell_data :
Utilities::unpack<std::vector<CellData>>(request, false))
{
cell_data.receiver = other_rank;
received_cells.emplace_back(std::move(cell_data));
}
[&received_cells](const unsigned int other_rank,
const std::vector<CellData> &request) -> int {
// We got a message from 'other_rank', so let us decode the
// message in the same way as we have assembled it above.
// Note that the cells just received do not contain
// information where they came from, and we have to add that
// ourselves for later use.
for (CellData cell_data : request)
{
cell_data.receiver = other_rank;
received_cells.emplace_back(std::move(cell_data));
}

// Nothing left to do here, we don't actually need to provide an
// answer:
return std::vector<char>();
};
// Nothing left to do here, we don't actually need to provide an
// answer:
return 0;
};

const auto read_answer = [](const unsigned int /*other_rank*/,
const std::vector<char> &answer) {
const int &answer) {
// We don't put anything into the answers, so nothing should
// have been coming out at this end either:
// have been coming out at this end either that differs from
// the default-sent zero integer:
(void)answer;
Assert(answer.size() == 0, ExcInternalError());
Assert(answer == 0, ExcInternalError());
};

Utilities::MPI::ConsensusAlgorithms::selector<std::vector<char>,
std::vector<char>>(
Utilities::MPI::ConsensusAlgorithms::selector<std::vector<CellData>, int>(
destinations,
create_request,
answer_request,
Expand Down
50 changes: 23 additions & 27 deletions source/grid/grid_tools.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <deal.II/base/mpi.h>
#include <deal.II/base/mpi.templates.h>
#include <deal.II/base/mpi_consensus_algorithms.h>
#include <deal.II/base/mpi_consensus_algorithms.templates.h>
#include <deal.II/base/quadrature_lib.h>
#include <deal.II/base/thread_management.h>

Expand Down Expand Up @@ -58,6 +59,7 @@

#include <deal.II/physics/transformations.h>


DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_real_distribution.hpp>
Expand Down Expand Up @@ -5985,6 +5987,9 @@ namespace GridTools
"The marked_vertices vector has to be either empty or its size has "
"to equal the number of vertices of the triangulation."));

using RequestType = std::vector<std::pair<unsigned int, Point<spacedim>>>;
using AnswerType = std::vector<unsigned int>;

// In the case that a marked_vertices vector has been given and none
// of its entries is true, we know that this process does not own
// any of the incoming points (and it will not send any data) so
Expand All @@ -5997,36 +6002,31 @@ namespace GridTools
const auto create_request = [&](const unsigned int other_rank) {
const auto other_rank_index = translate(other_rank);

std::vector<std::pair<unsigned int, Point<spacedim>>> temp;
temp.reserve(potential_owners_ptrs[other_rank_index + 1] -
potential_owners_ptrs[other_rank_index]);
RequestType request;
request.reserve(potential_owners_ptrs[other_rank_index + 1] -
potential_owners_ptrs[other_rank_index]);

for (unsigned int i = potential_owners_ptrs[other_rank_index];
i < potential_owners_ptrs[other_rank_index + 1];
++i)
temp.emplace_back(potential_owners_indices[i],
points[potential_owners_indices[i]]);
request.emplace_back(potential_owners_indices[i],
points[potential_owners_indices[i]]);

return Utilities::pack(temp, false);
return request;
};

const auto answer_request =
[&](const unsigned int & other_rank,
const std::vector<char> &request) -> std::vector<char> {
const auto recv_buffer_unpacked = Utilities::unpack<
std::vector<std::pair<unsigned int, Point<spacedim>>>>(request,
false);

std::vector<unsigned int> request_buffer_temp(
recv_buffer_unpacked.size(), 0);
[&](const unsigned int &other_rank,
const RequestType & request) -> AnswerType {
AnswerType answer(request.size(), 0);

if (has_relevant_vertices)
{
cell_hint = cache.get_triangulation().begin_active();

for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i)
for (unsigned int i = 0; i < request.size(); ++i)
{
const auto &index_and_point = recv_buffer_unpacked[i];
const auto &index_and_point = request[i];

const auto cells_and_reference_positions =
find_all_locally_owned_active_cells_around_point(
Expand All @@ -6051,27 +6051,24 @@ namespace GridTools
numbers::invalid_unsigned_int);
}

request_buffer_temp[i] = cells_and_reference_positions.size();
answer[i] = cells_and_reference_positions.size();
}
}

if (perform_handshake)
return Utilities::pack(request_buffer_temp, false);
return answer;
else
return {};
};

const auto process_answer = [&](const unsigned int other_rank,
const std::vector<char> &answer) {
const auto process_answer = [&](const unsigned int other_rank,
const AnswerType & answer) {
if (perform_handshake)
{
const auto recv_buffer_unpacked =
Utilities::unpack<std::vector<unsigned int>>(answer, false);

const auto other_rank_index = translate(other_rank);

for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i)
for (unsigned int j = 0; j < recv_buffer_unpacked[i]; ++j)
for (unsigned int i = 0; i < answer.size(); ++i)
for (unsigned int j = 0; j < answer[i]; ++j)
recv_components.emplace_back(
other_rank,
potential_owners_indices
Expand All @@ -6080,8 +6077,7 @@ namespace GridTools
}
};

Utilities::MPI::ConsensusAlgorithms::selector<std::vector<char>,
std::vector<char>>(
Utilities::MPI::ConsensusAlgorithms::selector<RequestType, AnswerType>(
potential_owners_ranks,
create_request,
answer_request,
Expand Down
34 changes: 11 additions & 23 deletions source/grid/tria_description.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <deal.II/base/geometry_info.h>
#include <deal.II/base/mpi.h>
#include <deal.II/base/mpi_consensus_algorithms.h>
#include <deal.II/base/mpi_consensus_algorithms.templates.h>

#include <deal.II/distributed/fully_distributed_tria.h>
#include <deal.II/distributed/tria.h>
Expand Down Expand Up @@ -123,32 +124,19 @@ namespace TriangulationDescription
const auto other_rank_index =
std::distance(relevant_processes.begin(), ptr);

return dealii::Utilities::pack(description_temp[other_rank_index],
false);
};

const auto answer_request = [&](const unsigned int,
const std::vector<char> &request) {
this->merge(
dealii::Utilities::unpack<DescriptionTemp<dim, spacedim>>(request,
false),
vertices_have_unique_ids);
return std::vector<char>();
};

const auto process_answer = [](const unsigned int,
const std::vector<char> &answer) {
(void)answer;
Assert(answer.size() == 0, ExcInternalError());
return description_temp[other_rank_index];
};

const auto answer_request =
[&](const unsigned int,
const DescriptionTemp<dim, spacedim> &request) {
this->merge(request, vertices_have_unique_ids);
return char{};
};

dealii::Utilities::MPI::ConsensusAlgorithms::
selector<std::vector<char>, std::vector<char>>(relevant_processes,
create_request,
answer_request,
process_answer,
comm);
dealii::Utilities::MPI::ConsensusAlgorithms::selector<
DescriptionTemp<dim, spacedim>,
char>(relevant_processes, create_request, answer_request, {}, comm);
}

/**
Expand Down
1 change: 1 addition & 0 deletions source/multigrid/mg_tools.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <deal.II/base/logstream.h>
#include <deal.II/base/mg_level_object.h>
#include <deal.II/base/mpi_compute_index_owner_internal.h>
#include <deal.II/base/mpi_consensus_algorithms.templates.h>
#include <deal.II/base/thread_management.h>

#include <deal.II/dofs/dof_accessor.h>
Expand Down