Skip to content

Commit

Permalink
Convert a few more locally-owned loops with filters.
Browse files Browse the repository at this point in the history
  • Loading branch information
bangerth committed Dec 15, 2021
1 parent 58c0c31 commit 178574d
Show file tree
Hide file tree
Showing 28 changed files with 400 additions and 404 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2819,9 +2819,9 @@ MGTwoLevelTransfer<dim, LinearAlgebra::distributed::Vector<Number>>::reinit(
IndexSet is_locally_owned_fine(cell_id_translator.size());
IndexSet is_locally_owned_coarse(cell_id_translator.size());

for (const auto &cell : dof_handler_fine.active_cell_iterators())
if (cell->is_locally_owned())
is_locally_owned_fine.add_index(cell_id_translator.translate(cell));
for (const auto &cell : dof_handler_fine.active_cell_iterators() |
IteratorFilters::LocallyOwnedCell())
is_locally_owned_fine.add_index(cell_id_translator.translate(cell));

for (const auto &cell : dof_handler_coarse.active_cell_iterators() |
IteratorFilters::LocallyOwnedCell())
Expand Down
3 changes: 2 additions & 1 deletion source/distributed/fully_distributed_tria.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <deal.II/distributed/fully_distributed_tria.h>

#include <deal.II/grid/filtered_iterator.h>
#include <deal.II/grid/grid_tools.h>

#include <fstream>
Expand Down Expand Up @@ -173,7 +174,7 @@ namespace parallel

// 4a) set all cells artificial (and set the actual
// (level_)subdomain_ids in the next step)
for (auto cell = this->begin(); cell != this->end(); ++cell)
for (const auto &cell : this->cell_iterators())
{
if (cell->is_active())
cell->set_subdomain_id(
Expand Down
15 changes: 8 additions & 7 deletions source/distributed/grid_refinement.cc
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

# include <deal.II/distributed/grid_refinement.h>

# include <deal.II/grid/filtered_iterator.h>
# include <deal.II/grid/grid_refinement.h>
# include <deal.II/grid/tria.h>
# include <deal.II/grid/tria_accessor.h>
Expand Down Expand Up @@ -110,13 +111,13 @@ namespace
ExcInternalError());

unsigned int owned_index = 0;
for (const auto &cell : tria.active_cell_iterators())
if (cell->subdomain_id() == tria.locally_owned_subdomain())
{
locally_owned_indicators(owned_index) =
criteria(cell->active_cell_index());
++owned_index;
}
for (const auto &cell :
tria.active_cell_iterators() | IteratorFilters::LocallyOwnedCell())
{
locally_owned_indicators(owned_index) =
criteria(cell->active_cell_index());
++owned_index;
}
Assert(owned_index == tria.n_locally_owned_active_cells(),
ExcInternalError());
}
Expand Down
19 changes: 9 additions & 10 deletions source/distributed/repartitioning_policy_tools.cc
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,10 @@ namespace RepartitioningPolicyTools

// step 1) check if all processes have enough cells

unsigned int n_locally_owned_active_cells = 0;
for (const auto &cell : tria_in.active_cell_iterators())
if (cell->is_locally_owned())
++n_locally_owned_active_cells;
const auto locally_owned_cells =
tria_in.active_cell_iterators() | IteratorFilters::LocallyOwnedCell();
const unsigned int n_locally_owned_active_cells =
std::distance(locally_owned_cells.begin(), locally_owned_cells.end());

const auto comm = tria_in.get_communicator();

Expand Down Expand Up @@ -293,12 +293,11 @@ namespace RepartitioningPolicyTools
const auto n_subdomains = Utilities::MPI::n_mpi_processes(mpi_communicator);

// determine weight of each cell
for (const auto &cell : tria->active_cell_iterators())
if (cell->is_locally_owned())
weights[partitioner->global_to_local(
cell->global_active_cell_index())] =
weighting_function(
cell, Triangulation<dim, spacedim>::CellStatus::CELL_PERSIST);
for (const auto &cell :
tria->active_cell_iterators() | IteratorFilters::LocallyOwnedCell())
weights[partitioner->global_to_local(cell->global_active_cell_index())] =
weighting_function(
cell, Triangulation<dim, spacedim>::CellStatus::CELL_PERSIST);

// determine weight of all the cells locally owned by this process
uint64_t process_local_weight = 0;
Expand Down
25 changes: 10 additions & 15 deletions source/distributed/shared_tria.cc
Original file line number Diff line number Diff line change
Expand Up @@ -282,14 +282,11 @@ namespace parallel
# ifdef DEBUG
{
// Assert that each cell is owned by a processor
unsigned int n_my_cells = 0;
typename parallel::shared::Triangulation<dim,
spacedim>::active_cell_iterator
cell = this->begin_active(),
endc = this->end();
for (; cell != endc; ++cell)
if (cell->is_locally_owned())
n_my_cells += 1;
const unsigned int n_my_cells = std::count_if(
this->begin_active(),
typename Triangulation<dim, spacedim>::active_cell_iterator(
this->end()),
[](const auto &i) { return (i.is_locally_owned()); });

const unsigned int total_cells =
Utilities::MPI::sum(n_my_cells, this->get_communicator());
Expand All @@ -301,13 +298,11 @@ namespace parallel
// cell is owned by a processor
if (settings & construct_multigrid_hierarchy)
{
unsigned int n_my_cells = 0;
typename parallel::shared::Triangulation<dim, spacedim>::cell_iterator
cell = this->begin(),
endc = this->end();
for (; cell != endc; ++cell)
if (cell->is_locally_owned_on_level())
n_my_cells += 1;
const unsigned int n_my_cells =
std::count_if(this->begin(), this->end(), [](const auto &i) {
return (i.is_locally_owned_on_level());
});


const unsigned int total_cells =
Utilities::MPI::sum(n_my_cells, this->get_communicator());
Expand Down
80 changes: 44 additions & 36 deletions source/distributed/tria_base.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <deal.II/distributed/shared_tria.h>
#include <deal.II/distributed/tria_base.h>

#include <deal.II/grid/filtered_iterator.h>
#include <deal.II/grid/grid_tools.h>
#include <deal.II/grid/tria.h>
#include <deal.II/grid/tria_accessor.h>
Expand Down Expand Up @@ -178,11 +179,16 @@ namespace parallel
}

if (this->n_levels() > 0)
for (const auto &cell : this->active_cell_iterators())
if (cell->subdomain_id() == my_subdomain)
++number_cache.n_locally_owned_active_cells;
number_cache.n_locally_owned_active_cells = std::count_if(
this->begin_active(),
typename Triangulation<dim, spacedim>::active_cell_iterator(
this->end()),
[](const auto &i) { return i.is_locally_owned(); });
else
number_cache.n_locally_owned_active_cells = 0;

// Potentially cast to a 64 bit type before accumulating to avoid overflow:
// Potentially cast to a 64 bit type before accumulating to avoid
// overflow:
number_cache.n_global_active_cells =
Utilities::MPI::sum(static_cast<types::global_cell_index>(
number_cache.n_locally_owned_active_cells),
Expand All @@ -191,7 +197,8 @@ namespace parallel
number_cache.n_global_levels =
Utilities::MPI::max(this->n_levels(), this->mpi_communicator);

// Store MPI ranks of level ghost owners of this processor on all levels.
// Store MPI ranks of level ghost owners of this processor on all
// levels.
if (this->is_multilevel_hierarchy_constructed() == true)
{
number_cache.level_ghost_owners.clear();
Expand All @@ -201,18 +208,15 @@ namespace parallel
return;

// find level ghost owners
for (typename Triangulation<dim, spacedim>::cell_iterator cell =
this->begin();
cell != this->end();
++cell)
for (const auto &cell : this->cell_iterators())
if (cell->level_subdomain_id() != numbers::artificial_subdomain_id &&
cell->level_subdomain_id() != this->locally_owned_subdomain())
this->number_cache.level_ghost_owners.insert(
cell->level_subdomain_id());

# ifdef DEBUG
// Check that level_ghost_owners is symmetric by sending a message to
// everyone
// Check that level_ghost_owners is symmetric by sending a message
// to everyone
{
int ierr = MPI_Barrier(this->mpi_communicator);
AssertThrowMPI(ierr);
Expand Down Expand Up @@ -1009,10 +1013,11 @@ namespace parallel
cell_sizes_variable_cumulative.end(),
cell_sizes_variable_cumulative.begin());

// Serialize cumulative variable size vector value-by-value.
// This way we can circumvent the overhead of storing the
// container object as a whole, since we know its size by
// the number of registered callback functions.
// Serialize cumulative variable size vector
// value-by-value. This way we can circumvent the overhead
// of storing the container object as a whole, since we
// know its size by the number of registered callback
// functions.
data_fixed_it->resize(n_callbacks_variable *
sizeof(unsigned int));
for (unsigned int i = 0; i < n_callbacks_variable; ++i)
Expand Down Expand Up @@ -1048,10 +1053,10 @@ namespace parallel
// functions (i.e. a cell that was not flagged with CELL_INVALID)
// and store the sizes of each buffer.
//
// To deal with the case that at least one of the processors does not own
// any cell at all, we will exchange the information about the data sizes
// among them later. The code in between is still well-defined, since the
// following loops will be skipped.
// To deal with the case that at least one of the processors does not
// own any cell at all, we will exchange the information about the data
// sizes among them later. The code in between is still well-defined,
// since the following loops will be skipped.
std::vector<unsigned int> local_sizes_fixed(
1 + n_callbacks_fixed + (variable_size_data_stored ? 1 : 0));
for (const auto &data_cell : packed_fixed_size_data)
Expand Down Expand Up @@ -1122,7 +1127,8 @@ namespace parallel
src_sizes_variable.end(),
std::vector<int>::size_type(0));

// Move every piece of packed fixed size data into the consecutive buffer.
// Move every piece of packed fixed size data into the consecutive
// buffer.
src_data_fixed.reserve(expected_size_fixed);
for (const auto &data_cell_fixed : packed_fixed_size_data)
{
Expand Down Expand Up @@ -1222,8 +1228,8 @@ namespace parallel
{
// We decode the handle returned by register_data_attach() back into
// a format we can use. All even handles belong to those callback
// functions which write/read variable size data, all odd handles interact
// with fixed size buffers.
// functions which write/read variable size data, all odd handles
// interact with fixed size buffers.
const bool callback_variable_transfer = (handle % 2 == 0);
const unsigned int callback_index = handle / 2;

Expand Down Expand Up @@ -1314,7 +1320,8 @@ namespace parallel
spacedim>::CELL_INVALID)
{
// Extract the corresponding values for offset and size from
// the cumulative sizes array stored in the fixed size buffer.
// the cumulative sizes array stored in the fixed size
// buffer.
if (callback_index == 0)
offset = 0;
else
Expand Down Expand Up @@ -1427,8 +1434,9 @@ namespace parallel
// ------------------

// Write cumulative sizes to file.
// Since each processor owns the same information about the data sizes,
// it is sufficient to let only the first processor perform this task.
// Since each processor owns the same information about the data
// sizes, it is sufficient to let only the first processor perform
// this task.
if (myrank == 0)
{
ierr = MPI_File_write_at(fh,
Expand All @@ -1445,8 +1453,8 @@ namespace parallel
const MPI_Offset size_header =
sizes_fixed_cumulative.size() * sizeof(unsigned int);

// Make sure we do the following computation in 64bit integers to be able
// to handle 4GB+ files:
// Make sure we do the following computation in 64bit integers to be
// able to handle 4GB+ files:
const MPI_Offset my_global_file_position =
size_header +
static_cast<MPI_Offset>(global_first_cell) * bytes_per_cell;
Expand Down Expand Up @@ -1518,8 +1526,8 @@ namespace parallel
const MPI_Offset my_global_file_position =
static_cast<MPI_Offset>(global_first_cell) * sizeof(unsigned int);

// It is very unlikely that a single process has more than 2 billion
// cells, but we might as well check.
// It is very unlikely that a single process has more than 2
// billion cells, but we might as well check.
AssertThrow(src_sizes_variable.size() <
static_cast<std::size_t>(
std::numeric_limits<int>::max()),
Expand Down Expand Up @@ -1639,9 +1647,9 @@ namespace parallel
AssertThrowMPI(ierr);

// Read cumulative sizes from file.
// Since all processors need the same information about the data sizes,
// let each of them retrieve it by reading from the same location in
// the file.
// Since all processors need the same information about the data
// sizes, let each of them retrieve it by reading from the same
// location in the file.
sizes_fixed_cumulative.resize(1 + n_attached_deserialize_fixed +
(variable_size_data_stored ? 1 : 0));
ierr = MPI_File_read_at(fh,
Expand All @@ -1661,8 +1669,8 @@ namespace parallel
const MPI_Offset size_header =
sizes_fixed_cumulative.size() * sizeof(unsigned int);

// Make sure we do the following computation in 64bit integers to be able
// to handle 4GB+ files:
// Make sure we do the following computation in 64bit integers to be
// able to handle 4GB+ files:
const MPI_Offset my_global_file_position =
size_header +
static_cast<MPI_Offset>(global_first_cell) * bytes_per_cell;
Expand Down Expand Up @@ -1736,8 +1744,8 @@ namespace parallel
AssertThrowMPI(ierr);


// Compute my data size in bytes and compute prefix sum. We do this in
// 64 bit to avoid overflow for files larger than 4 GB:
// Compute my data size in bytes and compute prefix sum. We do this
// in 64 bit to avoid overflow for files larger than 4 GB:
const std::uint64_t size_on_proc =
std::accumulate(dest_sizes_variable.begin(),
dest_sizes_variable.end(),
Expand Down
18 changes: 9 additions & 9 deletions source/dofs/dof_tools.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1061,16 +1061,16 @@ namespace DoFTools
std::vector<types::global_dof_index> dof_indices;
std::set<types::global_dof_index> global_dof_indices;

for (const auto &cell : dof_handler.active_cell_iterators())
if (cell->is_locally_owned())
{
dof_indices.resize(cell->get_fe().n_dofs_per_cell());
cell->get_dof_indices(dof_indices);
for (const auto &cell : dof_handler.active_cell_iterators() |
IteratorFilters::LocallyOwnedCell())
{
dof_indices.resize(cell->get_fe().n_dofs_per_cell());
cell->get_dof_indices(dof_indices);

for (const types::global_dof_index dof_index : dof_indices)
if (!dof_set.is_element(dof_index))
global_dof_indices.insert(dof_index);
}
for (const types::global_dof_index dof_index : dof_indices)
if (!dof_set.is_element(dof_index))
global_dof_indices.insert(dof_index);
}

dof_set.add_indices(global_dof_indices.begin(), global_dof_indices.end());

Expand Down
10 changes: 5 additions & 5 deletions source/grid/grid_tools.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3905,11 +3905,11 @@ namespace GridTools
// In a first step, obtain the weights of the locally owned
// cells. For all others, the weight remains at the zero the
// vector was initialized with above.
for (const auto &cell : triangulation.active_cell_iterators())
if (cell->is_locally_owned())
cell_weights[cell->active_cell_index()] =
triangulation.signals.cell_weight(
cell, Triangulation<dim, spacedim>::CellStatus::CELL_PERSIST);
for (const auto &cell : triangulation.active_cell_iterators() |
IteratorFilters::LocallyOwnedCell())
cell_weights[cell->active_cell_index()] =
triangulation.signals.cell_weight(
cell, Triangulation<dim, spacedim>::CellStatus::CELL_PERSIST);

// If this is a parallel triangulation, we then need to also
// get the weights for all other cells. We have asserted above
Expand Down
7 changes: 3 additions & 4 deletions source/grid/grid_tools_cache.cc
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,9 @@ namespace GridTools
typename Triangulation<dim, spacedim>::active_cell_iterator>>
boxes;
boxes.reserve(tria->n_active_cells());
for (const auto &cell : tria->active_cell_iterators())
if (cell->is_locally_owned())
boxes.emplace_back(
std::make_pair(mapping->get_bounding_box(cell), cell));
for (const auto &cell : tria->active_cell_iterators() |
IteratorFilters::LocallyOwnedCell())
boxes.emplace_back(mapping->get_bounding_box(cell), cell);

locally_owned_cell_bounding_boxes_rtree = pack_rtree(boxes);
update_flags =
Expand Down

0 comments on commit 178574d

Please sign in to comment.