Permalink
Browse files

Merge pull request #1646 from fweik/resort_flag

Resort flag
  • Loading branch information...
fweik committed Dec 4, 2017
2 parents 604fbbb + 037febd commit 3d4b341c9ce790dc80e28e93275db12880859cce
View
@@ -61,6 +61,9 @@ CellStructure cell_structure = {/* type */ CELL_STRUCTURE_NONEYET,
double max_range = 0.0;
/** On of Cells::Resort, annouces the level of resort needed.
*/
unsigned resort_particles = Cells::RESORT_NONE;
int rebuild_verletlist = 1;
/**
@@ -72,8 +75,7 @@ int rebuild_verletlist = 1;
*
* Pairs are sorted so that first.id < second.id
*/
std::vector<std::pair<int, int>>
get_pairs(double distance) {
std::vector<std::pair<int, int>> get_pairs(double distance) {
std::vector<std::pair<int, int>> ret;
auto const cutoff2 = distance * distance;
@@ -111,7 +113,7 @@ get_pairs(double distance) {
get_mi_vector(vec21, p1.r.p, p2.r.p);
vec21[2] = p1.r.p[2] - p2.r.p[2];
return sqrlen(vec21);
return sqrlen(vec21);
});
}
@@ -245,7 +247,7 @@ void cells_re_init(int new_cs) {
CELL_TRACE(fprintf(stderr, "%d: old cells deallocated\n", this_node));
/* to enforce initialization of the ghost cells */
resort_particles = 1;
resort_particles = Cells::RESORT_GLOBAL;
on_cell_structure_change();
}
@@ -272,11 +274,16 @@ void realloc_cells(int size) {
/*************************************************/
void announce_resort_particles() {
int sum;
void set_resort_particles(Cells::Resort level) {
resort_particles |= level;
assert(resort_particles & level);
}
unsigned const &get_resort_particles() { return resort_particles; }
MPI_Allreduce(&resort_particles, &sum, 1, MPI_INT, MPI_SUM, comm_cart);
resort_particles = (sum > 0) ? 1 : 0;
void announce_resort_particles() {
MPI_Allreduce(MPI_IN_PLACE, &resort_particles, 1, MPI_UNSIGNED, MPI_BOR,
comm_cart);
INTEG_TRACE(fprintf(stderr,
"%d: announce_resort_particles: resort_particles=%d\n",
@@ -324,7 +331,7 @@ void cells_resort_particles(int global_flag) {
/* Particles are now sorted, but verlet lists are invalid
and p_old has to be reset. */
resort_particles = 0;
resort_particles = Cells::RESORT_NONE;
rebuild_verletlist = 1;
on_resort_particles();
@@ -366,25 +373,27 @@ void cells_on_geometry_change(int flags) {
void check_resort_particles() {
const double skin2 = SQR(skin / 2.0);
resort_particles =
std::any_of(local_cells.particles().begin(),
local_cells.particles().end(), [&skin2](Particle const &p) {
return distance2(p.r.p, p.l.p_old) > skin2;
});
resort_particles |= (std::any_of(local_cells.particles().begin(),
local_cells.particles().end(),
[&skin2](Particle const &p) {
return distance2(p.r.p, p.l.p_old) > skin2;
}))
? Cells::RESORT_LOCAL
: Cells::RESORT_NONE;
announce_resort_particles();
}
/*************************************************/
void cells_update_ghosts() {
if (resort_particles) {
#ifdef LEES_EDWARDS
/* Communication step: number of ghosts and ghost information */
cells_resort_particles(CELL_GLOBAL_EXCHANGE);
#else
int global = (resort_particles & Cells::RESORT_GLOBAL)
? CELL_GLOBAL_EXCHANGE
: CELL_NEIGHBOR_EXCHANGE;
/* Communication step: number of ghosts and ghost information */
cells_resort_particles(CELL_NEIGHBOR_EXCHANGE);
#endif
cells_resort_particles(global);
} else
/* Communication step: ghost information */
ghost_communicator(&cell_structure.update_ghost_pos_comm);
View
@@ -97,6 +97,10 @@
/** Flag for exchange_and_sort_particles : Do neighbor exchange. */
#define CELL_NEIGHBOR_EXCHANGE 0
namespace Cells {
enum Resort : unsigned { RESORT_NONE = 0u, RESORT_LOCAL = 1u, RESORT_GLOBAL = 2u };
}
/** \name Flags for cells_on_geometry_change */
/*@{*/
@@ -289,6 +293,18 @@ int cells_get_n_particles();
*/
std::vector<std::pair<int, int>> mpi_get_pairs(double distance);
/**
* @brief Increase the local resort level at least to level.
*
* The changed level has to be commuicated via annouce_resort_particles.
*/
void set_resort_particles(Cells::Resort level);
/**
* @brief Get the currently scheduled resort level.
*/
unsigned const &get_resort_particles();
/** spread the particle resorting criterion across the nodes. */
void announce_resort_particles();
@@ -363,6 +363,7 @@ void mpi_place_particle(int pnode, int part, double p[3]) {
else
MPI_Send(p, 3, MPI_DOUBLE, pnode, SOME_TAG, comm_cart);
set_resort_particles(Cells::RESORT_GLOBAL);
on_particle_change();
}
@@ -374,6 +375,7 @@ void mpi_place_particle_slave(int pnode, int part) {
local_place_particle(part, p, 0);
}
set_resort_particles(Cells::RESORT_GLOBAL);
on_particle_change();
}
@@ -33,6 +33,11 @@
#include "lees_edwards_comms_manager.hpp"
#include "lees_edwards_domain_decomposition.hpp"
/** Returns pointer to the cell which corresponds to the position if
the position is in the nodes spatial domain otherwise a nullptr
pointer. */
Cell *dd_save_position_to_cell(double pos[3]);
/************************************************/
/** \name Defines */
/************************************************/
@@ -574,7 +579,7 @@ Cell *dd_save_position_to_cell(double pos[3]) {
for (i = 0; i < 3; i++) {
lpos = pos[i] - my_left[i];
cpos[i] = (int)(lpos * dd.inv_cell_size[i]) + 1;
cpos[i] = static_cast<int>(std::floor(lpos * dd.inv_cell_size[i])) + 1;
/* particles outside our box. Still take them if
VERY close or nonperiodic boundary */
@@ -596,37 +601,6 @@ Cell *dd_save_position_to_cell(double pos[3]) {
return &(cells[i]);
}
Cell *dd_position_to_cell(double pos[3]) {
int i, cpos[3];
double lpos;
for (i = 0; i < 3; i++) {
lpos = pos[i] - my_left[i];
cpos[i] = (int)(lpos * dd.inv_cell_size[i]) + 1;
if (cpos[i] < 1) {
cpos[i] = 1;
#ifdef ADDITIONAL_CHECKS
if (PERIODIC(i) && lpos < -ROUND_ERROR_PREC * box_l[i]) {
runtimeErrorMsg() << "particle @ (" << pos[0] << ", " << pos[1] << ", "
<< pos[2] << ") is outside of the allowed cell grid";
}
#endif
} else if (cpos[i] > dd.cell_grid[i]) {
cpos[i] = dd.cell_grid[i];
#ifdef ADDITIONAL_CHECKS
if (PERIODIC(i) && lpos > local_box_l[i] + ROUND_ERROR_PREC * box_l[i]) {
runtimeErrorMsg() << "particle @ (" << pos[0] << ", " << pos[1] << ", "
<< pos[2] << ") is outside of the allowed cell grid";
}
#endif
}
}
i = get_linear_index(cpos[0], cpos[1], cpos[2], dd.ghost_cell_grid);
return &cells[i];
}
void dd_position_to_cell_indices(double pos[3], int *idx) {
int i;
double lpos;
@@ -817,7 +791,7 @@ void dd_topology_init(CellPList *old) {
cell_structure.type = CELL_STRUCTURE_DOMDEC;
cell_structure.position_to_node = map_position_node_array;
cell_structure.position_to_cell = dd_position_to_cell;
cell_structure.position_to_cell = dd_save_position_to_cell;
/* set up new domain decomposition cell structure */
dd_create_cell_grid();
@@ -156,9 +156,6 @@ void dd_topology_release();
*/
void dd_exchange_and_sort_particles(int global_flag);
/** implements \ref CellStructure::position_to_cell. */
Cell *dd_position_to_cell(double pos[3]);
/** Get three cell indices (coordinates in cell gird) from particle position */
void dd_position_to_cell_indices(double pos[3], int *idx);
@@ -176,11 +173,6 @@ int calc_processor_min_num_cells();
*/
int dd_fill_comm_cell_lists(Cell **part_lists, int lc[3], int hc[3]);
/** Returns pointer to the cell which corresponds to the position if
the position is in the nodes spatial domain otherwise a nullptr
pointer. */
Cell *dd_save_position_to_cell(double pos[3]);
/** Of every two communication rounds, set the first receivers to prefetch and
* poststore */
void dd_assign_prefetches(GhostCommunicator *comm);
@@ -154,7 +154,7 @@ void IBM_UpdateParticlePositions(ParticleRange particles)
// Check if the particle might have crossed a box border (criterion see e-mail Axel 28.8.2014)
// if possible resort_particles = 1
const double dist2 = distance2( p[j].r.p, p[j].l.p_old);
if ( dist2 > skin2 ) { resort_particles = 1; }
if ( dist2 > skin2 ) { set_resort_particles(Cells::RESORT_LOCAL); }
}
}
View
@@ -144,7 +144,7 @@ void on_integration_start() {
INTEG_TRACE(fprintf(
stderr,
"%d: on_integration_start: reinit_thermo = %d, resort_particles=%d\n",
this_node, reinit_thermo, resort_particles));
this_node, reinit_thermo, get_resort_particles()));
/********************************************/
/* sanity checks */
@@ -217,6 +217,10 @@ void on_integration_start() {
runtimeErrorMsg() << "Nodes disagree about cell system type.";
}
if(!Utils::Mpi::all_compare(comm_cart, get_resort_particles())) {
runtimeErrorMsg() << "Nodes disagree about resort type.";
}
if(!Utils::Mpi::all_compare(comm_cart, cell_structure.use_verlet_list)) {
runtimeErrorMsg() << "Nodes disagree about use of verlet lists.";
}
@@ -239,8 +243,7 @@ void on_observable_calc() {
/* Prepare particle structure: Communication step: number of ghosts and ghost
* information */
if (resort_particles)
cells_resort_particles(CELL_GLOBAL_EXCHANGE);
cells_update_ghosts();
#ifdef ELECTROSTATICS
if (reinit_electrostatics) {
@@ -287,7 +290,7 @@ void on_observable_calc() {
void on_particle_change() {
EVENT_TRACE(fprintf(stderr, "%d: on_particle_change\n", this_node));
resort_particles = 1;
set_resort_particles(Cells::RESORT_LOCAL);
reinit_electrostatics = 1;
reinit_magnetostatics = 1;
View
@@ -85,7 +85,6 @@ double skin = 0.0;
double skin2 = 0.0;
bool skin_set = false;
int resort_particles = 1;
int recalc_forces = 1;
double verlet_reuse = 0.0;
@@ -837,7 +836,8 @@ void propagate_press_box_pos_and_rescale_npt() {
this_node, p.r.p[0], p.r.p[1], p.r.p[2]));
}
resort_particles = 1;
set_resort_particles(Cells::RESORT_LOCAL);
/* Apply new volume to the box-length, communicate it, and account for
* necessary adjustments to the cell geometry */
@@ -951,7 +951,7 @@ void propagate_pos() {
}
/* Verlet criterion check */
if (distance2(p.r.p, p.l.p_old) > skin2)
resort_particles = 1;
set_resort_particles(Cells::RESORT_LOCAL);
}
}
@@ -1024,7 +1024,7 @@ void propagate_vel_pos() {
p.r.p[1] += box_l[1];
p.l.i[1]--;
}
resort_particles = 1;
set_resort_particles(Cells::RESORT_LOCAL);
}
/* Branch prediction on most systems should mean there is minimal cost
* here */
@@ -1051,11 +1051,11 @@ void propagate_vel_pos() {
if (SQR(p.r.p[0] - p.l.p_old[0]) + SQR(p.r.p[1] - p.l.p_old[1]) +
SQR(p.r.p[2] - p.l.p_old[2]) >
skin2)
resort_particles = 1;
set_resort_particles(Cells::RESORT_LOCAL);
}
#ifdef LEES_EDWARDS /* would be nice to be more refined about this */
resort_particles = 1;
set_resort_particles(Cells::RESORT_GLOBAL);
#endif
announce_resort_particles();
View
@@ -75,8 +75,6 @@ extern double skin;
/** True iff the user has changed the skin setting. */
extern bool skin_set;
/** If non-zero, the particle data will be resorted before the next integration. */
extern int resort_particles;
/** If non-zero, the forces will be recalculated before the next integration. */
extern int recalc_forces;
/** Average number of integration steps the verlet list has been re
View
@@ -92,7 +92,7 @@ void layered_get_mi_vector(double res[3], double a[3], double b[3]) {
}
Cell *layered_position_to_cell(double pos[3]) {
int cpos = (int)((pos[2] - my_left[2]) * layer_h_i) + 1;
int cpos = static_cast<int>(std::floor((pos[2] - my_left[2]) * layer_h_i)) + 1;
if (cpos < 1) {
if (!LAYERED_BTM_NEIGHBOR)
cpos = 1;
@@ -59,10 +59,10 @@ void lees_edwards_step_boundaries(){
/* request a new verlet list */
rebuild_verletlist = 1;
/* request a redo of particle-cell assignment */
resort_particles = 1;
set_resort_particles(Cells::RESORT_GLOBAL);
/* Only part of the comms system needs to be rebuilt,
but it is still very slow. */
cells_on_geometry_change( CELL_FLAG_LEES_EDWARDS );
@@ -121,9 +121,10 @@ bool steepest_descent_step(void) {
f_max = std::max(f_max, f);
f_max = std::max(f_max, t);
dp2_max = std::max(dp2_max, dp2);
resort_particles = 1;
}
set_resort_particles(Cells::RESORT_LOCAL);
MINIMIZE_ENERGY_TRACE(
printf("f_max %e resort_particles %d\n", f_max, resort_particles));
announce_resort_particles();
View
@@ -446,6 +446,6 @@ void mpi_mpiio_common_read(const char *filename, unsigned fields) {
if (rank == 0)
clear_particle_node();
on_particle_change();
}
Oops, something went wrong.

0 comments on commit 3d4b341

Please sign in to comment.