Skip to content

Commit

Permalink
Bugs in the ghost communication of domain decomposition fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
Hans-Jörg Limbach committed Jan 5, 2004
1 parent 1a87c21 commit d8f65a4
Show file tree
Hide file tree
Showing 5 changed files with 46 additions and 16 deletions.
15 changes: 10 additions & 5 deletions communication.c
View file
Expand Up @@ -16,6 +16,7 @@
#include "config.h"
#include "communication.h"
#include "interaction_data.h"
#include "particle_data.h"
#include "integrate.h"
#include "cells.h"
#include "global.h"
Expand Down Expand Up @@ -955,9 +956,11 @@ void mpi_get_particles(Particle *result, IntList *bi)
if (sizes[pnode] > 0) {
if (pnode == this_node) {
for (c = 0; c < local_cells.n; c++) {
Particle *part;
int npart;
cell = local_cells.cell[c];
Particle *part = cell->part;
int npart = cell->n;
part = cell->part;
npart = cell->n;
memcpy(&result[g], part, npart*sizeof(Particle));
g += npart;
if (bi) {
Expand Down Expand Up @@ -1045,10 +1048,12 @@ void mpi_get_particles_slave(int pnode, int bi)

g = 0;
for (c = 0; c < local_cells.n; c++) {
Particle *part;
int npart;
cell = local_cells.cell[c];
Particle *part = cell->part;
int npart = cell->n;
memcpy(&result[g],part,npart*sizeof(Particle));
part = cell->part;
npart = cell->n;
memcpy(&result[g],part,npart*sizeof(Particle));
g+=cell->n;
if (bi) {
int pc;
Expand Down
2 changes: 1 addition & 1 deletion debug.h
View file
Expand Up @@ -52,7 +52,7 @@

/* #define COMM_DEBUG */
/* #define INTEG_DEBUG */
/* #define CELL_DEBUG */
/* #define CELL_DEBUG */
/* #define GHOST_DEBUG */
/* #define GRID_DEBUG */
/* #define VERLET_DEBUG */
Expand Down
37 changes: 30 additions & 7 deletions domain_decomposition.c
View file
Expand Up @@ -98,6 +98,7 @@ void dd_create_cell_grid()
int i,n_local_cells,new_cells,try=1;
double cell_range[3], min_box_l;
CELL_TRACE(fprintf(stderr, "%d: dd_create_cell_grid: max_range %f\n",this_node,max_range));
CELL_TRACE(fprintf(stderr, "%d: dd_create_cell_grid: local_box %f-%f, %f-%f, %f-%f,\n",this_node,my_left[0],my_right[0],my_left[1],my_right[1],my_left[2],my_right[2]));

/* initialize */
min_box_l = dmin(dmin(local_box_l[0],local_box_l[1]),local_box_l[2]);
Expand Down Expand Up @@ -410,13 +411,23 @@ int dd_append_particles(ParticleList *pl, int fold_dir)

if (cpos[dir] < 1) {
cpos[dir] = 1;
flag=1;
CELL_TRACE(if(fold_coord==2){fprintf(stderr, "%d: dd_append_particles: particle %d (%f,%f,%f) not inside node domain.\n", this_node,pl->part[p].p.identity,pl->part[p].r.p[0],pl->part[p].r.p[1],pl->part[p].r.p[2]);});
#ifdef PARTIAL_PERIODIC
if( PERIODIC(dir) )
#endif
{
flag=1;
CELL_TRACE(if(fold_coord==2){fprintf(stderr, "%d: dd_append_particles: particle %d (%f,%f,%f) not inside node domain.\n", this_node,pl->part[p].p.identity,pl->part[p].r.p[0],pl->part[p].r.p[1],pl->part[p].r.p[2]);});
}
}
else if (cpos[dir] > dd.cell_grid[dir]) {
cpos[dir] = dd.cell_grid[dir];
flag=1;
CELL_TRACE(if(fold_coord==2){fprintf(stderr, "%d: dd_append_particles: particle %d (%f,%f,%f) not inside node domain.\n", this_node,pl->part[p].p.identity,pl->part[p].r.p[0],pl->part[p].r.p[1],pl->part[p].r.p[2]);});
#ifdef PARTIAL_PERIODIC
if( PERIODIC(dir) )
#endif
{
flag=1;
CELL_TRACE(if(fold_coord==2){fprintf(stderr, "%d: dd_append_particles: particle %d (%f,%f,%f) not inside node domain.\n", this_node,pl->part[p].p.identity,pl->part[p].r.p[0],pl->part[p].r.p[1],pl->part[p].r.p[2]);});
}
}
}
c = get_linear_index(cpos[0],cpos[1],cpos[2], dd.ghost_cell_grid);
Expand Down Expand Up @@ -500,7 +511,7 @@ void dd_topology_release()
/************************************************************/
void dd_exchange_and_sort_particles(int global_flag)
{
int dir, c, p, finished=0,i;
int dir, c, p, finished=0;
ParticleList *cell,*sort_cell, send_buf_l, send_buf_r, recv_buf_l, recv_buf_r;
Particle *part;
CELL_TRACE(fprintf(stderr,"%d: dd_exchange_and_sort_particles(%d):\n",this_node,global_flag));
Expand All @@ -526,6 +537,7 @@ void dd_exchange_and_sort_particles(int global_flag)
if( PERIODIC(dir) || (boundary[2*dir]==0) )
#endif
{
CELL_TRACE(fprintf(stderr,"%d: dd_ex_and_sort_p: send part left %d\n",this_node,part[p].p.identity));
local_particles[part[p].p.identity] = NULL;
move_indexed_particle(&send_buf_l, cell, p);
if(p < cell->n) p--;
Expand All @@ -537,6 +549,7 @@ void dd_exchange_and_sort_particles(int global_flag)
if( PERIODIC(dir) || (boundary[2*dir+1]==0) )
#endif
{
CELL_TRACE(fprintf(stderr,"%d: dd_ex_and_sort_p: send part right %d\n",this_node,part[p].p.identity));
local_particles[part[p].p.identity] = NULL;
move_indexed_particle(&send_buf_r, cell, p);
if(p < cell->n) p--;
Expand Down Expand Up @@ -579,10 +592,16 @@ void dd_exchange_and_sort_particles(int global_flag)
send_particles(&send_buf_r, node_neighbors[2*dir+1]);
}
/* sort received particles to cells */
CELL_TRACE(fprintf(stderr,"%d: dd_exchange_and_sort_particles: exchange done dir=%d\n",this_node,dir));
if(dd_append_particles(&recv_buf_l, 2*dir ) && dir == 2) finished = 0;
if(dd_append_particles(&recv_buf_r, 2*dir+1) && dir == 2) finished = 0;

/* reset send/recv buffers */
send_buf_l.n = 0;
send_buf_r.n = 0;
recv_buf_l.n = 0;
recv_buf_r.n = 0;
MPI_Barrier(MPI_COMM_WORLD);
CELL_TRACE(fprintf(stderr,"%d: dd_exchange_and_sort_particles: exchange done dir=%d\n",this_node,dir));
MPI_Barrier(MPI_COMM_WORLD);
}
else {
/* Single node direction case (no communication) */
Expand Down Expand Up @@ -629,6 +648,10 @@ void dd_exchange_and_sort_particles(int global_flag)
}
}
}
MPI_Barrier(MPI_COMM_WORLD);
CELL_TRACE(fprintf(stderr,"%d: dd_exchange_and_sort_particles: (single node direction) exchange done dir=%d\n",this_node,dir));
MPI_Barrier(MPI_COMM_WORLD);

}
}
/* Communicate wether particle exchange is finished */
Expand Down
2 changes: 1 addition & 1 deletion mmm1d.c
View file
Expand Up @@ -425,8 +425,8 @@ double mmm1d_coulomb_pair_energy(Particle *p1, Particle *p2, double d[3], double
/* far range formula */
double rxy = sqrt(rxy2);
double rxy_d = rxy*L_i;
E = 0;
int bp;
E = 0;

for (bp = 1; bp < mmm1d_params.bessel_cutoff; bp++) {
double fq = C_2PI*bp;
Expand Down
6 changes: 4 additions & 2 deletions particle_data.c
View file
Expand Up @@ -1554,9 +1554,11 @@ void local_remove_all_particles()
n_total_particles = 0;
max_seen_particle = -1;
for (c = 0; c < local_cells.n; c++) {
Particle *p;
int i, np;
cell = local_cells.cell[c];
Particle *p = cell->part;
int i, np = cell->n;
p = cell->part;
np = cell->n;
for (i = 0; i < np; i++)
realloc_intlist(&p[i].bl, 0);
cell->n = 0;
Expand Down

0 comments on commit d8f65a4

Please sign in to comment.