From 9db34bc18e6509ce057930346cccaedbd7bc70be Mon Sep 17 00:00:00 2001 From: Steve Plimpton Date: Mon, 27 Jul 2020 16:15:41 -0600 Subject: [PATCH] cosmetic changes to CommTiled for comm multi style --- src/comm_tiled.cpp | 429 +++++++++++++++++++++++---------------------- src/comm_tiled.h | 6 +- 2 files changed, 228 insertions(+), 207 deletions(-) diff --git a/src/comm_tiled.cpp b/src/comm_tiled.cpp index 88a5c02c6ba..a57437d88e6 100644 --- a/src/comm_tiled.cpp +++ b/src/comm_tiled.cpp @@ -73,9 +73,8 @@ CommTiled::~CommTiled() memory->destroy(overlap); deallocate_swap(nswap); memory->sfree(rcbinfo); - if (mode == Comm::MULTI) { - memory->destroy(cutghostmulti); - } + + if (mode == Comm::MULTI) memory->destroy(cutghostmulti); } /* ---------------------------------------------------------------------- @@ -84,8 +83,6 @@ CommTiled::~CommTiled() void CommTiled::init_buffers() { - sendbox_multi = NULL; - cutghostmulti = NULL; buf_send = buf_recv = NULL; maxsend = maxrecv = BUFMIN; grow_send(maxsend,2); @@ -98,6 +95,9 @@ void CommTiled::init_buffers() allocate_swap(nswap); rcbinfo = NULL; + + cutghostmulti = NULL; + sendbox_multi = NULL; } /* ---------------------------------------------------------------------- */ @@ -106,13 +106,10 @@ void CommTiled::init() { Comm::init(); - // memory for multi-style communication - if (mode == Comm::MULTI) { + if (mode == Comm::SINGLE && cutghostmulti) memory->destroy(cutghostmulti); + + if (mode == Comm::MULTI && !cutghostmulti) memory->create(cutghostmulti,atom->ntypes+1,3,"comm:cutghostmulti"); - } - if (mode == Comm::SINGLE && cutghostmulti) { - memory->destroy(cutghostmulti); - } int bufextra_old = bufextra; init_exchange(); @@ -122,7 +119,6 @@ void CommTiled::init() if (triclinic) error->all(FLERR,"Cannot yet use comm_style tiled with triclinic box"); - } /* ---------------------------------------------------------------------- @@ -133,7 +129,7 @@ void CommTiled::init() void CommTiled::setup() { int i,j,n; - int ntypes = atom->ntypes; + // domain properties used in setup method and methods it calls dimension = domain->dimension; @@ -144,6 +140,7 @@ void CommTiled::setup() subhi = domain->subhi; int *periodicity = domain->periodicity; + int ntypes = atom->ntypes; // set function pointers @@ -185,7 +182,6 @@ void CommTiled::setup() cutghost[0] = cutghost[1] = cutghost[2] = cut; - if ((periodicity[0] && cut > prd[0]) || (periodicity[1] && cut > prd[1]) || (dimension == 3 && periodicity[2] && cut > prd[2])) @@ -318,113 +314,132 @@ void CommTiled::setup() // = obox in other 2 dims // if sbox touches other proc's sub-box boundaries in lower dims, // extend sbox in those lower dims to include ghost atoms - + // single mode and multi mode + double oboxlo[3],oboxhi[3],sbox[6],sbox_multi[6]; + if (mode == Comm::SINGLE) { - for (i = 0; i < noverlap; i++) { - pbc_flag[iswap][i] = 0; - pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = - pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; - - (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); - - if (i < noverlap1) { - sbox[0] = MAX(oboxlo[0],lo1[0]); - sbox[1] = MAX(oboxlo[1],lo1[1]); - sbox[2] = MAX(oboxlo[2],lo1[2]); - sbox[3] = MIN(oboxhi[0],hi1[0]); - sbox[4] = MIN(oboxhi[1],hi1[1]); - sbox[5] = MIN(oboxhi[2],hi1[2]); - } else { - pbc_flag[iswap][i] = 1; - if (idir == 0) pbc[iswap][i][idim] = 1; - else pbc[iswap][i][idim] = -1; - sbox[0] = MAX(oboxlo[0],lo2[0]); - sbox[1] = MAX(oboxlo[1],lo2[1]); - sbox[2] = MAX(oboxlo[2],lo2[2]); - sbox[3] = MIN(oboxhi[0],hi2[0]); - sbox[4] = MIN(oboxhi[1],hi2[1]); - sbox[5] = MIN(oboxhi[2],hi2[2]); - } - - if (idir == 0) { - sbox[idim] = sublo[idim]; - if (i < noverlap1) sbox[3+idim] = MIN(sbox[3+idim]+cut,subhi[idim]); - else sbox[3+idim] = MIN(sbox[3+idim]-prd[idim]+cut,subhi[idim]); - } else { - if (i < noverlap1) sbox[idim] = MAX(sbox[idim]-cut,sublo[idim]); - else sbox[idim] = MAX(sbox[idim]+prd[idim]-cut,sublo[idim]); - sbox[3+idim] = subhi[idim]; - } - - if (idim >= 1) { - if (sbox[0] == oboxlo[0]) sbox[0] -= cut; - if (sbox[3] == oboxhi[0]) sbox[3] += cut; - } - if (idim == 2) { - if (sbox[1] == oboxlo[1]) sbox[1] -= cut; - if (sbox[4] == oboxhi[1]) sbox[4] += cut; - } - - memcpy(sendbox[iswap][i],sbox,6*sizeof(double)); - } - } - else{ + for (i = 0; i < noverlap; i++) { + pbc_flag[iswap][i] = 0; + pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = + pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; + + (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); + + if (i < noverlap1) { + sbox[0] = MAX(oboxlo[0],lo1[0]); + sbox[1] = MAX(oboxlo[1],lo1[1]); + sbox[2] = MAX(oboxlo[2],lo1[2]); + sbox[3] = MIN(oboxhi[0],hi1[0]); + sbox[4] = MIN(oboxhi[1],hi1[1]); + sbox[5] = MIN(oboxhi[2],hi1[2]); + } else { + pbc_flag[iswap][i] = 1; + if (idir == 0) pbc[iswap][i][idim] = 1; + else pbc[iswap][i][idim] = -1; + sbox[0] = MAX(oboxlo[0],lo2[0]); + sbox[1] = MAX(oboxlo[1],lo2[1]); + sbox[2] = MAX(oboxlo[2],lo2[2]); + sbox[3] = MIN(oboxhi[0],hi2[0]); + sbox[4] = MIN(oboxhi[1],hi2[1]); + sbox[5] = MIN(oboxhi[2],hi2[2]); + } + + if (idir == 0) { + sbox[idim] = sublo[idim]; + if (i < noverlap1) sbox[3+idim] = MIN(sbox[3+idim]+cut,subhi[idim]); + else sbox[3+idim] = MIN(sbox[3+idim]-prd[idim]+cut,subhi[idim]); + } else { + if (i < noverlap1) sbox[idim] = MAX(sbox[idim]-cut,sublo[idim]); + else sbox[idim] = MAX(sbox[idim]+prd[idim]-cut,sublo[idim]); + sbox[3+idim] = subhi[idim]; + } + + if (idim >= 1) { + if (sbox[0] == oboxlo[0]) sbox[0] -= cut; + if (sbox[3] == oboxhi[0]) sbox[3] += cut; + } + if (idim == 2) { + if (sbox[1] == oboxlo[1]) sbox[1] -= cut; + if (sbox[4] == oboxhi[1]) sbox[4] += cut; + } + + memcpy(sendbox[iswap][i],sbox,6*sizeof(double)); + } + } + + if (mode == Comm::MULTI) { for (i = 0; i < noverlap; i++) { - pbc_flag[iswap][i] = 0; - pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = - pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; - - (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); - - if (i < noverlap1) { - sbox[0] = MAX(oboxlo[0],lo1[0]); - sbox[1] = MAX(oboxlo[1],lo1[1]); - sbox[2] = MAX(oboxlo[2],lo1[2]); - sbox[3] = MIN(oboxhi[0],hi1[0]); - sbox[4] = MIN(oboxhi[1],hi1[1]); - sbox[5] = MIN(oboxhi[2],hi1[2]); - } else { - pbc_flag[iswap][i] = 1; - if (idir == 0) pbc[iswap][i][idim] = 1; - else pbc[iswap][i][idim] = -1; - sbox[0] = MAX(oboxlo[0],lo2[0]); - sbox[1] = MAX(oboxlo[1],lo2[1]); - sbox[2] = MAX(oboxlo[2],lo2[2]); - sbox[3] = MIN(oboxhi[0],hi2[0]); - sbox[4] = MIN(oboxhi[1],hi2[1]); - sbox[5] = MIN(oboxhi[2],hi2[2]); - } - for (int itype = 1; itype <= atom->ntypes; itype++) { - sbox_multi[0] = sbox[0]; - sbox_multi[1] = sbox[1]; - sbox_multi[2] = sbox[2]; - sbox_multi[3] = sbox[3]; - sbox_multi[4] = sbox[4]; - sbox_multi[5] = sbox[5]; - if (idir == 0) { - sbox_multi[idim] = sublo[idim]; - if (i < noverlap1) sbox_multi[3+idim] = MIN(sbox_multi[3+idim]+cutghostmulti[itype][idim],subhi[idim]); - else sbox_multi[3+idim] = MIN(sbox_multi[3+idim]-prd[idim]+cutghostmulti[itype][idim],subhi[idim]); - } else { - if (i < noverlap1) sbox_multi[idim] = MAX(sbox_multi[idim]-cutghostmulti[itype][idim],sublo[idim]); - else sbox_multi[idim] = MAX(sbox_multi[idim]+prd[idim]-cutghostmulti[itype][idim],sublo[idim]); - sbox_multi[3+idim] = subhi[idim]; - } - - if (idim >= 1) { - if (sbox_multi[0] == oboxlo[0]) sbox_multi[0] -= cutghostmulti[itype][idim]; - if (sbox_multi[3] == oboxhi[0]) sbox_multi[3] += cutghostmulti[itype][idim]; - } - if (idim == 2) { - if (sbox_multi[1] == oboxlo[1]) sbox_multi[1] -= cutghostmulti[itype][idim]; - if (sbox_multi[4] == oboxhi[1]) sbox_multi[4] += cutghostmulti[itype][idim]; - } - - memcpy(sendbox_multi[iswap][i][itype],sbox_multi,6*sizeof(double)); - } - } - } + pbc_flag[iswap][i] = 0; + pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = + pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; + + (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); + + if (i < noverlap1) { + sbox[0] = MAX(oboxlo[0],lo1[0]); + sbox[1] = MAX(oboxlo[1],lo1[1]); + sbox[2] = MAX(oboxlo[2],lo1[2]); + sbox[3] = MIN(oboxhi[0],hi1[0]); + sbox[4] = MIN(oboxhi[1],hi1[1]); + sbox[5] = MIN(oboxhi[2],hi1[2]); + } else { + pbc_flag[iswap][i] = 1; + if (idir == 0) pbc[iswap][i][idim] = 1; + else pbc[iswap][i][idim] = -1; + sbox[0] = MAX(oboxlo[0],lo2[0]); + sbox[1] = MAX(oboxlo[1],lo2[1]); + sbox[2] = MAX(oboxlo[2],lo2[2]); + sbox[3] = MIN(oboxhi[0],hi2[0]); + sbox[4] = MIN(oboxhi[1],hi2[1]); + sbox[5] = MIN(oboxhi[2],hi2[2]); + } + + for (int itype = 1; itype <= atom->ntypes; itype++) { + sbox_multi[0] = sbox[0]; + sbox_multi[1] = sbox[1]; + sbox_multi[2] = sbox[2]; + sbox_multi[3] = sbox[3]; + sbox_multi[4] = sbox[4]; + sbox_multi[5] = sbox[5]; + if (idir == 0) { + sbox_multi[idim] = sublo[idim]; + if (i < noverlap1) + sbox_multi[3+idim] = + MIN(sbox_multi[3+idim]+cutghostmulti[itype][idim],subhi[idim]); + else + sbox_multi[3+idim] = + MIN(sbox_multi[3+idim]-prd[idim]+cutghostmulti[itype][idim], + subhi[idim]); + } else { + if (i < noverlap1) + sbox_multi[idim] = + MAX(sbox_multi[idim]-cutghostmulti[itype][idim],sublo[idim]); + else + sbox_multi[idim] = + MAX(sbox_multi[idim]+prd[idim]-cutghostmulti[itype][idim], + sublo[idim]); + sbox_multi[3+idim] = subhi[idim]; + } + + if (idim >= 1) { + if (sbox_multi[0] == oboxlo[0]) + sbox_multi[0] -= cutghostmulti[itype][idim]; + if (sbox_multi[3] == oboxhi[0]) + sbox_multi[3] += cutghostmulti[itype][idim]; + } + if (idim == 2) { + if (sbox_multi[1] == oboxlo[1]) + sbox_multi[1] -= cutghostmulti[itype][idim]; + if (sbox_multi[4] == oboxhi[1]) + sbox_multi[4] += cutghostmulti[itype][idim]; + } + + memcpy(sendbox_multi[iswap][i][itype],sbox_multi,6*sizeof(double)); + } + } + } + iswap++; } } @@ -703,7 +718,7 @@ void CommTiled::reverse_comm() atoms exchanged with procs that touch sub-box in each of 3 dims send out atoms that have left my box, receive ones entering my box atoms will be lost if not inside a touching proc's box - can happen if atom moves outside of non-periodic bounary + can happen if atom moves outside of non-periodic boundary or if atom moves more than one proc away this routine called before every reneighboring for triclinic, atoms must be in lamda coords (0-1) before exchange is called @@ -864,104 +879,107 @@ void CommTiled::borders() // for x-dim swaps, check owned atoms // for yz-dim swaps, check owned and ghost atoms // store sent atom indices in sendlist for use in future timesteps - // NOTE: assume SINGLE mode, add logic for MULTI mode later - + // single mode and multi mode + x = atom->x; if (iswap % 2 == 0) nlast = atom->nlocal + atom->nghost; ncountall = 0; for (m = 0; m < nsendproc[iswap]; m++) { + if (mode == Comm::SINGLE) { - bbox = sendbox[iswap][m]; - xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; - xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; - - ncount = 0; - - if (!bordergroup) { - for (i = 0; i < nlast; i++) { - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; - } - } - } else { - ngroup = atom->nfirst; - for (i = 0; i < ngroup; i++) { - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + bbox = sendbox[iswap][m]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + + ncount = 0; + + if (!bordergroup) { + for (i = 0; i < nlast; i++) { + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); sendlist[iswap][m][ncount++] = i; - } - } - for (i = atom->nlocal; i < nlast; i++) { - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; - } - } - } + } + } + } else { + ngroup = atom->nfirst; + for (i = 0; i < ngroup; i++) { + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + for (i = atom->nlocal; i < nlast; i++) { + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + } + + sendnum[iswap][m] = ncount; + smaxone = MAX(smaxone,ncount); + ncountall += ncount; - sendnum[iswap][m] = ncount; - smaxone = MAX(smaxone,ncount); - ncountall += ncount; - } - else{ - int* type=atom->type; - int itype; - ncount = 0; - - if (!bordergroup) { - for (i = 0; i < nlast; i++) { - itype=type[i]; - bbox = sendbox_multi[iswap][m][itype]; - xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; - xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; - } - } } else { - ngroup = atom->nfirst; - for (i = 0; i < ngroup; i++) { - itype=type[i]; - bbox = sendbox_multi[iswap][m][itype]; - xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; - xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; - } - } - for (i = atom->nlocal; i < nlast; i++) { - itype=type[i]; - bbox = sendbox_multi[iswap][m][itype]; - xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; - xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + + int* type=atom->type; + int itype; + ncount = 0; + + if (!bordergroup) { + for (i = 0; i < nlast; i++) { + itype=type[i]; + bbox = sendbox_multi[iswap][m][itype]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); sendlist[iswap][m][ncount++] = i; - } - } + } + } + } else { + ngroup = atom->nfirst; + for (i = 0; i < ngroup; i++) { + itype=type[i]; + bbox = sendbox_multi[iswap][m][itype]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + for (i = atom->nlocal; i < nlast; i++) { + itype=type[i]; + bbox = sendbox_multi[iswap][m][itype]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + } + + sendnum[iswap][m] = ncount; + smaxone = MAX(smaxone,ncount); + ncountall += ncount; } - - sendnum[iswap][m] = ncount; - smaxone = MAX(smaxone,ncount); - ncountall += ncount; - } } + smaxall = MAX(smaxall,ncountall); // send sendnum counts to procs who recv from me except self @@ -2136,6 +2154,7 @@ void CommTiled::deallocate_swap(int n) memory->destroy(pbc[i]); memory->destroy(sendbox[i]); memory->destroy(sendbox_multi[i]); + delete [] maxsendlist[i]; for (int j = 0; j < nprocmax[i]; j++) memory->destroy(sendlist[i][j]); diff --git a/src/comm_tiled.h b/src/comm_tiled.h index 018c5b36968..416e2a5317f 100644 --- a/src/comm_tiled.h +++ b/src/comm_tiled.h @@ -69,14 +69,16 @@ class CommTiled : public Comm { int **size_reverse_recv; // # of values to recv in each reverse swap/proc int **forward_recv_offset; // forward comm offsets in buf_recv per swap/proc int **reverse_recv_offset; // reverse comm offsets in buf_recv per swap/proc - double **cutghostmulti; // cutghost on a per-type basis int ***sendlist; // list of atoms to send per swap/proc int **maxsendlist; // max size of send list per swap/proc int **pbc_flag; // general flag for sending atoms thru PBC int ***pbc; // dimension flags for PBC adjustments double ***sendbox; // bounding box of atoms to send per swap/proc - double ****sendbox_multi; // bounding box of atoms to send per swap/proc for multi comm + + double **cutghostmulti; // cutghost on a per-type basis + double ****sendbox_multi; // bounding box of atoms to send + // per swap/proc for multi comm // exchange comm info, proc lists do not include self