From 5e84f43241edfec7754d3ebfc369154bf249d992 Mon Sep 17 00:00:00 2001 From: asalmgren Date: Sun, 25 Sep 2022 09:38:51 -0700 Subject: [PATCH] make tagging routines EB_aware (#2962) --- Src/AmrCore/AMReX_ErrorList.cpp | 220 ++++++++++++++++++++++++------ Src/Base/AMReX_VisMF.H | 2 - Src/Particle/AMReX_ParticleInit.H | 4 - 3 files changed, 180 insertions(+), 46 deletions(-) diff --git a/Src/AmrCore/AMReX_ErrorList.cpp b/Src/AmrCore/AMReX_ErrorList.cpp index 7f37324123a..6dcb5565227 100644 --- a/Src/AmrCore/AMReX_ErrorList.cpp +++ b/Src/AmrCore/AMReX_ErrorList.cpp @@ -300,78 +300,218 @@ AMRErrorTag::operator() (TagBoxArray& tba, if (m_test == GRAD) { - ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept +#ifdef AMREX_USE_EB + if (mf->hasEBFabFactory()) { + auto const& ebfact = + dynamic_cast(mf->Factory()); + auto const& flags = ebfact.getMultiEBCellFlagFab().arrays(); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + auto const& dat = datma[bi]; + auto const& flag = flags[bi]; + + Real ax = 0.; Real ay = 0.; + if (flag(i,j,k).isConnected(1,0,0)) { + ax = amrex::max(ax,amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k))); + } + if (flag(i,j,k).isConnected(-1,0,0)) { + ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k))); + } + if (flag(i,j,k).isConnected(0,1,0)) { + ay = amrex::max(ay,amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k))); + } + if (flag(i,j,k).isConnected(0,-1,0)) { + ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k))); + } +#if AMREX_SPACEDIM > 2 + Real az = 0.; + if (flag(i,j,k).isConnected(0,0,1)) { + az = amrex::max(az,amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k))); + } + if (flag(i,j,k).isConnected(0,0,-1)) { + az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1))); + } +#endif + if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) { + tagma[bi](i,j,k) = tag_update; + } + }); + } else +#endif { - auto const& dat = datma[bi]; - auto ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)); - ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k))); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + auto const& dat = datma[bi]; + + Real ax = 0.; + ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)); + ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k))); #if AMREX_SPACEDIM == 1 - if (ax >= threshold) { tagma[bi](i,j,k) = tag_update;} + if (ax >= threshold) { tagma[bi](i,j,k) = tag_update;} #else - auto ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)); - ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k))); + Real ay = 0.; + ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)); + ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k))); #if AMREX_SPACEDIM > 2 - auto az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)); - az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1))); -#endif - if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) { - tagma[bi](i,j,k) = tag_update; - } -#endif - }); + Real az = 0.; + az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)); + az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1))); +#endif // DIM > 2 + if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) { + tagma[bi](i,j,k) = tag_update; + } +#endif // DIM > 1 + }); + } } else if (m_test == RELGRAD) { - ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept +#ifdef AMREX_USE_EB + if (mf->hasEBFabFactory()) { + auto const& ebfact = + dynamic_cast(mf->Factory()); + auto const& flags = ebfact.getMultiEBCellFlagFab().arrays(); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + auto const& dat = datma[bi]; + auto const& flag = flags[bi]; + + Real ax = 0.; Real ay = 0.; + + if (flag(i,j,k).isConnected(1,0,0)) { + ax = amrex::max(ax,amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k))); + } + if (flag(i,j,k).isConnected(-1,0,0)) { + ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k))); + } + if (flag(i,j,k).isConnected(0,1,0)) { + ay = amrex::max(ay,amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k))); + } + if (flag(i,j,k).isConnected(0,-1,0)) { + ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k))); + } +#if AMREX_SPACEDIM > 2 + Real az = 0.; + if (flag(i,j,k).isConnected(0,0,1)) { + az = amrex::max(az,amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k))); + } + if (flag(i,j,k).isConnected(0,0,-1)) { + az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1))); + } +#endif // DIM > 2 + if (amrex::max(AMREX_D_DECL(ax,ay,az)) + >= threshold * amrex::Math::abs(dat(i,j,k))) { + tagma[bi](i,j,k) = tag_update; + } + }); + } else +#endif { - auto const& dat = datma[bi]; - auto ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)); - ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k))); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + auto const& dat = datma[bi]; + + Real ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)); + ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k))); #if AMREX_SPACEDIM == 1 - if (ax >= threshold * amrex::Math::abs(dat(i,j,k))) { tagma[bi](i,j,k) = tag_update;} + if (ax >= threshold * amrex::Math::abs(dat(i,j,k))) { tagma[bi](i,j,k) = tag_update;} #else - auto ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)); - ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k))); + Real ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)); + ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k))); #if AMREX_SPACEDIM > 2 - auto az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)); - az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1))); -#endif - if (amrex::max(AMREX_D_DECL(ax,ay,az)) - >= threshold * amrex::Math::abs(dat(i,j,k))) { - tagma[bi](i,j,k) = tag_update; - } -#endif - }); + Real az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)); + az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1))); +#endif // DIM > 2 + if (amrex::max(AMREX_D_DECL(ax,ay,az)) + >= threshold * amrex::Math::abs(dat(i,j,k))) { + tagma[bi](i,j,k) = tag_update; + } +#endif // DIM > 1 + }); + } } else if (m_test == LESS) { - ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept +#ifdef AMREX_USE_EB + if (mf->hasEBFabFactory()) { + auto const& ebfact = + dynamic_cast(mf->Factory()); + auto const& flags = ebfact.getMultiEBCellFlagFab().arrays(); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt; + auto const& flag = flags[bi]; + if (!flag(i,j,k).isCovered()) { + if (datma[bi](i,j,k) * vol <= threshold) { + tagma[bi](i,j,k) = tag_update; + } + } + }); + } else +#endif { + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept { Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt; if (datma[bi](i,j,k) * vol <= threshold) { tagma[bi](i,j,k) = tag_update; } }); + } } else if (m_test == GREATER) { +#ifdef AMREX_USE_EB + if (mf->hasEBFabFactory()) { + auto const& ebfact = + dynamic_cast(mf->Factory()); + auto const& flags = ebfact.getMultiEBCellFlagFab().arrays(); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt; + auto const& flag = flags[bi]; + if (!flag(i,j,k).isCovered()) { + if (datma[bi](i,j,k) * vol >= threshold) { + tagma[bi](i,j,k) = tag_update; + } + } + }); + } else +#endif ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept { Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt; - if (datma[bi](i,j,k) * vol >= threshold) { - tagma[bi](i,j,k) = tag_update; - } + if (datma[bi](i,j,k) * vol >= threshold) { + tagma[bi](i,j,k) = tag_update; + } }); } else if (m_test == VORT) { const Real fac = threshold * Real(std::pow(2,level)); - ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept +#ifdef AMREX_USE_EB + if (mf->hasEBFabFactory()) { + auto const& ebfact = + dynamic_cast(mf->Factory()); + auto const& flags = ebfact.getMultiEBCellFlagFab().arrays(); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + auto const& flag = flags[bi]; + if (!flag(i,j,k).isCovered()) { + if (datma[bi](i,j,k) >= fac) { + tagma[bi](i,j,k) = tag_update; + } + } + }); + } else +#endif { - if (datma[bi](i,j,k) >= fac) { - tagma[bi](i,j,k) = tag_update; - } - }); + ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept + { + if (datma[bi](i,j,k) >= fac) { + tagma[bi](i,j,k) = tag_update; + } + }); + } } else { diff --git a/Src/Base/AMReX_VisMF.H b/Src/Base/AMReX_VisMF.H index 12777a08307..bfab54abf8d 100644 --- a/Src/Base/AMReX_VisMF.H +++ b/Src/Base/AMReX_VisMF.H @@ -638,7 +638,6 @@ Read (FabArray& fa, const std::string& name) } int totalioreqs = nboxes; - int messtotal = 0; int reqspending = 0; int iopfileindex; std::deque iopreads; @@ -669,7 +668,6 @@ Read (FabArray& fa, const std::string& name) } } else { ParallelDescriptor::Send(vreads, tryproc, readtag); - ++messtotal; ++reqspending; } availablefiles.erase(afilesiter); diff --git a/Src/Particle/AMReX_ParticleInit.H b/Src/Particle/AMReX_ParticleInit.H index ee8afc778e2..03320383ada 100644 --- a/Src/Particle/AMReX_ParticleInit.H +++ b/Src/Particle/AMReX_ParticleInit.H @@ -1022,8 +1022,6 @@ InitRandom (Long icount, ParticleLocData pld; - int cnt = 0; - Vector, Gpu::HostVector > > host_particles; host_particles.reserve(15); host_particles.resize(finestLevel()+1); @@ -1079,8 +1077,6 @@ InitRandom (Long icount, for (int i = 0; i < NArrayInt; i++) { host_int_attribs[pld.m_lev][ind][i].push_back(pdata.int_array_data[i]); } - - cnt++; } }