Skip to content

Commit

Permalink
make tagging routines EB_aware (AMReX-Codes#2962)
Browse files Browse the repository at this point in the history
  • Loading branch information
@asalmgren
asalmgren committed Sep 25, 2022
1 parent 8b367b0 commit 5e84f43
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 46 deletions.
220 changes: 180 additions & 40 deletions Src/AmrCore/AMReX_ErrorList.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,78 +300,218 @@ AMRErrorTag::operator() (TagBoxArray& tba,

if (m_test == GRAD)
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];
auto const& flag = flags[bi];

Real ax = 0.; Real ay = 0.;
if (flag(i,j,k).isConnected(1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(-1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
}
if (flag(i,j,k).isConnected(0,1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,-1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
}
#if AMREX_SPACEDIM > 2
Real az = 0.;
if (flag(i,j,k).isConnected(0,0,1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,0,-1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
}
#endif
if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
});
} else
#endif
{
auto const& dat = datma[bi];
auto ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];

Real ax = 0.;
ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
#if AMREX_SPACEDIM == 1
if (ax >= threshold) { tagma[bi](i,j,k) = tag_update;}
if (ax >= threshold) { tagma[bi](i,j,k) = tag_update;}
#else
auto ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
Real ay = 0.;
ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
#if AMREX_SPACEDIM > 2
auto az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif
if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
#endif
});
Real az = 0.;
az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif // DIM > 2
if (amrex::max(AMREX_D_DECL(ax,ay,az)) >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
#endif // DIM > 1
});
}
}
else if (m_test == RELGRAD)
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];
auto const& flag = flags[bi];

Real ax = 0.; Real ay = 0.;

if (flag(i,j,k).isConnected(1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(-1,0,0)) {
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
}
if (flag(i,j,k).isConnected(0,1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,-1,0)) {
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
}
#if AMREX_SPACEDIM > 2
Real az = 0.;
if (flag(i,j,k).isConnected(0,0,1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k)));
}
if (flag(i,j,k).isConnected(0,0,-1)) {
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
}
#endif // DIM > 2
if (amrex::max(AMREX_D_DECL(ax,ay,az))
>= threshold * amrex::Math::abs(dat(i,j,k))) {
tagma[bi](i,j,k) = tag_update;
}
});
} else
#endif
{
auto const& dat = datma[bi];
auto ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& dat = datma[bi];

Real ax = amrex::Math::abs(dat(i+1,j,k) - dat(i,j,k));
ax = amrex::max(ax,amrex::Math::abs(dat(i,j,k) - dat(i-1,j,k)));
#if AMREX_SPACEDIM == 1
if (ax >= threshold * amrex::Math::abs(dat(i,j,k))) { tagma[bi](i,j,k) = tag_update;}
if (ax >= threshold * amrex::Math::abs(dat(i,j,k))) { tagma[bi](i,j,k) = tag_update;}
#else
auto ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
Real ay = amrex::Math::abs(dat(i,j+1,k) - dat(i,j,k));
ay = amrex::max(ay,amrex::Math::abs(dat(i,j,k) - dat(i,j-1,k)));
#if AMREX_SPACEDIM > 2
auto az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif
if (amrex::max(AMREX_D_DECL(ax,ay,az))
>= threshold * amrex::Math::abs(dat(i,j,k))) {
tagma[bi](i,j,k) = tag_update;
}
#endif
});
Real az = amrex::Math::abs(dat(i,j,k+1) - dat(i,j,k));
az = amrex::max(az,amrex::Math::abs(dat(i,j,k) - dat(i,j,k-1)));
#endif // DIM > 2
if (amrex::max(AMREX_D_DECL(ax,ay,az))
>= threshold * amrex::Math::abs(dat(i,j,k))) {
tagma[bi](i,j,k) = tag_update;
}
#endif // DIM > 1
});
}
}
else if (m_test == LESS)
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
auto const& flag = flags[bi];
if (!flag(i,j,k).isCovered()) {
if (datma[bi](i,j,k) * vol <= threshold) {
tagma[bi](i,j,k) = tag_update;
}
}
});
} else
#endif
{
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept {
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
if (datma[bi](i,j,k) * vol <= threshold) {
tagma[bi](i,j,k) = tag_update;
}
});
}
}
else if (m_test == GREATER)
{
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
auto const& flag = flags[bi];
if (!flag(i,j,k).isCovered()) {
if (datma[bi](i,j,k) * vol >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
}
});
} else
#endif
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
Real vol = volume_weighting ? Geometry::Volume(IntVect{AMREX_D_DECL(i,j,k)}, geomdata) : 1.0_rt;
if (datma[bi](i,j,k) * vol >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
if (datma[bi](i,j,k) * vol >= threshold) {
tagma[bi](i,j,k) = tag_update;
}
});
}
else if (m_test == VORT)
{
const Real fac = threshold * Real(std::pow(2,level));
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
#ifdef AMREX_USE_EB
if (mf->hasEBFabFactory()) {
auto const& ebfact =
dynamic_cast<amrex::EBFArrayBoxFactory const&>(mf->Factory());
auto const& flags = ebfact.getMultiEBCellFlagFab().arrays();
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
auto const& flag = flags[bi];
if (!flag(i,j,k).isCovered()) {
if (datma[bi](i,j,k) >= fac) {
tagma[bi](i,j,k) = tag_update;
}
}
});
} else
#endif
{
if (datma[bi](i,j,k) >= fac) {
tagma[bi](i,j,k) = tag_update;
}
});
ParallelFor(tba, [=] AMREX_GPU_DEVICE (int bi, int i, int j, int k) noexcept
{
if (datma[bi](i,j,k) >= fac) {
tagma[bi](i,j,k) = tag_update;
}
});
}
}
else
{
Expand Down
2 changes: 0 additions & 2 deletions Src/Base/AMReX_VisMF.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,6 @@ Read (FabArray<FAB>& fa, const std::string& name)
}

int totalioreqs = nboxes;
int messtotal = 0;
int reqspending = 0;
int iopfileindex;
std::deque<int> iopreads;
Expand Down Expand Up @@ -669,7 +668,6 @@ Read (FabArray<FAB>& fa, const std::string& name)
}
} else {
ParallelDescriptor::Send(vreads, tryproc, readtag);
++messtotal;
++reqspending;
}
availablefiles.erase(afilesiter);
Expand Down
4 changes: 0 additions & 4 deletions Src/Particle/AMReX_ParticleInit.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1022,8 +1022,6 @@ InitRandom (Long icount,

ParticleLocData pld;

int cnt = 0;

Vector<std::map<std::pair<int, int>, Gpu::HostVector<ParticleType> > > host_particles;
host_particles.reserve(15);
host_particles.resize(finestLevel()+1);
Expand Down Expand Up @@ -1079,8 +1077,6 @@ InitRandom (Long icount,
for (int i = 0; i < NArrayInt; i++) {
host_int_attribs[pld.m_lev][ind][i].push_back(pdata.int_array_data[i]);
}

cnt++;
}
}

Expand Down

0 comments on commit 5e84f43

Please sign in to comment.