Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update spray test to use uniform spray initialize function, time was … #141

Merged
merged 2 commits into from
Nov 18, 2022
Merged

Update spray test to use uniform spray initialize function, time was … #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0988036
Update spray test to use uniform spray initialize function, time was …
Nov 18, 2022
b81a06a
Update submodule with bug fix
Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 19 additions & 200 deletions Exec/RegTests/SprayTest/SprayParticlesInitInsert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,7 @@

#include "SprayParticles.H"
#include "SprayInjection.H"
#include "pelelm_prob.H"

amrex::IntVect
unflatten_particles(const amrex::ULong idx, const amrex::IntVect& max_parts)
{
amrex::IntVect indx;
amrex::ULong cidx = idx;
amrex::ULong d1 = max_parts[0];
#if AMREX_SPACEDIM > 2
amrex::ULong d2 = max_parts[1];
indx[2] = int(cidx / (d1 * d2));
cidx -= amrex::ULong(indx[2]) * d1 * d2;
#endif
indx[1] = int(cidx / d1);
indx[0] = int(cidx % d1);
return indx;
}

bool
SprayParticleContainer::injectParticles(
amrex::Real /*time*/,
Expand All @@ -38,189 +22,24 @@ SprayParticleContainer::InitSprayParticles(
if (!init_parts) {
return;
}
const int lev = 0;
const int MyProc = amrex::ParallelDescriptor::MyProc();
const int NProcs = amrex::ParallelDescriptor::NProcs();
int NRedist = prob_parm.numRedist; // Number of times to redistribute
// TODO: This might be overkill but issues persisted at high Summit node
// counts
if (NRedist < 0) {
NRedist = 1;
if (NProcs <= 1024) {
NRedist = 2;
} else if (NProcs <= 2048) {
NRedist = 4;
} else if (NProcs <= 4096) {
NRedist = 8;
}
}
amrex::Real part_dia = prob_parm.partDia;
amrex::Real T_ref = prob_parm.partTemp;
const int pstateVel = m_sprayIndx.pstateVel;
const int pstateDia = m_sprayIndx.pstateDia;
const int pstateT = m_sprayIndx.pstateT;
const int pstateY = m_sprayIndx.pstateY;
const amrex::IntVect num_part = prob_parm.partNum;
const amrex::RealVect part_vel = prob_parm.partVel;
// Reference values for the particles
amrex::Real part_vals[NAR_SPR + NSR_SPR];
const int level = 0;
amrex::ParmParse pp("prob");
int numRedist = -1;
amrex::IntVect partNum(AMREX_D_DECL(100, 100, 100));
// Find the number of redistributions during particle initialization
pp.query("init_redist", numRedist);
pp.query("num_particles", partNum);
amrex::RealVect partVel;
std::array<amrex::Real, AMREX_SPACEDIM> pvel = {0.0};
pp.query<amrex::Real>("part_vel", pvel);
for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
part_vals[pstateVel + dir] = part_vel[dir];
}
part_vals[pstateT] = T_ref;
part_vals[pstateDia] = part_dia;
for (int sp = 0; sp < SPRAY_FUEL_NUM; ++sp) {
part_vals[pstateY + sp] = 0.;
partVel[dir] = pvel[dir];
}
part_vals[pstateY] = 1.; // Only use the first fuel species
const amrex::RealVect dx_part(AMREX_D_DECL(
Geom(lev).ProbLength(0) / amrex::Real(num_part[0]),
Geom(lev).ProbLength(1) / amrex::Real(num_part[1]),
Geom(lev).ProbLength(2) / amrex::Real(num_part[2])));
AMREX_D_TERM(amrex::ULong np0 = num_part[0];, amrex::ULong np1 = num_part[1];
, amrex::ULong np2 = num_part[2];);
const amrex::ULong total_part_num = AMREX_D_TERM(np0, *np1, *np2);
amrex::ULong parts_pp = total_part_num / amrex::ULong(NProcs);
// Number of particles per processor to be initialized
amrex::ULong cur_parts_pp = parts_pp;
// Give any remaining particles to the last processor
if (MyProc == NProcs - 1) {
cur_parts_pp += (total_part_num % amrex::ULong(NProcs));
}
// Starting particle for this processor
const amrex::ULong first_part = amrex::ULong(MyProc) * parts_pp;
amrex::Gpu::HostVector<ParticleType> nparticles;
amrex::Vector<amrex::Gpu::HostVector<amrex::Real>> nreals;
if (NAR_SPR > 0) {
nreals.resize(NAR_SPR);
}
for (amrex::ULong prc = 0; prc < cur_parts_pp; ++prc) {
amrex::ULong cur_part = first_part + prc;
amrex::IntVect indx = unflatten_particles(cur_part, num_part);
ParticleType p;
p.id() = ParticleType::NextID();
p.cpu() = amrex::ParallelDescriptor::MyProc();
for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
p.pos(dir) = (amrex::Real(indx[dir]) + 0.5) * dx_part[dir];
}
for (int n = 0; n < NSR_SPR; ++n) {
p.rdata(n) = part_vals[n];
}
for (int n = 0; n < NAR_SPR; ++n) {
nreals[n].push_back(part_vals[n]);
}
nparticles.push_back(p);
}
amrex::ParticleLocData pld;
// Only copy particle data for certain processors at a time
int NRchunk = NProcs / NRedist;
for (int nr = 0; nr < NRedist; ++nr) {
std::map<std::pair<int, int>, amrex::Gpu::HostVector<ParticleType>>
host_particles;
std::map<
std::pair<int, int>,
std::array<amrex::Gpu::HostVector<amrex::Real>, NAR_SPR>>
host_real_attribs;
if (m_verbose > 0) {
amrex::Print() << "Redistributing from processor " << nr * NRchunk
<< " to " << (nr + 1) * NRchunk - 1 << '\n';
}
for (int which = nr * NRchunk; which < (nr + 1) * NRchunk; ++which) {
if (which == MyProc) {
while (!nparticles.empty()) {
// Retrieve the last particle entry and add it to host_particles
ParticleType& p = nparticles.back();
bool where = Where(p, pld);
if (!where) {
amrex::Abort("Bad particle");
}
std::pair<int, int> ind(pld.m_grid, pld.m_tile);
host_particles[ind].push_back(p);
for (int n = 0; n < NAR_SPR; ++n) {
host_real_attribs[ind][n].push_back(nreals[n].back());
}
// Remove the particle just read
nparticles.pop_back();
for (int n = 0; n < NAR_SPR; ++n) {
nreals[n].pop_back();
}
}
} // if (which == MyProc)
} // for (int which ...
for (auto& kv : host_particles) {
auto grid = kv.first.first;
auto tile = kv.first.second;
const auto& src_tile = kv.second;
auto& dst_tile = GetParticles(lev)[std::make_pair(grid, tile)];
auto old_size = dst_tile.GetArrayOfStructs().size();
auto new_size = old_size + src_tile.size();
dst_tile.resize(new_size);

// Copy the AoS part of the host particles to the GPU
amrex::Gpu::copy(
amrex::Gpu::hostToDevice, src_tile.begin(), src_tile.end(),
dst_tile.GetArrayOfStructs().begin() + old_size);
for (int i = 0; i < NAR_SPR; ++i) {
amrex::Gpu::copy(
amrex::Gpu::hostToDevice,
host_real_attribs[std::make_pair(grid, tile)][i].begin(),
host_real_attribs[std::make_pair(grid, tile)][i].end(),
dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size);
}
}
Redistribute();
} // for (int nr ...
// Now copy over any remaining processors
for (int which = NRedist * NRchunk; which < NProcs; ++which) {
std::map<std::pair<int, int>, amrex::Gpu::HostVector<ParticleType>>
host_particles;
std::map<
std::pair<int, int>,
std::array<amrex::Gpu::HostVector<amrex::Real>, NAR_SPR>>
host_real_attribs;
if (m_verbose > 0) {
amrex::Print() << "Redistributing from processor " << NRedist * NRchunk
<< " to " << NProcs << '\n';
}
if (which == MyProc) {
while (!nparticles.empty()) {
// Retrieve the last particle entry and add it to host_particles
ParticleType& p = nparticles.back();
Where(p, pld);
std::pair<int, int> ind(pld.m_grid, pld.m_tile);
host_particles[ind].push_back(p);
for (int n = 0; n < NAR_SPR; ++n) {
host_real_attribs[ind][n].push_back(nreals[n].back());
}
// Remove the particle just read
nparticles.pop_back();
for (int n = 0; n < NAR_SPR; ++n) {
nreals[n].pop_back();
}
}
} // if (which == MyProc)
for (auto& kv : host_particles) {
auto grid = kv.first.first;
auto tile = kv.first.second;
const auto& src_tile = kv.second;
auto& dst_tile = GetParticles(lev)[std::make_pair(grid, tile)];
auto old_size = dst_tile.GetArrayOfStructs().size();
auto new_size = old_size + src_tile.size();
dst_tile.resize(new_size);

// Copy the AoS part of the host particles to the GPU
amrex::Gpu::copy(
amrex::Gpu::hostToDevice, src_tile.begin(), src_tile.end(),
dst_tile.GetArrayOfStructs().begin() + old_size);
for (int i = 0; i < NAR_SPR; ++i) {
amrex::Gpu::copy(
amrex::Gpu::hostToDevice,
host_real_attribs[std::make_pair(grid, tile)][i].begin(),
host_real_attribs[std::make_pair(grid, tile)][i].end(),
dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size);
}
}
Redistribute();
} // for (int which ...
amrex::Gpu::streamSynchronize();
amrex::Real partDia, partTemp;
pp.get("part_dia", partDia);
pp.get("part_temp", partTemp);
std::array<amrex::Real, SPRAY_FUEL_NUM> partY = {0.0};
partY[0] = 1.;
uniformSprayInit(
partNum, partVel, partDia, partTemp, partY.begin(), level, numRedist);
}
10 changes: 0 additions & 10 deletions Exec/RegTests/SprayTest/pelelm_prob.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,4 @@ void PeleLM::readProbParm()
pp.query("init_vel", PeleLM::prob_parm->vel);
pp.query("init_N2", PeleLM::prob_parm->Y_N2);
pp.query("init_O2", PeleLM::prob_parm->Y_O2);
// Find the number of redistributions during particle initialization
pp.query("init_redist", PeleLM::prob_parm->numRedist);
pp.query("num_particles", PeleLM::prob_parm->partNum);
std::array<amrex::Real, AMREX_SPACEDIM> pvel;
pp.query<amrex::Real>("part_vel", pvel);
for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
PeleLM::prob_parm->partVel[dir] = pvel[dir];
}
pp.get("part_dia", PeleLM::prob_parm->partDia);
pp.get("part_temp", PeleLM::prob_parm->partTemp);
}
5 changes: 0 additions & 5 deletions Exec/RegTests/SprayTest/pelelm_prob_parm.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,5 @@ struct ProbParm
amrex::Real Y_O2 = 0.233;
amrex::Real Y_N2 = 0.767;
amrex::Real vel = 0.;
amrex::IntVect partNum = amrex::IntVect(AMREX_D_DECL(100, 100, 100));
amrex::Real partTemp = 300.;
amrex::Real partDia = 1.E-3;
amrex::RealVect partVel = amrex::RealVect(AMREX_D_DECL(0., 0., 0.));
int numRedist = -1;
};
#endif
2 changes: 1 addition & 1 deletion Submodules/PeleMP
Open in desktop
Submodule PeleMP updated 31 files
+13 −18 Docs/sphinx/Equations.rst
+20 −201 Exec/SprayTests/PeleC/HPC_spray_test/SprayParticlesInitInsert.cpp
+0 −47 Exec/SprayTests/PeleC/HPC_spray_test/prob.cpp
+0 −5 Exec/SprayTests/PeleC/HPC_spray_test/prob_parm.H
+12 −197 Exec/SprayTests/PeleC/TG_GPU/SprayParticlesInitInsert.cpp
+0 −6 Exec/SprayTests/PeleC/TG_GPU/prob.cpp
+0 −1 Exec/SprayTests/PeleC/TG_GPU/prob_parm.H
+12 −197 Exec/SprayTests/PeleC/gammaTG/SprayParticlesInitInsert.cpp
+0 −6 Exec/SprayTests/PeleC/gammaTG/prob.cpp
+0 −1 Exec/SprayTests/PeleC/gammaTG/prob_parm.H
+3 −3 Exec/SprayTests/PeleLM/Multijet/SprayParticlesInitInsert.cpp
+1 −1 Exec/SprayTests/PeleLM/Multijet/pelelm_prob.cpp
+19 −195 Exec/SprayTests/PeleLM/dist_test/SprayParticlesInitInsert.cpp
+1 −1 Exec/SprayTests/PeleLM/dist_test/open-input
+0 −8 Exec/SprayTests/PeleLM/dist_test/pelelm_prob.cpp
+0 −4 Exec/SprayTests/PeleLM/dist_test/pelelm_prob_parm.H
+13 −202 Exec/SprayTests/PeleLMeX/TG/SprayParticlesInitInsert.cpp
+0 −7 Exec/SprayTests/PeleLMeX/TG/pelelm_prob.cpp
+0 −1 Exec/SprayTests/PeleLMeX/TG/pelelm_prob_parm.H
+19 −200 Exec/SprayTests/PeleLMeX/dist_test/SprayParticlesInitInsert.cpp
+6 −17 Exec/SprayTests/PeleLMeX/dist_test/first-input
+0 −10 Exec/SprayTests/PeleLMeX/dist_test/pelelm_prob.cpp
+0 −5 Exec/SprayTests/PeleLMeX/dist_test/pelelm_prob_parm.H
+8 −8 README.rst
+30 −39 Source/PP_Spray/Drag.H
+1 −21 Source/PP_Spray/SprayFuelData.H
+211 −0 Source/PP_Spray/SprayInjection.H
+1 −2 Source/PP_Spray/SprayInterpolation.H
+3 −3 Source/PP_Spray/SprayJet.H
+2 −2 Source/PP_Spray/SprayJet.cpp
+13 −2 Source/PP_Spray/SprayParticles.H