diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index abffbefc860..3e6a6113fc8 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -1346,7 +1346,8 @@ protected: * \param lev_max * \param local_grid */ - bool EnforcePeriodicWhere (ParticleType& prt, ParticleLocData& pld, + template + bool EnforcePeriodicWhere (P& prt, ParticleLocData& pld, int lev_min = 0, int lev_max = -1, int local_grid=-1) const; public: @@ -1378,7 +1379,8 @@ private: void RedistributeMPI (std::map >& not_ours, int lev_min = 0, int lev_max = 0, int nGrow = 0, int local=0); - void locateParticle (ParticleType& p, ParticleLocData& pld, + template + void locateParticle (P& p, ParticleLocData& pld, int lev_min, int lev_max, int nGrow, int local_grid=-1) const; void Initialize (); diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 9a5007f86ec..034bc50cd6f 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -164,9 +164,10 @@ ParticleContainer_impl template class Allocator> +template bool ParticleContainer_impl -::EnforcePeriodicWhere (ParticleType& p, +::EnforcePeriodicWhere (P& p, ParticleLocData& pld, int lev_min, int lev_max, @@ -312,8 +313,9 @@ ParticleContainer_impl::resizeDa template class Allocator> +template void -ParticleContainer_impl::locateParticle (ParticleType& p, ParticleLocData& pld, +ParticleContainer_impl::locateParticle (P& p, ParticleLocData& pld, int lev_min, int lev_max, int nGrow, int local_grid) const { bool success; diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index a9849ca8a3c..cfc55dfd550 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -919,7 +919,7 @@ ParticleContainer_impl int* iptr = istuff.dataPtr(); RTYPE* rptr = rstuff.dataPtr(); - ParticleType p; + Particle ptemp; ParticleLocData pld; Vector, Gpu::HostVector > > host_particles; @@ -937,41 +937,43 @@ ParticleContainer_impl host_int_attribs.resize(finest_level_in_file+1); for (int i = 0; i < cnt; i++) { - if (convert_ids) { + // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct + // for backwards compatibility with readers + if (!ParticleType::is_soa_particle && convert_ids) { std::int32_t xi, yi; std::uint32_t xu, yu; xi = iptr[0]; yi = iptr[1]; std::memcpy(&xu, &xi, sizeof(xi)); std::memcpy(&yu, &yi, sizeof(yi)); - p.m_idcpu = ((std::uint64_t)xu) << 32 | yu; + ptemp.m_idcpu = ((std::uint64_t)xu) << 32 | yu; } else { - p.id() = iptr[0]; - p.cpu() = iptr[1]; + ptemp.id() = iptr[0]; + ptemp.cpu() = iptr[1]; } iptr += 2; for (int j = 0; j < NStructInt; j++) { - p.idata(j) = *iptr; + ptemp.idata(j) = *iptr; ++iptr; } - AMREX_ASSERT(p.id() > 0); + AMREX_ASSERT(ptemp.id() > 0); - AMREX_D_TERM(p.pos(0) = ParticleReal(rptr[0]);, - p.pos(1) = ParticleReal(rptr[1]);, - p.pos(2) = ParticleReal(rptr[2]);); + AMREX_D_TERM(ptemp.pos(0) = ParticleReal(rptr[0]);, + ptemp.pos(1) = ParticleReal(rptr[1]);, + ptemp.pos(2) = ParticleReal(rptr[2]);); rptr += AMREX_SPACEDIM; for (int j = 0; j < NStructReal; j++) { - p.rdata(j) = ParticleReal(*rptr); + ptemp.rdata(j) = ParticleReal(*rptr); ++rptr; } - locateParticle(p, pld, 0, finestLevel(), 0); + locateParticle(ptemp, pld, 0, finestLevel(), 0); std::pair ind(grd, pld.m_tile); @@ -979,35 +981,69 @@ ParticleContainer_impl host_int_attribs[lev][ind].resize(NumIntComps()); // add the struct - host_particles[lev][ind].push_back(p); + if constexpr(!ParticleType::is_soa_particle) + { + host_particles[lev][ind].push_back(ptemp); - // add the real... - for (int icomp = 0; icomp < NumRealComps(); icomp++) { - host_real_attribs[lev][ind][icomp].push_back(*rptr); - ++rptr; - } + // add the real... + for (int icomp = 0; icomp < NumRealComps(); icomp++) { + host_real_attribs[lev][ind][icomp].push_back(*rptr); + ++rptr; + } - // ... and int array data - for (int icomp = 0; icomp < NumIntComps(); icomp++) { - host_int_attribs[lev][ind][icomp].push_back(*iptr); - ++iptr; + // ... and int array data + for (int icomp = 0; icomp < NumIntComps(); icomp++) { + host_int_attribs[lev][ind][icomp].push_back(*iptr); + ++iptr; + } + } else { + host_particles[lev][ind]; + + for (int j = 0; j < AMREX_SPACEDIM; j++) { + host_real_attribs[pld.m_lev][ind][j].push_back(ptemp.pos(j)); + } + + host_int_attribs[pld.m_lev][ind][0].push_back(ptemp.id()); + host_int_attribs[pld.m_lev][ind][1].push_back(ptemp.cpu()); + + // read all other SoA + // add the real... + for (int icomp = AMREX_SPACEDIM; icomp < NumRealComps(); icomp++) { + host_real_attribs[lev][ind][icomp].push_back(*rptr); + ++rptr; + } + + // ... and int array data + for (int icomp = 2; icomp < NumIntComps(); icomp++) { + host_int_attribs[lev][ind][icomp].push_back(*iptr); + ++iptr; + } } } for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) - { + { for (auto& kv : host_particles[host_lev]) { auto grid = kv.first.first; auto tile = kv.first.second; const auto& src_tile = kv.second; auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile); - auto old_size = dst_tile.GetArrayOfStructs().size(); - auto new_size = old_size + src_tile.size(); + auto old_size = dst_tile.size(); + auto new_size = old_size; + if constexpr(!ParticleType::is_soa_particle) + { + new_size += src_tile.size(); + } else { + new_size += host_real_attribs[host_lev][std::make_pair(grid,tile)][0].size(); + } dst_tile.resize(new_size); - Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), - dst_tile.GetArrayOfStructs().begin() + old_size); + if constexpr(!ParticleType::is_soa_particle) + { + Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), + dst_tile.GetArrayOfStructs().begin() + old_size); + } for (int i = 0; i < NumRealComps(); ++i) { Gpu::copyAsync(Gpu::hostToDevice, diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index 9f1fe712c64..3b5ee15fdb0 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -226,6 +226,9 @@ struct ParticleTileData { AMREX_ASSERT(index < m_size); SuperParticleType sp; + for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];} + sp.id() = m_idata[0][index]; + sp.cpu() = m_idata[1][index]; for (int i = 0; i < NAR; ++i) sp.rdata(i) = m_rdata[i][index]; for (int i = 0; i < NAI; ++i) @@ -595,6 +598,9 @@ struct ConstParticleTileData { AMREX_ASSERT(index < m_size); SuperParticleType sp; + for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];} + sp.id() = m_idata[0][index]; + sp.cpu() = m_idata[1][index]; for (int i = 0; i < NAR; ++i) sp.rdata(i) = m_rdata[i][index]; for (int i = 0; i < NAI; ++i) diff --git a/Src/Particle/AMReX_WriteBinaryParticleData.H b/Src/Particle/AMReX_WriteBinaryParticleData.H index 7767972998d..f182eb2b3e8 100644 --- a/Src/Particle/AMReX_WriteBinaryParticleData.H +++ b/Src/Particle/AMReX_WriteBinaryParticleData.H @@ -13,7 +13,7 @@ struct KeepValidFilter AMREX_GPU_HOST_DEVICE int operator() (const SrcData& src, int i) const noexcept { - return (src.m_aos[i].id() > 0); + return (src.id(i) > 0); } }; @@ -197,7 +197,7 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l for (int i = 0; i < tiles.size(); i++) { const auto& ptile = pc.ParticlesAt(lev, grid, tiles[i]); const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tiles[i])); - int np_tile = ptile.GetArrayOfStructs().numParticles(); + int np_tile = ptile.numParticles(); typename PC::IntVector offsets(np_tile); int num_copies = Scan::ExclusiveSum(np_tile, pflags.begin(), offsets.begin(), Scan::retSum); @@ -283,44 +283,70 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l const Long rChunkSize = AMREX_SPACEDIM + num_output_real; rdata.resize(np*rChunkSize); - int* iptr = idata.dataPtr(); + int* iptr = idata.dataPtr(); ParticleReal* rptr = rdata.dataPtr(); for (unsigned i = 0; i < tiles.size(); i++) { const auto& ptile = pc.ParticlesAt(lev, grid, tiles[i]); const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tiles[i])); - for (int pindex = 0; pindex < ptile.GetArrayOfStructs().numParticles(); ++pindex) { - const auto& aos = ptile.GetArrayOfStructs(); - const auto& p = aos[pindex]; + for (int pindex = 0; pindex < ptile.numParticles(); ++pindex) { if (pflags[pindex]) { - packParticleIDs(iptr, p, is_checkpoint); - iptr += 2; - - for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j); - rptr += AMREX_SPACEDIM; + const auto& soa = ptile.GetStructOfArrays(); - for (int j = 0; j < PC::NStructInt; j++) { - if (write_int_comp[j]) { - *iptr = p.idata(j); - ++iptr; + // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct + // for backwards compatibility with readers + if constexpr(!PC::ParticleType::is_soa_particle) + { + const auto& aos = ptile.GetArrayOfStructs(); + const auto& p = aos[pindex]; + + // Int: id, cpu + packParticleIDs(iptr, p, is_checkpoint); + iptr += 2; + + // Real: positions + for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j); + rptr += AMREX_SPACEDIM; + + // extra AoS Int components + for (int j = 0; j < PC::NStructInt; j++) { + if (write_int_comp[j]) { + *iptr = p.idata(j); + ++iptr; + } + } + // extra AoS Real components + for (int j = 0; j < PC::NStructReal; j++) { + if (write_real_comp[j]) { + *rptr = p.rdata(j); + ++rptr; + } } } + else { + amrex::ignore_unused(is_checkpoint); + // Int: id, cpu + *iptr = soa.GetIntData(0)[pindex]; + iptr += 1; + *iptr = soa.GetIntData(1)[pindex]; + iptr += 1; + + // Real: position + for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = soa.GetRealData(j)[pindex]; + rptr += AMREX_SPACEDIM; + } - const auto& soa = ptile.GetStructOfArrays(); - for (int j = 0; j < pc.NumIntComps(); j++) { + // extra SoA int data + const int int_start_offset = PC::ParticleType::is_soa_particle ? 2 : 0; // pure SoA: skip id, cpu + for (int j = int_start_offset; j < pc.NumIntComps(); j++) { if (write_int_comp[PC::NStructInt+j]) { *iptr = soa.GetIntData(j)[pindex]; ++iptr; } } - for (int j = 0; j < PC::NStructReal; j++) { - if (write_real_comp[j]) { - *rptr = p.rdata(j); - ++rptr; - } - } - - for (int j = 0; j < pc.NumRealComps(); j++) { + // extra SoA Real components + const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: skip positions + for (int j = real_start_offset; j < pc.NumRealComps(); j++) { if (write_real_comp[PC::NStructReal+j]) { *rptr = (ParticleReal) soa.GetRealData(j)[pindex]; ++rptr; @@ -647,8 +673,7 @@ void WriteBinaryParticleDataAsync (PC const& pc, { int gid = pti.index(); const auto& ptile = pc.ParticlesAt(lev, pti); - const auto& aos = ptile.GetArrayOfStructs(); - const auto pstruct = aos().dataPtr(); + const auto& ptd = ptile.getConstParticleTileData(); const int np = ptile.numParticles(); ReduceOps reduce_op; @@ -658,7 +683,7 @@ void WriteBinaryParticleDataAsync (PC const& pc, reduce_op.eval(np, reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { - return (pstruct[i].id() > 0) ? 1 : 0; + return (ptd.id(i) > 0) ? 1 : 0; }); int np_valid = amrex::get<0>(reduce_data.value(reduce_op)); @@ -938,30 +963,43 @@ void WriteBinaryParticleDataAsync (PC const& pc, for (unsigned i = 0; i < tile_map[grid].size(); i++) { auto ptile_index = std::make_pair(grid, tile_map[grid][i]); const auto& pbox = (*myptiles)[lev][ptile_index]; - for (int pindex = 0; - pindex < pbox.GetArrayOfStructs().numParticles(); ++pindex) + const auto ptd = pbox.getConstParticleTileData(); + for (int pindex = 0; pindex < pbox.numParticles(); ++pindex) { - const auto& aos = pbox.GetArrayOfStructs(); - const auto& p = aos[pindex]; + const auto& soa = pbox.GetStructOfArrays(); + const auto& p = make_particle{}(ptd, pindex); if (p.id() <= 0) continue; - // always write these - particle_detail::packParticleIDs(iptr, p, is_checkpoint); - iptr += 2; - - // optionally write these - for (int j = 0; j < NStructInt; j++) + // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct + // for backwards compatibility with readers + if constexpr(!PC::ParticleType::is_soa_particle) { - if (write_int_comp[j]) + // Ints: id, cpu + particle_detail::packParticleIDs(iptr, p, is_checkpoint); + iptr += 2; + + // extra AoS Int components + for (int j = 0; j < NStructInt; j++) { - *iptr = p.idata(j); - ++iptr; + if (write_int_comp[j]) + { + *iptr = p.idata(j); + ++iptr; + } } } + else { + // Ints: id, cpu + *iptr = soa.GetIntData(0)[pindex]; + iptr += 1; + *iptr = soa.GetIntData(1)[pindex]; + iptr += 1; + } - const auto& soa = pbox.GetStructOfArrays(); - for (int j = 0; j < nic; j++) + // extra SoA Ints + const int int_start_offset = PC::ParticleType::is_soa_particle ? 2 : 0; // pure SoA: skip id, cpu + for (int j = int_start_offset; j < nic; j++) { if (write_int_comp[NStructInt+j]) { @@ -987,30 +1025,40 @@ void WriteBinaryParticleDataAsync (PC const& pc, for (unsigned i = 0; i < tile_map[grid].size(); i++) { auto ptile_index = std::make_pair(grid, tile_map[grid][i]); const auto& pbox = (*myptiles)[lev][ptile_index]; + const auto ptd = pbox.getConstParticleTileData(); for (int pindex = 0; - pindex < pbox.GetArrayOfStructs().numParticles(); ++pindex) + pindex < pbox.numParticles(); ++pindex) { - const auto& aos = pbox.GetArrayOfStructs(); - const auto& p = aos[pindex]; + const auto& soa = pbox.GetStructOfArrays(); + const auto& p = make_particle{}(ptd, pindex); if (p.id() <= 0) continue; - // always write these - for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j); - rptr += AMREX_SPACEDIM; - - // optionally write these - for (int j = 0; j < NStructReal; j++) + if constexpr(!PC::ParticleType::is_soa_particle) { - if (write_real_comp[j]) + // Real: position + for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j); + rptr += AMREX_SPACEDIM; + + // extra AoS real + for (int j = 0; j < NStructReal; j++) { - *rptr = p.rdata(j); - ++rptr; + if (write_real_comp[j]) + { + *rptr = p.rdata(j); + ++rptr; + } } } + else { + // Real: position + for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = soa.GetRealData(j)[pindex]; + rptr += AMREX_SPACEDIM; + } - const auto& soa = pbox.GetStructOfArrays(); - for (int j = 0; j < nrc; j++) + // extra SoA real + const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: positions + for (int j = real_start_offset; j < nrc; j++) { if (write_real_comp[NStructReal+j]) { diff --git a/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt b/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt new file mode 100644 index 00000000000..087d3ad4998 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt @@ -0,0 +1,14 @@ +# This tests requires particle support +if (NOT AMReX_PARTICLES) + return() +endif () + +foreach(D IN LISTS AMReX_SPACEDIM) + set(_sources main.cpp) + set(_input_files inputs) + + setup_test(${D} _sources _input_files) + + unset(_sources) + unset(_input_files) +endforeach() diff --git a/Tests/Particles/CheckpointRestartSOA/GNUmakefile b/Tests/Particles/CheckpointRestartSOA/GNUmakefile new file mode 100644 index 00000000000..b3510b88d2e --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA/GNUmakefile @@ -0,0 +1,31 @@ +AMREX_HOME = ../../../ + +# DEBUG = TRUE +DEBUG = FALSE + +DIM = 3 + +COMP = gnu + +PRECISION = DOUBLE + +USE_MPI = TRUE +MPI_THREAD_MULTIPLE = FALSE + +USE_OMP = FALSE + +TINY_PROFILE = TRUE + +USE_PARTICLES = TRUE + +################################################### + +EBASE = main + +include $(AMREX_HOME)/Tools/GNUMake/Make.defs + +include ./Make.package +include $(AMREX_HOME)/Src/Base/Make.package +include $(AMREX_HOME)/Src/Particle/Make.package + +include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/CheckpointRestartSOA/Make.package b/Tests/Particles/CheckpointRestartSOA/Make.package new file mode 100644 index 00000000000..6b4b865e8fc --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA/Make.package @@ -0,0 +1 @@ +CEXE_sources += main.cpp diff --git a/Tests/Particles/CheckpointRestartSOA/inputs b/Tests/Particles/CheckpointRestartSOA/inputs new file mode 100644 index 00000000000..81855c5947d --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA/inputs @@ -0,0 +1,26 @@ +# Domain size +ncells = 64 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +max_grid_size = 8 + +# Number of levels +nlevs = 1 + +# Number of components in the multifabs +ncomp = 6 + +# Number of particles per cell +nppc = 2 + +# Number of plot files to write +nplotfile = 1 + +# Number of plot files to write +nparticlefile = 1 + +# Whether to check the correctness of Checkpoint / Restart +restart_check = 1 + +directory=. diff --git a/Tests/Particles/CheckpointRestartSOA/main.cpp b/Tests/Particles/CheckpointRestartSOA/main.cpp new file mode 100644 index 00000000000..b6ccc365964 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA/main.cpp @@ -0,0 +1,245 @@ +#include +#include +#include +#include + +#include + +using namespace amrex; + +void set_grids_nested (Vector& domains, + Vector& grids, + Vector& ref_ratio); +void test (); + +int main(int argc, char* argv[]) +{ + amrex::Initialize(argc,argv); + test(); + amrex::Finalize(); +} + +void test () +{ + const int nghost = 0; + int ncells, max_grid_size, ncomp, nlevs, nppc; + int restart_check = 0, nplotfile = 1, nparticlefile = 1; + std::string directory; + + ParmParse pp; + pp.get("ncells", ncells); + pp.get("max_grid_size", max_grid_size); + pp.get("ncomp", ncomp); + pp.get("nlevs", nlevs); + pp.get("nppc", nppc); + pp.query("nplotfile", nplotfile); + pp.query("nparticlefile", nparticlefile); + pp.query("restart_check", restart_check); + pp.query("directory", directory); + + if (!directory.empty() && directory.back() != '/') { + // Include separator if one was not provided + directory += "/"; + } + + Vector domains; + Vector ba; + Vector ref_ratio; + + set_grids_nested(domains, ba, ref_ratio); + + RealBox real_box; + for (int n = 0; n < AMREX_SPACEDIM; n++) { + real_box.setLo(n, 0.0); + real_box.setHi(n, 1.0); + } + + // This sets the boundary conditions to be doubly or triply periodic + int is_per[] = {AMREX_D_DECL(1,1,1)}; + + // This defines a Geometry object for each level + Vector geom(nlevs); + geom[0].define(domains[0], &real_box, CoordSys::cartesian, is_per); + for (int lev = 1; lev < nlevs; lev++) { + geom[lev].define(domains[lev], &real_box, CoordSys::cartesian, is_per); + } + + Vector dmap(nlevs); + + // write some mesh data too, because tools like yt expect them to be there + Vector > mf(nlevs); + for (int lev = 0; lev < nlevs; lev++) { + dmap[lev] = DistributionMapping{ba[lev]}; + mf[lev] = std::make_unique(ba[lev], dmap[lev], ncomp, nghost); + mf[lev]->setVal(lev); + } + + // these don't really matter, make something up + const Real time = 0.0; + + Vector varnames; + for (int i = 0; i < ncomp; ++i) + { + varnames.push_back("component_" + std::to_string(i)); + } + + Vector level_steps(nlevs, 0); + + char fname[512]; + for (int ts = 0; ts < nplotfile; ts++) { + std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts); + + amrex::Print() << "Writing plot file [" << fname << "] ..." << std::endl; + + WriteMultiLevelPlotfile(fname, nlevs, amrex::GetVecOfConstPtrs(mf), + varnames, geom, time, level_steps, ref_ratio); + + amrex::Print() << " done \n"; + } + + // Add some particles + constexpr int NReal = 12; + constexpr int NInt = 4; + + typedef ParticleContainerPureSoA MyPC; + MyPC myPC(geom, dmap, ba, ref_ratio); + myPC.SetVerbose(false); + + int num_particles = nppc * AMREX_D_TERM(ncells, * ncells, * ncells); + bool serialize = false; + int iseed = 451; + MyPC::ParticleInitData pdata = {{}, {}, + {1.0, 2.0, 3.0, 4.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0}, + {5, 14, 15, 16}}; + + if (nparticlefile > 0) { + amrex::Print() << "Init particles ..." << std::endl; + + myPC.InitRandom(num_particles, iseed, pdata, serialize); + + amrex::Print() << " done \n"; + + Vector particle_realnames; + for (int i = 0; i < NReal; ++i) + particle_realnames.push_back("particle_real_component_" + std::to_string(i)); + + Vector particle_intnames; + for (int i = 0; i < NInt; ++i) + particle_intnames.push_back("particle_int_component_" + std::to_string(i)); + + for (int ts = 0; ts < nparticlefile; ts++) { + std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts); + + amrex::Print() << "Writing particle file [" << fname << "] ..." << std::endl; + + myPC.Checkpoint(fname, "particle0", false, particle_realnames, particle_intnames); + + amrex::Print() << " done \n"; + } + } + + AsyncOut::Finish(); + ParallelDescriptor::Barrier(); + + char directory_path[512]; + if (restart_check && nparticlefile > 0) + { + MyPC newPC(geom, dmap, ba, ref_ratio); + std::snprintf(directory_path, sizeof directory_path, "%s%s", directory.c_str(), "plt00000"); + newPC.Restart(directory_path, "particle0"); + + using PType = typename MyPC::SuperParticleType; + + for (int icomp=0; icomp Real + { + return p.rdata(icomp); + }); + + auto sm_old = amrex::ReduceSum(myPC, + [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real + { + return p.rdata(icomp); + }); + + ParallelDescriptor::ReduceRealSum(sm_new); + ParallelDescriptor::ReduceRealSum(sm_old); + + amrex::Print() << sm_old << " " << sm_new << "\n"; + AMREX_ALWAYS_ASSERT(sm_old == sm_new); + } + + for (int icomp=0; icomp Real + { + return p.idata(icomp); + }); + + auto sm_old = amrex::ReduceSum(myPC, + [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real + { + return p.idata(icomp); + }); + + ParallelDescriptor::ReduceRealSum(sm_new); + ParallelDescriptor::ReduceRealSum(sm_old); + + amrex::Print() << sm_old << " " << sm_new << "\n"; + AMREX_ALWAYS_ASSERT(sm_old == sm_new); + } + } +} + +void set_grids_nested (Vector& domains, + Vector& grids, + Vector& ref_ratio) +{ + int ncells, max_grid_size, nlevs; + + ParmParse pp; + pp.get("ncells", ncells); + pp.get("max_grid_size", max_grid_size); + pp.get("nlevs", nlevs); + + AMREX_ALWAYS_ASSERT(nlevs < 2); // relax this later + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + IntVect domain_hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); + + domains.resize(nlevs); + domains[0].setSmall(domain_lo); + domains[0].setBig(domain_hi); + + ref_ratio.resize(nlevs-1); + for (int lev = 1; lev < nlevs; lev++) + ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); + + grids.resize(nlevs); + grids[0].define(domains[0]); + + // Now we make the refined level be the center eighth of the domain + if (nlevs > 1) { + int n_fine = ncells*ref_ratio[0][0]; + IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4)); + IntVect refined_hi(AMREX_D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1)); + + // Build a box for the level 1 domain + Box refined_patch(refined_lo, refined_hi); + grids[1].define(refined_patch); + } + + // break the BoxArrays at both levels into max_grid_size^3 boxes + for (int lev = 0; lev < nlevs; lev++) { + grids[lev].maxSize(max_grid_size); + } + + for (int lev = 1; lev < nlevs; lev++) { + domains[lev] = amrex::refine(domains[lev-1], ref_ratio[lev-1]); + } +} diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt b/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt new file mode 100644 index 00000000000..087d3ad4998 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt @@ -0,0 +1,14 @@ +# This tests requires particle support +if (NOT AMReX_PARTICLES) + return() +endif () + +foreach(D IN LISTS AMReX_SPACEDIM) + set(_sources main.cpp) + set(_input_files inputs) + + setup_test(${D} _sources _input_files) + + unset(_sources) + unset(_input_files) +endforeach() diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile b/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile new file mode 120000 index 00000000000..402f7774f22 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile @@ -0,0 +1 @@ +../CheckpointRestartSOA/GNUmakefile \ No newline at end of file diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package b/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package new file mode 120000 index 00000000000..6fb6dc981b7 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package @@ -0,0 +1 @@ +../CheckpointRestartSOA/Make.package \ No newline at end of file diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs b/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs new file mode 100644 index 00000000000..8efbea33ad4 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs @@ -0,0 +1,28 @@ +# Domain size +ncells = 64 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +max_grid_size = 8 + +# Number of levels +nlevs = 1 + +# Number of components in the multifabs +ncomp = 6 + +# Number of particles per cell +nppc = 2 + +# Number of plot files to write +nplotfile = 1 + +# Number of plot files to write +nparticlefile = 1 + +# Whether to check the correctness of Checkpoint / Restart +restart_check = 1 + +directory=. + +amrex.async_out=1 \ No newline at end of file diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp b/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp new file mode 120000 index 00000000000..36e9ca62484 --- /dev/null +++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp @@ -0,0 +1 @@ +../CheckpointRestartSOA/main.cpp \ No newline at end of file