Skip to content
This repository has been archived by the owner on Jun 27, 2024. It is now read-only.

Commit

Permalink
Prepare the PR to merge with PeleLMeX (#20)
Browse files Browse the repository at this point in the history 
* give options for EB boundaries, now only support 1st and 2nd types

* deprecate level by level solution for now

* add the flag only for the linker

* update the interface

* delete a file
  • Loading branch information
@wjge
wjge committed Nov 13, 2023
1 parent c000a23 commit c7cfa69
Show file tree
Hide file tree
Showing 8 changed files with 102 additions and 60 deletions.
3 changes: 1 addition & 2 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,11 @@ if(PELERAD_ENABLE_CUDA)
cmake_policy(SET CMP0104 OLD)
endif()
enable_language(CUDA)
# include(${AMREX_HOME_DIR}/Tools/CMake/AMReX_SetupCUDA.cmake)
endif()

if(PELERAD_ENABLE_HIP)
add_definitions(-DPELERAD_USE_HIP=ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++fs")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++fs")
endif()

target_include_directories(PeleRad INTERFACE
Expand Down
2 changes: 2 additions & 0 deletions inputs/inputs.tstPOneMultiEB
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,5 @@ consolidation = 0

lo_bc = Periodic Periodic Periodic
hi_bc = Periodic Periodic Periodic

ebbc_type = 1
2 changes: 2 additions & 0 deletions inputs/inputs.tstPOneSingleEB
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,5 @@ consolidation = 0

lo_bc = Periodic Periodic Periodic
hi_bc = Periodic Periodic Periodic

ebbc_type = 1
12 changes: 8 additions & 4 deletions src/MLMGParam.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ struct MLMGParam
bool use_hypre_;
amrex::Array<amrex::LinOpBCType, AMREX_SPACEDIM> lobc_;
amrex::Array<amrex::LinOpBCType, AMREX_SPACEDIM> hibc_;
int ebbc_type_;
std::string kppath_;

AMREX_GPU_HOST
Expand All @@ -41,8 +42,8 @@ struct MLMGParam
max_bottom_iter_(20),
reltol_(1.e-4),
abstol_(1.e-4),
bottom_reltol_(1.e-5),
bottom_abstol_(1.e-5),
bottom_reltol_(1.e-6),
bottom_abstol_(1.e-6),
linop_maxorder_(3),
max_coarsening_level_(20),
agg_grid_size_(-1),
Expand All @@ -51,7 +52,8 @@ struct MLMGParam
consolidation_(1),
composite_solve_(1),
fine_level_solve_only_(0),
use_hypre_(0)
use_hypre_(0),
ebbc_type_(2)
{
}

Expand Down Expand Up @@ -83,12 +85,14 @@ struct MLMGParam
pp_.getarr("lo_bc", lo_bc_char_, 0, AMREX_SPACEDIM);
pp_.getarr("hi_bc", hi_bc_char_, 0, AMREX_SPACEDIM);

std::map<std::string, amrex::LinOpBCType> bctype;
std::unordered_map<std::string, amrex::LinOpBCType> bctype;
bctype["Dirichlet"] = amrex::LinOpBCType::Dirichlet;
bctype["Periodic"] = amrex::LinOpBCType::Periodic;
bctype["Neumann"] = amrex::LinOpBCType::Neumann;
bctype["Robin"] = amrex::LinOpBCType::Robin;

pp_.query("ebbc_type", ebbc_type_);

amrex::Print() << "Define radiation BC:"
<< "\n";

Expand Down
1 change: 0 additions & 1 deletion src/Make.package
View file

This file was deleted.

9 changes: 4 additions & 5 deletions src/POneMultiEB.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,10 @@ class POneMultiEB
GetArrOfPtrs(face_bcoef), bcoef, geom);
mlabec.setBCoeffs(ilev, amrex::GetArrOfConstPtrs(face_bcoef));

amrex::MultiFab beta(grids_[ilev], dmap_[ilev], 1, 1,
amrex::MFInfo(), *factory_[ilev]);
beta.setVal(1.0);

mlabec.setEBDirichlet(ilev, solution, beta);
if (mlmgpp_.ebbc_type_ == 1)
{
mlabec.setEBDirichlet(ilev, solution, 1.0);
}
}
auto const tol_rel = mlmgpp_.reltol_;
auto const tol_abs = mlmgpp_.abstol_;
Expand Down
9 changes: 4 additions & 5 deletions src/POneSingleEB.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,10 @@ class POneSingleEB

mlabec.setBCoeffs(0, amrex::GetArrOfConstPtrs(face_bcoef));

// this is the BCoef associated with an EB face
amrex::MultiFab beta(grids, dmap, 1, 0, amrex::MFInfo(), *factory);
beta.setVal(1.0);

mlabec.setEBDirichlet(0, solution, beta);
if (mlmgpp_.ebbc_type_ == 1)
{
mlabec.setEBDirichlet(0, solution, 1.0);
}

amrex::MLMG mlmg(mlabec);
mlmg.setMaxIter(max_iter);
Expand Down
124 changes: 81 additions & 43 deletions src/PeleLMRad.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
#include <POneMultiEB.hpp>
#else
#include <POneMulti.hpp>
// #include <POneMultiLevbyLev.hpp>
#endif

#include <POneMultiLevbyLev.hpp>
#include <PlanckMean.hpp>
#include <SpectralModels.hpp>

Expand All @@ -21,16 +21,16 @@ namespace PeleRad
class Radiation
{
private:
MLMGParam mlmgpp_;

int ref_ratio_;

PlanckMean radprop;

amrex::Vector<amrex::Geometry>& geom_;
amrex::Vector<amrex::BoxArray>& grids_;
amrex::Vector<amrex::DistributionMapping>& dmap_;

RadComps rc_;

MLMGParam mlmgpp_;

amrex::Vector<amrex::MultiFab> solution_;
amrex::Vector<amrex::MultiFab> rhs_;
amrex::Vector<amrex::MultiFab> acoef_;
Expand All @@ -41,43 +41,33 @@ class Radiation

amrex::Vector<amrex::MultiFab> absc_;

RadComps rc_;

bool composite_solve_;
// bool composite_solve_;

#ifdef AMREX_USE_EB
std::unique_ptr<POneMultiEB> rte_;
#else
std::unique_ptr<POneMulti> rte_;
#endif

std::unique_ptr<POneMultiLevbyLev> rtelevbylev_;
// std::unique_ptr<POneMultiLevbyLev> rtelevbylev_;

#ifdef AMREX_USE_EB
amrex::Vector<amrex::EBFArrayBoxFactory const*>& factory_;
amrex::Vector<amrex::EBFArrayBoxFactory const*> ebfactVec_;
#endif

public:
AMREX_GPU_HOST
Radiation(amrex::Vector<amrex::Geometry>& geom,
amrex::Vector<amrex::BoxArray>& grids,
amrex::Vector<amrex::DistributionMapping>& dmap, RadComps rc,
amrex::ParmParse const& mlmgpp, int const& ref_ratio
amrex::ParmParse const& mlmgpp
#ifdef AMREX_USE_EB
,
amrex::Vector<amrex::EBFArrayBoxFactory const*>& factory
amrex::Vector<
std::unique_ptr<amrex::FabFactory<amrex::FArrayBox>>> const& factory
#endif
)
: geom_(geom),
grids_(grids),
dmap_(dmap),
rc_(rc),
mlmgpp_(mlmgpp),
ref_ratio_(ref_ratio)
#ifdef AMREX_USE_EB
,
factory_(factory)
#endif
: geom_(geom), grids_(grids), dmap_(dmap), rc_(rc), mlmgpp_(mlmgpp)
{
if (amrex::ParallelDescriptor::IOProcessor()) rc_.checkIndices();

Expand All @@ -86,37 +76,49 @@ class Radiation
solution_.resize(nlevels);
rhs_.resize(nlevels);
acoef_.resize(nlevels);

bcoef_.resize(nlevels);

robin_a_.resize(nlevels);
robin_b_.resize(nlevels);
robin_f_.resize(nlevels);
absc_.resize(nlevels);

#ifdef AMREX_USE_EB
for (int ilev = 0; ilev < nlevels; ++ilev)
{
ebfactVec_.push_back(&(
static_cast<amrex::EBFArrayBoxFactory const&>(*factory[ilev])));
}

initVars(grids, dmap, ebfactVec_);
#else
initVars(grids, dmap);
#endif

loadSpecModel();

composite_solve_ = mlmgpp_.composite_solve_;
// composite_solve_ = mlmgpp_.composite_solve_;

if (composite_solve_)
{
// if (composite_solve_)
// {

#ifdef AMREX_USE_EB
rte_ = std::make_unique<POneMultiEB>(mlmgpp_, geom_, grids_, dmap_,
factory_, solution_, rhs_, acoef_, bcoef_, robin_a_, robin_b_,
robin_f_);
rte_ = std::make_unique<POneMultiEB>(mlmgpp_, geom_, grids_, dmap_,
ebfactVec_, solution_, rhs_, acoef_, bcoef_, robin_a_, robin_b_,
robin_f_);
#else
rte_ = std::make_unique<POneMulti>(mlmgpp_, geom_, grids_, dmap_,
solution_, rhs_, acoef_, bcoef_, robin_a_, robin_b_, robin_f_);
rte_ = std::make_unique<POneMulti>(mlmgpp_, geom_, grids_, dmap_,
solution_, rhs_, acoef_, bcoef_, robin_a_, robin_b_, robin_f_);
#endif
}
else
{
rtelevbylev_ = std::make_unique<POneMultiLevbyLev>(mlmgpp_,
ref_ratio_, geom_, grids_, dmap_, solution_, rhs_, acoef_,
bcoef_, robin_a_, robin_b_, robin_f_);
}
// }
// else
// {
// level by level option is not ready
// rtelevbylev_ =
// std::make_unique<POneMultiLevbyLev>(mlmgpp_, 2,
// geom_, grids_, dmap_, solution_, rhs_, acoef_, bcoef_,
// robin_a_, robin_b_, robin_f_);
// }
}

AMREX_GPU_HOST
Expand Down Expand Up @@ -145,7 +147,7 @@ class Radiation
,
int ilev)
{
// amrex::Print() << "update radiative properties"<< "\n";
// amrex::Print() << "update radiative properties \n";

auto const& kpco2 = radprop.kpco2();
auto const& kph2o = radprop.kph2o();
Expand Down Expand Up @@ -279,14 +281,50 @@ class Radiation
}
}

#ifdef AMREX_USE_EB
void initVars(amrex::Vector<amrex::BoxArray> const& grids,
amrex::Vector<amrex::DistributionMapping> const& dmap,
amrex::Vector<amrex::EBFArrayBoxFactory const*>& factory)
{
amrex::IntVect ng = amrex::IntVect { 1 };
grids_ = grids;
dmap_ = dmap;

for (int ilev = 0; ilev < grids.size(); ++ilev)
{
solution_[ilev].define(grids[ilev], dmap[ilev], 1, ng,
amrex::MFInfo(), *factory[ilev]);
rhs_[ilev].define(
grids[ilev], dmap[ilev], 1, 0, amrex::MFInfo(), *factory[ilev]);
acoef_[ilev].define(
grids[ilev], dmap[ilev], 1, 0, amrex::MFInfo(), *factory[ilev]);
bcoef_[ilev].define(grids[ilev], dmap[ilev], 1, ng, amrex::MFInfo(),
*factory[ilev]);
robin_a_[ilev].define(grids[ilev], dmap[ilev], 1, ng,
amrex::MFInfo(), *factory[ilev]);
robin_b_[ilev].define(grids[ilev], dmap[ilev], 1, ng,
amrex::MFInfo(), *factory[ilev]);
robin_f_[ilev].define(grids[ilev], dmap[ilev], 1, ng,
amrex::MFInfo(), *factory[ilev]);
absc_[ilev].define(
grids[ilev], dmap[ilev], 1, 0, amrex::MFInfo(), *factory[ilev]);

solution_[ilev].setVal(0.0, 0, 1, ng);
bcoef_[ilev].setVal(1.0, 0, 1, ng);
}
}
#endif

void evaluateRad()
{
// std::cout << "begin of evaluateRad() \n";

for (int ilev = 0; ilev < grids_.size(); ++ilev)
bcoef_[ilev].FillBoundary();
if (composite_solve_)
rte_->solve();
else
rtelevbylev_->solve();
// if (composite_solve_)
rte_->solve();
// else
// rtelevbylev_->solve();
}

void calcRadSource(amrex::MFIter const& mfi,
Expand Down

0 comments on commit c7cfa69

Please sign in to comment.