Skip to content
This repository has been archived by the owner on Jun 27, 2024. It is now read-only.

Add one test for EB support #16

Merged
merged 6 commits into from
Oct 10, 2023
Merged

Add one test for EB support #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3436f40
init commit for EB
wjge Aug 24, 2023
be4cbaa
generate the rotated box
wjge Aug 24, 2023
fabd2b5
a test case starts to work
wjge Oct 5, 2023
9c241cf
clean up
wjge Oct 5, 2023
a2e212f
no ghost is needed for this test
wjge Oct 6, 2023
9e72281
fix one seg fault
wjge Oct 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
-DPELERAD_DIM=3 \
-DPELERAD_ENABLE_MPI=OFF \
-DPELERAD_ENABLE_OPENMP=OFF \
-DPELERAD_ENABLE_EB=OFF \
-DPELERAD_ENABLE_EB=ON \
-DPELERAD_ENABLE_LINEARSOLVERS=ON \
-DPELERAD_ENABLE_HYPRE=OFF \
-DPELERAD_ENABLE_TESTS=ON \
Expand Down Expand Up @@ -79,7 +79,7 @@ jobs:
-DPELERAD_DIM=3 \
-DPELERAD_ENABLE_MPI=ON \
-DPELERAD_ENABLE_OPENMP=OFF \
-DPELERAD_ENABLE_EB=OFF \
-DPELERAD_ENABLE_EB=ON \
-DPELERAD_ENABLE_LINEARSOLVERS=ON \
-DPELERAD_ENABLE_HYPRE=OFF \
-DPELERAD_ENABLE_TESTS=ON \
Expand Down
23 changes: 23 additions & 0 deletions inputs/inputs.tstPOneSingleEB
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#amr
max_level = 0
n_cell = 64
max_grid_size = 32
plot_file_name = "pltSingleEB"

#mlmg
use_hypre = 0
verbose = 0
bottom_verbose = 0
max_iter = 10
max_fmg_iter = 0
max_coarsening_level = 10
reltol = 1.e-5
abstol = 1.e-5
bottom_reltol = 1.e-5
bottom_abstol = 1.e-5
linop_maxorder = 2
agglomeration = 1
consolidation = 0

lo_bc = Periodic Periodic Periodic
hi_bc = Periodic Periodic Periodic
29 changes: 0 additions & 29 deletions inputs/inputs.tstPOneSolve
View file
Open in desktop

This file was deleted.

26 changes: 0 additions & 26 deletions inputs/inputs.tstPOneSolveEB
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion scripts/build_frontier_hip.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ cmake \
-DCMAKE_CXX_COMPILER=$(which CC)\
-DPELERAD_DIM=3 \
-DPELERAD_ENABLE_MPI=OFF \
-DPELERAD_ENABLE_EB=OFF \
-DPELERAD_ENABLE_EB=ON \
-DPELERAD_ENABLE_LINEARSOLVERS=ON \
-DPELERAD_ENABLE_HYPRE=OFF \
-DPELERAD_ENABLE_PARTICLES=OFF \
Expand Down
168 changes: 168 additions & 0 deletions src/POneSingleEB.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#ifndef PONESINGLE_HPP
#define PONESINGLE_HPP

#include <AMRParam.hpp>
#include <AMReX_EB2.H>
#include <AMReX_EBMultiFabUtil.H>
#include <AMReX_FArrayBox.H>
#include <AMReX_MLEBABecLap.H>
#include <AMReX_MLMG.H>
#include <AMReX_MultiFabUtil.H>
#include <AMReX_ParmParse.H>
#include <MLMGParam.hpp>

namespace PeleRad
{

class POneSingleEB
{
private:
MLMGParam mlmgpp_;

public:
amrex::Geometry const& geom_;
amrex::BoxArray const& grids_;
amrex::DistributionMapping const& dmap_;

std::unique_ptr<amrex::EBFArrayBoxFactory> const& factory_;

amrex::MultiFab& solution_;
amrex::MultiFab const& rhs_;
amrex::MultiFab const& acoef_;
amrex::Array<amrex::MultiFab, AMREX_SPACEDIM>& bcoef_;

amrex::MultiFab const& robin_a_;
amrex::MultiFab const& robin_b_;
amrex::MultiFab const& robin_f_;

amrex::Real const ascalar = 1.0;
amrex::Real const bscalar = 1.0 / 3.0;

// constructor
POneSingleEB(MLMGParam const& mlmgpp, amrex::Geometry const& geom,
amrex::BoxArray const& grids, amrex::DistributionMapping const& dmap,
std::unique_ptr<amrex::EBFArrayBoxFactory> const& factory,
amrex::MultiFab& solution, amrex::MultiFab const& rhs,
amrex::MultiFab const& acoef,
amrex::Array<amrex::MultiFab, AMREX_SPACEDIM>& bcoef,
amrex::MultiFab const& robin_a, amrex::MultiFab const& robin_b,
amrex::MultiFab const& robin_f)
: mlmgpp_(mlmgpp),
geom_(geom),
grids_(grids),
dmap_(dmap),
factory_(factory),
solution_(solution),
rhs_(rhs),
acoef_(acoef),
bcoef_(bcoef),
robin_a_(robin_a),
robin_b_(robin_b),
robin_f_(robin_f) {};

void solve()
{
amrex::MultiFab const& vfrc = factory_->getVolFrac();

auto const max_coarsening_level = mlmgpp_.max_coarsening_level_;
auto const max_iter = mlmgpp_.max_iter_;
auto const max_fmg_iter = mlmgpp_.max_fmg_iter_;
auto const verbose = mlmgpp_.verbose_;
auto const bottom_verbose = mlmgpp_.bottom_verbose_;
auto const tol_rel = mlmgpp_.reltol_;
auto const tol_abs = mlmgpp_.abstol_;
auto const use_hypre = mlmgpp_.use_hypre_;

auto const& lobc = mlmgpp_.lobc_;
auto const& hibc = mlmgpp_.hibc_;

auto const& geom = geom_;
auto const& grids = grids_;
auto const& dmap = dmap_;
auto const& factory = factory_;

auto& solution = solution_;
auto const& rhs = rhs_;
auto const& acoef = acoef_;
auto& bcoef = bcoef_;
auto const& robin_a = robin_a_;
auto const& robin_b = robin_b_;
auto const& robin_f = robin_f_;

amrex::MLEBABecLap mlabec({ geom }, { grids }, { dmap },
amrex::LPInfo().setMaxCoarseningLevel(max_coarsening_level),
{ factory.get() });

mlabec.setDomainBC(lobc, hibc);

// mlabec.setLevelBC(0, &solution, &robin_a, &robin_b, &robin_f);
mlabec.setLevelBC(0, nullptr);

mlabec.setScalars(ascalar, bscalar);

mlabec.setACoeffs(0, acoef);

// amrex::Array<amrex::MultiFab, AMREX_SPACEDIM> face_bcoef;
/*
for (int idim = 0; idim < AMREX_SPACEDIM; ++idim)
{
bcoef[idim].define(
amrex::convert(grids,
amrex::IntVect::TheDimensionVector(idim)), dmap, 1, 0,
amrex::MFInfo(), *factory);

const amrex::BoxArray& ba = amrex::convert(
bcoef[idim].boxArray(),
amrex::IntVect::TheDimensionVector(idim));
face_bcoef[idim].define(ba,
bcoef.DistributionMap(), 1, 0);

}
*/
// amrex::average_cellcenter_to_face(
// GetArrOfPtrs(face_bcoef), bcoef, geom);

// mlabec.setBCoeffs(0, amrex::GetArrOfConstPtrs(face_bcoef));
mlabec.setBCoeffs(0, amrex::GetArrOfConstPtrs(bcoef));

// this is the BCoef associated iwth an EB face
amrex::MultiFab beta(grids, dmap, 1, 0, amrex::MFInfo(), *factory);
beta.setVal(1.0);

mlabec.setEBDirichlet(0, solution, beta);

amrex::MLMG mlmg(mlabec);
mlmg.setMaxIter(max_iter);
mlmg.setMaxFmgIter(max_fmg_iter);
mlmg.setVerbose(verbose);
mlmg.setBottomVerbose(bottom_verbose);

if (use_hypre) mlmg.setBottomSolver(amrex::MLMG::BottomSolver::hypre);

mlmg.solve({ &solution }, { &rhs }, tol_rel, tol_abs);
}

void calcRadSource(amrex::MultiFab& rad_src)
{
for (amrex::MFIter mfi(rad_src); mfi.isValid(); ++mfi)
{
amrex::Box const& bx = mfi.validbox();

auto const& rhsfab = rhs_.array(mfi);
auto const& solfab = solution_.array(mfi);
auto const& acfab = acoef_.array(mfi);

auto radfab = rad_src.array(mfi);

amrex::ParallelFor(bx,
[=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
radfab(i, j, k, 4)
= acfab(i, j, k) * solfab(i, j, k) - rhsfab(i, j, k);
});
}
}
};

}

#endif
17 changes: 17 additions & 0 deletions tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ target_link_libraries(PeleRad_POneSingle.exe PRIVATE PeleRad Boost::unit_test_fr
target_compile_definitions(PeleRad_POneSingle.exe PRIVATE BOOST_TEST_DYN_LINK)
add_test(NAME PeleRad_POneSingle_Test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ./PeleRad_POneSingle.exe "${CMAKE_SOURCE_DIR}/inputs/inputs.tstPOneSingle")

if(PELERAD_ENABLE_EB)
add_executable(PeleRad_POneSingleEB.exe tstPOneSingleEB.cpp ut_main.cpp)
target_include_directories(PeleRad_POneSingleEB.exe PUBLIC ${AMREX_HOME_DIR}/Src/Base ${AMREX_HOME_DIR}/Src/LinearSolvers/MLMG ${AMREX_HOME_DIR}/Src/EB)
target_link_libraries(PeleRad_POneSingleEB.exe PRIVATE PeleRad Boost::unit_test_framework amrex)
target_compile_definitions(PeleRad_POneSingleEB.exe PRIVATE BOOST_TEST_DYN_LINK)
add_test(NAME PeleRad_POneSingleEB_Test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ./PeleRad_POneSingleEB.exe "${CMAKE_SOURCE_DIR}/inputs/inputs.tstPOneSingleEB")
endif()

add_executable(PeleRad_POneMulti.exe tstPOneMulti.cpp ut_main.cpp)
target_include_directories(PeleRad_POneMulti.exe PUBLIC ${AMREX_HOME_DIR}/Src/Base ${AMREX_HOME_DIR}/Src/LinearSolvers/MLMG)
target_link_libraries(PeleRad_POneMulti.exe PRIVATE PeleRad Boost::unit_test_framework amrex)
Expand Down Expand Up @@ -64,6 +72,14 @@ target_link_libraries(PeleRad_POneSingle.exe PRIVATE PeleRad Boost::unit_test_fr
target_compile_definitions(PeleRad_POneSingle.exe PRIVATE BOOST_TEST_DYN_LINK)
add_test(NAME PeleRad_POneSingle_Test COMMAND ./PeleRad_POneSingle.exe "${CMAKE_SOURCE_DIR}/inputs/inputs.tstPOneSingle" )

if(PELERAD_ENABLE_EB)
add_executable(PeleRad_POneSingleEB.exe tstPOneSingleEB.cpp ut_main.cpp)
target_include_directories(PeleRad_POneSingleEB.exe PUBLIC ${AMREX_HOME_DIR}/Src/Base ${AMREX_HOME_DIR}/Src/LinearSolvers/MLMG ${AMREX_HOME_DIR}/Src/EB)
target_link_libraries(PeleRad_POneSingleEB.exe PRIVATE PeleRad Boost::unit_test_framework amrex)
target_compile_definitions(PeleRad_POneSingleEB.exe PRIVATE BOOST_TEST_DYN_LINK)
add_test(NAME PeleRad_POneSingleEB_Test COMMAND ./PeleRad_POneSingleEB.exe "${CMAKE_SOURCE_DIR}/inputs/inputs.tstPOneSingleEB" )
endif()

add_executable(PeleRad_POneMulti.exe tstPOneMulti.cpp ut_main.cpp)
target_include_directories(PeleRad_POneMulti.exe PUBLIC ${AMREX_HOME_DIR}/Src/Base ${AMREX_HOME_DIR}/Src/LinearSolvers/MLMG)
target_link_libraries(PeleRad_POneMulti.exe PRIVATE PeleRad Boost::unit_test_framework amrex)
Expand All @@ -90,6 +106,7 @@ setup_target_for_cuda_compilation(PeleRad_AmrexParallelFor.exe)
setup_target_for_cuda_compilation(PeleRad_BoostReadDatabase.exe)
setup_target_for_cuda_compilation(PeleRad_AmrexGetRadProp.exe)
setup_target_for_cuda_compilation(PeleRad_POneSingle.exe)
setup_target_for_cuda_compilation(PeleRad_POneSingleEB.exe)
setup_target_for_cuda_compilation(PeleRad_POneMulti.exe)
setup_target_for_cuda_compilation(PeleRad_POneSingleAF.exe)
setup_target_for_cuda_compilation(PeleRad_POneMultiAF.exe)
Expand Down