From 7be0f75ad53ed079cc01ab0658437dedfd8a1928 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 25 Jan 2024 00:38:14 -0600 Subject: [PATCH 001/476] Adding cmake file for ANL workstation Adding cmake file for building SCREAM on ANL workstation --- components/eamxx/cmake/machine-files/anlgce.cmake | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/anlgce.cmake diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake new file mode 100644 index 000000000000..37570d58eec3 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -0,0 +1,7 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") From 107da5a2ba6c6e9c077c2f0f038d61bafb0c5e3b Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 19 Mar 2024 19:40:28 -0500 Subject: [PATCH 002/476] Support for EAMXX/SCREAM testing in ANL GCE Adding support for EAMXX standalone testing on ANL GCE machine (Run EAMXX standalone tests on ANL GCE using "./scripts/test-all-scream -m anlgce ...") --- components/eamxx/cmake/machine-files/anlgce.cmake | 5 +++++ components/eamxx/scripts/machines_specs.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 37570d58eec3..079000059d19 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -2,6 +2,11 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index df89ae7b5add..3c7f90dff3fa 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -75,7 +75,10 @@ ["mpic++","mpif90","mpicc"], "srun --mpi=pmi2 -l -N 1 --kill-on-bad-exit --cpu_bind=cores", "/lcrc/group/e3sm/baselines/chrys/intel/scream"), - + "anlgce" : ([". /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/sh", "module purge", "module load autoconf/2.69-bmnwajj automake/1.16.3-r7w24o4 libtool/2.4.6-uh3mpsu m4/1.4.19-7fztfyz cmake/3.20.5-zyz2eld gcc/11.1.0-qsjmpcg zlib/1.2.11-p7dmb5p", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), From efcc246b2a6cd43f5a8fca470e5c768701413c7b Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 24 Jan 2024 13:21:51 -0600 Subject: [PATCH 003/476] Updating SCORPIO to v1.6.1 SCORPIO v1.6.0 includes the following fixes/enhancements, * Support for reading ADIOS BP files and updates to the BP to NetCDF conversion tool to integrate with CIME/E3SM * Adding a SCORPIO CMake package during install * Support for new CDF5 types in the C interface * Support for pio_set_fill() * Fix for performance issues while writing large output files (hang during enddef() call) * Multiple fixes including fix for build issues on Perlmutter SCORPIO v1.6.1 includes the following fixes, * Fixes for ADIOS to NetCDF conversion tool hang/crashes in certain scenarios --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index de0b1ca2200f..16e01694a011 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit de0b1ca2200f62c6eb5e3fd40147965409e97123 +Subproject commit 16e01694a011b50bd5ff68277aa2de204dfad36d From c3cc055f39e74863ea030d848c2b8f2e1b7d8b66 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 25 Jan 2024 00:42:27 -0600 Subject: [PATCH 004/476] Option to specify I/O type for hist/rest files Adding an option to specify the SCORPIO I/O type (PnetCDF, ADIOS etc) to use for history and restart files via SCREAM YAML configuration files. The default YAML configuration files are modified to add a new parameter, "iotype", that can be used to specify the I/O types to use for history and restart files. The default is specified such that the history files are written out using PnetCDF (uses the pnetcdf iotype) while the I/O type used for restart files is specified as "default" (picked up via CIME for E3SM runs) Some function interfaces were modified to accept the new I/O type. Tests that use the modified function interfaces were updated accordingly --- .../cime_config/namelist_defaults_scream.xml | 1 + .../control/intensive_observation_period.cpp | 4 +- .../src/physics/spa/spa_functions_impl.hpp | 2 +- .../grid/remap/horiz_interp_remapper_data.cpp | 2 +- .../share/grid/remap/vertical_remapper.cpp | 2 +- .../eamxx/src/share/io/scorpio_input.cpp | 5 +- .../eamxx/src/share/io/scorpio_input.hpp | 1 + .../src/share/io/scream_io_file_specs.hpp | 2 + .../eamxx/src/share/io/scream_io_utils.hpp | 34 +++++++++++++ .../src/share/io/scream_output_manager.cpp | 10 ++-- .../src/share/io/scream_scorpio_interface.F90 | 49 ++++++++++++++----- .../src/share/io/scream_scorpio_interface.cpp | 16 +++--- .../src/share/io/scream_scorpio_interface.hpp | 2 +- .../io/scream_scorpio_interface_iso_c2f.F90 | 5 +- .../src/share/io/tests/io_remap_test.cpp | 2 +- .../share/tests/coarsening_remapper_tests.cpp | 2 +- .../tests/refining_remapper_p2p_tests.cpp | 2 +- .../share/tests/vertical_remapper_tests.cpp | 2 +- .../share/util/eamxx_time_interpolation.cpp | 2 +- .../homme_shoc_cld_p3_rrtmgp/output.yaml | 1 + 20 files changed, 110 insertions(+), 36 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 07ef2793f38c..6c240f3542da 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -503,6 +503,7 @@ be lost if SCREAM_HACK_XML is not enabled. ./${CASE}.scream + default ${REST_N} ${REST_OPTION} diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index d925b21bfaa3..ff2335c940b0 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -45,7 +45,7 @@ void read_dimensionless_variable_from_file(const std::string& filename, int ncid, varid, err1, err2; bool was_open = scorpio::is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - scorpio::register_file(filename,scorpio::FileMode::Read); + scorpio::register_file(filename,scorpio::FileMode::Read,0); } ncid = scorpio::get_file_ncid_c2f (filename.c_str()); err1 = PIOc_inq_varid(ncid,varname.c_str(),&varid); @@ -87,7 +87,7 @@ void read_variable_from_file(const std::string& filename, } // Read into data - scorpio::register_file(filename, scorpio::FileMode::Read); + scorpio::register_file(filename, scorpio::FileMode::Read,0); std::string io_decomp_tag = varname+","+filename; scorpio::register_variable(filename, varname, varname, dimnames, vartype, io_decomp_tag); std::vector dof_offsets(data_size); diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 748983006a19..eee2b093a2c3 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -78,7 +78,7 @@ create_horiz_remapper ( { using namespace ShortFieldTagsNames; - scorpio::register_file(spa_data_file,scorpio::Read); + scorpio::register_file(spa_data_file,scorpio::Read,0); const int nlevs_data = scorpio::get_dimlen(spa_data_file,"lev"); const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index d44fc6aa2f0f..298a156ae240 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -38,7 +38,7 @@ get_my_triplets (const std::string& map_file) const using namespace ShortFieldTagsNames; // 1. Load the map file chunking it evenly across all ranks - scorpio::register_file(map_file,scorpio::FileMode::Read); + scorpio::register_file(map_file,scorpio::FileMode::Read,0); // 1.1 Create a "helper" grid, with as many dofs as the number // of triplets in the map file, and divided linearly across ranks diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 9abefed9eed7..59b1f7b39153 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -54,7 +54,7 @@ VerticalRemapper (const grid_ptr_type& src_grid, // remapping the target field will be defined on the same DOFs // as the source field, but will have a different number of // vertical levels. - scorpio::register_file(map_file,scorpio::FileMode::Read); + scorpio::register_file(map_file,scorpio::FileMode::Read,0); m_num_remap_levs = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index c4f023cc396f..858fcdd6b1b9 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -350,7 +350,10 @@ void AtmosphereInput::finalize() /* ---------------------------------------------------------- */ void AtmosphereInput::init_scorpio_structures() { - scorpio::register_file(m_filename,scorpio::Read); + std::string iotype_str = m_params.get("iotype", "default"); + int iotype = str2iotype(iotype_str); + + scorpio::register_file(m_filename,scorpio::Read,iotype); // Register variables with netCDF file. register_variables(); diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 811d5f6f7712..4655b58777e9 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -2,6 +2,7 @@ #define SCREAM_SCORPIO_INPUT_HPP #include "share/io/scream_scorpio_interface.hpp" +#include "share/io/scream_io_utils.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/grids_manager.hpp" diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp index 0e9ab2e042af..c80db36e094f 100644 --- a/components/eamxx/src/share/io/scream_io_file_specs.hpp +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -80,6 +80,8 @@ struct IOFileSpecs { bool is_open = false; std::string filename; + int iotype = 0; + // If positive, flush the output file every these many snapshots int flush_frequency = std::numeric_limits::max(); diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 8cbcd39a1a2c..13417bfaec3c 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -59,6 +59,40 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } +inline int str2iotype(const std::string &str) +{ + if(str == "default"){ + return 0; + } + else if(str == "netcdf"){ + return 1; + } + else if(str == "pnetcdf"){ + return 2; + } + else if(str == "adios"){ + return 3; + } + else if(str == "hdf5"){ + return 4; + } + else{ + return 0; + } +} + +inline std::string iotype2str(int iotype) +{ + switch(iotype){ + case 0: return "default"; + case 1: return "netcdf"; + case 2: return "pnetcdf"; + case 3: return "adios"; + case 4: return "hdf5"; + default: return "default"; + } +} + std::string find_filename_in_rpointer ( const std::string& casename, const bool model_restart, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index e9f8f5887025..fc438ea74833 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -237,8 +237,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { - - scorpio::register_file(last_output_filename,scorpio::Read); + scorpio::register_file(last_output_filename,scorpio::Read,m_output_file_specs.iotype); int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); scorpio::eam_pio_closefile(last_output_filename); @@ -680,6 +679,11 @@ set_params (const ekat::ParameterList& params, m_checkpoint_file_specs.ftype = FileType::HistoryRestart; } } + + // Set the iotype to use for the output file + std::string iotype = m_params.get("iotype", "default"); + m_output_file_specs.iotype = str2iotype(iotype); + m_checkpoint_file_specs.iotype = str2iotype(iotype); } /*===============================================================================================*/ @@ -698,7 +702,7 @@ setup_file ( IOFileSpecs& filespecs, const auto& filename = filespecs.filename; // Register new netCDF file for output. Check if we need to append to an existing file auto mode = m_resume_output_file ? Append : Write; - register_file(filename,mode); + register_file(filename,mode,filespecs.iotype); if (m_resume_output_file) { eam_pio_redef(filename); } diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 50401f7f3051..4bb9c7efa543 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -45,7 +45,8 @@ module scream_scorpio_interface !------------ use pio_types, only: iosystem_desc_t, file_desc_t, var_desc_t, io_desc_t, & pio_noerr, pio_global, & - PIO_int, PIO_real, PIO_double, PIO_float=>PIO_real + PIO_int, PIO_real, PIO_double, PIO_float=>PIO_real,& + pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_adios use pio_kinds, only: PIO_OFFSET_KIND use mpi, only: mpi_abort, mpi_comm_size, mpi_comm_rank @@ -79,7 +80,7 @@ module scream_scorpio_interface eam_update_time, & ! Update the timestamp (i.e. time variable) for a given pio netCDF file read_time_at_index ! Returns the time stamp for a specific time index - private :: errorHandle, get_coord, is_read, is_write, is_append + private :: errorHandle, get_coord, is_read, is_write, is_append, scream_iotype_to_pio_iotype ! Universal PIO variables for the module integer :: atm_mpicom @@ -193,10 +194,11 @@ module scream_scorpio_interface !=====================================================================! ! Register a PIO file to be used for input/output operations. ! If file is already open, ensures file_purpose matches the current one - subroutine register_file(filename,file_purpose) + subroutine register_file(filename,file_purpose,iotype) character(len=*), intent(in) :: filename integer, intent(in) :: file_purpose + integer, intent(in) :: iotype type(pio_atm_file_t), pointer :: pio_file @@ -204,7 +206,7 @@ subroutine register_file(filename,file_purpose) call errorHandle("PIO ERROR: local pio_subsystem pointer has not been established yet.",-999) endif - call get_pio_atm_file(filename,pio_file,file_purpose) + call get_pio_atm_file(filename,pio_file,file_purpose,iotype) end subroutine register_file !=====================================================================! ! Mandatory call to finish the variable and dimension definition phase @@ -867,26 +869,46 @@ function is_eam_pio_subsystem_inited() result(is_it) bind(c) is_it = LOGICAL(associated(pio_subsystem),kind=c_bool) end function is_eam_pio_subsystem_inited +!=====================================================================! + function scream_iotype_to_pio_iotype(siotype) result(piotype) + integer, intent(in) :: siotype + integer :: piotype + + if(siotype == 0) then + piotype = pio_iotype + else if(siotype == 1) then + piotype = pio_iotype_netcdf + else if(siotype == 2) then + piotype = pio_iotype_pnetcdf + else if(siotype == 3) then + piotype = pio_iotype_adios + else + piotype = pio_iotype + end if + end function scream_iotype_to_pio_iotype !=====================================================================! ! Create a pio netCDF file with the appropriate name. - subroutine eam_pio_createfile(File,fname) + subroutine eam_pio_createfile(File,fname,iotype) use pio, only: pio_createfile use pio_types, only: pio_clobber type(file_desc_t), intent(inout) :: File ! Pio file Handle character(len=*), intent(in) :: fname ! Pio file name + integer, intent(in) :: iotype !-- integer :: retval ! PIO error return value integer :: mode ! Mode for how to handle the new file + integer :: piotype mode = ior(pio_mode,pio_clobber) ! Set to CLOBBER for now, TODO: fix to allow for optional mode type like in CAM - retval = pio_createfile(pio_subsystem,File,pio_iotype,fname,mode) + piotype = scream_iotype_to_pio_iotype(iotype) + retval = pio_createfile(pio_subsystem,File,piotype,fname,mode) call errorHandle("PIO ERROR: unable to create file: "//trim(fname),retval) end subroutine eam_pio_createfile !=====================================================================! ! Open an already existing netCDF file. - subroutine eam_pio_openfile(pio_file,fname) + subroutine eam_pio_openfile(pio_file,fname,iotype) use pio, only: pio_openfile use pio_types, only: pio_write, pio_nowrite @@ -895,13 +917,17 @@ subroutine eam_pio_openfile(pio_file,fname) !-- integer :: retval ! PIO error return value integer :: mode ! Mode for how to handle the new file + integer, intent(in) :: iotype + + integer :: piotype if (is_read(pio_file%purpose)) then mode = pio_nowrite else mode = pio_write endif - retval = pio_openfile(pio_subsystem,pio_file%pioFileDesc,pio_iotype,fname,mode) + piotype = scream_iotype_to_pio_iotype(iotype) + retval = pio_openfile(pio_subsystem,pio_file%pioFileDesc,piotype,fname,mode) call errorHandle("PIO ERROR: unable to open file: "//trim(fname),retval) if (is_append(pio_file%purpose)) then @@ -1361,12 +1387,13 @@ subroutine lookup_pio_atm_file(filename,pio_file,found,pio_file_list_ptr_in) end subroutine lookup_pio_atm_file !=====================================================================! ! Create a new pio file pointer based on filename. - subroutine get_pio_atm_file(filename,pio_file,purpose) + subroutine get_pio_atm_file(filename,pio_file,purpose,iotype) use pio, only: PIO_inq_dimid, PIO_inq_dimlen character(len=*),intent(in) :: filename ! Name of file to be found type(pio_atm_file_t), pointer :: pio_file ! Pointer to pio_atm_output structure associated with this filename integer,intent(in) :: purpose ! Purpose for this file lookup, 0 = find already existing, 1 = create new as output, 2 = open new as input + integer,intent(in) :: iotype logical :: found type(pio_file_list_t), pointer :: new_list_item @@ -1404,7 +1431,7 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) pio_file%purpose = purpose if (is_read(purpose) .or. is_append(purpose)) then ! Either read or append to existing file. Either way, file must exist on disk - call eam_pio_openfile(pio_file,trim(pio_file%filename)) + call eam_pio_openfile(pio_file,trim(pio_file%filename),iotype) ! Update the numRecs to match the number of recs in this file. ierr = pio_inq_dimid(pio_file%pioFileDesc,"time",time_id) if (ierr.ne.0) then @@ -1417,7 +1444,7 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) end if elseif (is_write(purpose)) then ! New output file - call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename)) + call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename),iotype) call eam_pio_createHeader(pio_file%pioFileDesc) else call errorHandle("PIO Error: get_pio_atm_file with filename = "//trim(filename)//", purpose (int) assigned to this lookup is not valid" ,-999) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index a74ceecd3930..9c193d8ae4ad 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -15,7 +15,7 @@ using scream::Int; extern "C" { // Fortran routines to be called from C++ - void register_file_c2f(const char*&& filename, const int& mode); + void register_file_c2f(const char*&& filename, const int& mode, const int& iotype); int get_file_mode_c2f(const char*&& filename); void set_decomp_c2f(const char*&& filename); void set_dof_c2f(const char*&& filename,const char*&& varname,const Int dof_len,const std::int64_t *x_dof); @@ -94,8 +94,8 @@ void eam_pio_finalize() { eam_pio_finalize_c2f(); } /* ----------------------------------------------------------------- */ -void register_file(const std::string& filename, const FileMode mode) { - register_file_c2f(filename.c_str(),mode); +void register_file(const std::string& filename, const FileMode mode, const int iotype) { + register_file_c2f(filename.c_str(),mode,iotype); } /* ----------------------------------------------------------------- */ void eam_pio_closefile(const std::string& filename) { @@ -118,7 +118,7 @@ int get_dimlen(const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -154,7 +154,7 @@ bool has_dim (const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -181,7 +181,7 @@ bool has_variable (const std::string& filename, const std::string& varname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -212,7 +212,7 @@ bool has_attribute (const std::string& filename, const std::string& varname, con bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } // Get file id @@ -513,7 +513,7 @@ ekat::any get_any_attribute (const std::string& filename, const std::string& att } /* ----------------------------------------------------------------- */ ekat::any get_any_attribute (const std::string& filename, const std::string& var_name, const std::string& att_name) { - register_file(filename,Read); + register_file(filename,Read,0); auto ncid = get_file_ncid_c2f (filename.c_str()); EKAT_REQUIRE_MSG (ncid>=0, "[get_any_attribute] Error! Could not retrieve file ncid.\n" diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index b64ca3ef25fd..acaf587d4470 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -33,7 +33,7 @@ namespace scorpio { void eam_pio_closefile(const std::string& filename); void eam_flush_file(const std::string& filename); /* Register a new file to be used for input/output with the scorpio module */ - void register_file(const std::string& filename, const FileMode mode); + void register_file(const std::string& filename, const FileMode mode, int iotype); /* Sets the IO decompostion for all variables in a particular filename. Required after all variables have been registered. Called once per file. */ int get_dimlen(const std::string& filename, const std::string& dimname); bool has_dim(const std::string& filename, const std::string& dimname); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 index 9e50842b245e..2f1263beef7d 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 @@ -76,15 +76,16 @@ function is_file_open_c2f(filename_in,purpose) result(res) bind(c) endif end function is_file_open_c2f !=====================================================================! - subroutine register_file_c2f(filename_in,purpose) bind(c) + subroutine register_file_c2f(filename_in,purpose,iotype) bind(c) use scream_scorpio_interface, only : register_file type(c_ptr), intent(in) :: filename_in integer(kind=c_int), intent(in) :: purpose + integer(kind=c_int), intent(in) :: iotype character(len=256) :: filename call convert_c_string(filename_in,filename) - call register_file(trim(filename),purpose) + call register_file(trim(filename),purpose,iotype) end subroutine register_file_c2f !=====================================================================! diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index bfed937636e2..2d8c0aa2aef4 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -106,7 +106,7 @@ TEST_CASE("io_remap_test","io_remap_test") // Write remap data to file const std::string remap_filename = "remap_weights_np"+std::to_string(io_comm.size())+".nc"; - scorpio::register_file(remap_filename, scorpio::FileMode::Write); + scorpio::register_file(remap_filename, scorpio::FileMode::Write,0); scorpio::register_dimension(remap_filename,"n_a", "n_a", ncols_src, true); scorpio::register_dimension(remap_filename,"n_b", "n_b", ncols_tgt, true); diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index 13832c490441..f7951a484454 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -235,7 +235,7 @@ void create_remap_file(const std::string& filename, const int ngdofs_tgt) const int ngdofs_src = ngdofs_tgt + 1; const int nnz = 2*ngdofs_tgt; - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_file(filename, scorpio::FileMode::Write,0); scorpio::register_dimension(filename,"n_a", "n_a", ngdofs_src, true); scorpio::register_dimension(filename,"n_b", "n_b", ngdofs_tgt, true); diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp index 00ee58ef503c..0fc4d05a8b80 100644 --- a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -122,7 +122,7 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { // Existing dofs are "copied", added dofs are averaged from neighbors const int nnz = ngdofs_src + 2*(ngdofs_src-1); - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_file(filename, scorpio::FileMode::Write,0); scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 6ec4a40c1236..db05558aa54a 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -97,7 +97,7 @@ Real data_func(const int col, const int vec, const Real pres) { void create_remap_file(const std::string& filename, const int nlevs, const std::vector& dofs_p, const std::vector& p_tgt) { - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_file(filename, scorpio::FileMode::Write,0); scorpio::register_dimension(filename,"lev","lev",nlevs, false); diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index 3515bcdaac91..c8cdccbc9148 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -273,7 +273,7 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { if (ii==0) { ts_ref = ts_file_start; } - scorpio::register_file(filename,scorpio::Read); + scorpio::register_file(filename,scorpio::Read,0); const int ntime = scorpio::get_dimlen(filename,"time"); for (int tt=0; tt Date: Thu, 1 Feb 2024 15:08:15 -0600 Subject: [PATCH 005/476] Adding ADIOS2 build for hipcc on Frontier Adding path to new ADIOS2 build with hipcc for Frontier. Also see E3SM-Project/scorpio#528 --- cime_config/machines/config_machines.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 0e2915a8c19a..b0e083c6f518 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1472,6 +1472,10 @@ spread threads + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + From ceb43365edb1677b503186e06196c0e2636c0cde Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Thu, 7 Mar 2024 18:02:17 -0500 Subject: [PATCH 006/476] Reset MPICH GPU support for conversion tool Disable MPICH/MPI GPU support for ADIOS to NetCDF conversion tool. Without this change the conversion tool crashes (with message "MPIDI_CRAY_init: GPU_SUPPORT_ENABLED is requested, but GTL library is not linked" - most likely due to the way SCREAM builds/links SCORPIO) on Frontier. --- cime_config/customize/case_post_run_io.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cime_config/customize/case_post_run_io.py b/cime_config/customize/case_post_run_io.py index bb44477ebb57..83419aa81348 100755 --- a/cime_config/customize/case_post_run_io.py +++ b/cime_config/customize/case_post_run_io.py @@ -74,6 +74,12 @@ def _convert_adios_to_nc(case): # Load the environment case.load_env(reset=True) + # Reset MPICH/MPI GPU support, if enabled + is_mpich_gpu_enabled = os.environ.get('MPICH_GPU_SUPPORT_ENABLED') + if int(0 if is_mpich_gpu_enabled is None else is_mpich_gpu_enabled) == 1: + logger.info("Resetting support for GPU in MPICH/MPI library (since its not used by the tool)") + os.environ['MPICH_GPU_SUPPORT_ENABLED'] = str(0); + run_func = lambda: run_cmd(cmd, from_dir=rundir)[0] # Run the modified case From 282405d8846bacf668da8d86c0c9bee118a41626 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Mon, 11 Mar 2024 13:36:08 -0500 Subject: [PATCH 007/476] Moving SPIO I/O type utils inside scorpio nspace Moving SCORPIO I/O type utils inside the SCORPIO namespace. Also adding a new enum for all supported SCORPIO I/O types. --- .../control/intensive_observation_period.cpp | 4 +- .../src/physics/spa/spa_functions_impl.hpp | 2 +- .../grid/remap/horiz_interp_remapper_data.cpp | 2 +- .../share/grid/remap/vertical_remapper.cpp | 2 +- .../eamxx/src/share/io/scorpio_input.cpp | 2 +- .../eamxx/src/share/io/scorpio_input.hpp | 1 - .../eamxx/src/share/io/scream_io_utils.hpp | 34 -------------- .../src/share/io/scream_output_manager.cpp | 4 +- .../src/share/io/scream_scorpio_interface.cpp | 10 ++-- .../src/share/io/scream_scorpio_interface.hpp | 47 ++++++++++++++++++- .../src/share/io/tests/io_remap_test.cpp | 2 +- .../share/tests/coarsening_remapper_tests.cpp | 2 +- .../tests/refining_remapper_p2p_tests.cpp | 2 +- .../share/tests/vertical_remapper_tests.cpp | 2 +- .../share/util/eamxx_time_interpolation.cpp | 2 +- 15 files changed, 64 insertions(+), 54 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index ff2335c940b0..d925b21bfaa3 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -45,7 +45,7 @@ void read_dimensionless_variable_from_file(const std::string& filename, int ncid, varid, err1, err2; bool was_open = scorpio::is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - scorpio::register_file(filename,scorpio::FileMode::Read,0); + scorpio::register_file(filename,scorpio::FileMode::Read); } ncid = scorpio::get_file_ncid_c2f (filename.c_str()); err1 = PIOc_inq_varid(ncid,varname.c_str(),&varid); @@ -87,7 +87,7 @@ void read_variable_from_file(const std::string& filename, } // Read into data - scorpio::register_file(filename, scorpio::FileMode::Read,0); + scorpio::register_file(filename, scorpio::FileMode::Read); std::string io_decomp_tag = varname+","+filename; scorpio::register_variable(filename, varname, varname, dimnames, vartype, io_decomp_tag); std::vector dof_offsets(data_size); diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index eee2b093a2c3..748983006a19 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -78,7 +78,7 @@ create_horiz_remapper ( { using namespace ShortFieldTagsNames; - scorpio::register_file(spa_data_file,scorpio::Read,0); + scorpio::register_file(spa_data_file,scorpio::Read); const int nlevs_data = scorpio::get_dimlen(spa_data_file,"lev"); const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index 298a156ae240..d44fc6aa2f0f 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -38,7 +38,7 @@ get_my_triplets (const std::string& map_file) const using namespace ShortFieldTagsNames; // 1. Load the map file chunking it evenly across all ranks - scorpio::register_file(map_file,scorpio::FileMode::Read,0); + scorpio::register_file(map_file,scorpio::FileMode::Read); // 1.1 Create a "helper" grid, with as many dofs as the number // of triplets in the map file, and divided linearly across ranks diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 59b1f7b39153..9abefed9eed7 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -54,7 +54,7 @@ VerticalRemapper (const grid_ptr_type& src_grid, // remapping the target field will be defined on the same DOFs // as the source field, but will have a different number of // vertical levels. - scorpio::register_file(map_file,scorpio::FileMode::Read,0); + scorpio::register_file(map_file,scorpio::FileMode::Read); m_num_remap_levs = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 858fcdd6b1b9..af0886588699 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -351,7 +351,7 @@ void AtmosphereInput::finalize() void AtmosphereInput::init_scorpio_structures() { std::string iotype_str = m_params.get("iotype", "default"); - int iotype = str2iotype(iotype_str); + int iotype = scorpio::str2iotype(iotype_str); scorpio::register_file(m_filename,scorpio::Read,iotype); diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 4655b58777e9..811d5f6f7712 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -2,7 +2,6 @@ #define SCREAM_SCORPIO_INPUT_HPP #include "share/io/scream_scorpio_interface.hpp" -#include "share/io/scream_io_utils.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/grids_manager.hpp" diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 13417bfaec3c..8cbcd39a1a2c 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -59,40 +59,6 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } -inline int str2iotype(const std::string &str) -{ - if(str == "default"){ - return 0; - } - else if(str == "netcdf"){ - return 1; - } - else if(str == "pnetcdf"){ - return 2; - } - else if(str == "adios"){ - return 3; - } - else if(str == "hdf5"){ - return 4; - } - else{ - return 0; - } -} - -inline std::string iotype2str(int iotype) -{ - switch(iotype){ - case 0: return "default"; - case 1: return "netcdf"; - case 2: return "pnetcdf"; - case 3: return "adios"; - case 4: return "hdf5"; - default: return "default"; - } -} - std::string find_filename_in_rpointer ( const std::string& casename, const bool model_restart, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index fc438ea74833..2458a4e95c84 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -682,8 +682,8 @@ set_params (const ekat::ParameterList& params, // Set the iotype to use for the output file std::string iotype = m_params.get("iotype", "default"); - m_output_file_specs.iotype = str2iotype(iotype); - m_checkpoint_file_specs.iotype = str2iotype(iotype); + m_output_file_specs.iotype = scorpio::str2iotype(iotype); + m_checkpoint_file_specs.iotype = scorpio::str2iotype(iotype); } /*===============================================================================================*/ diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 9c193d8ae4ad..ead1aa0fbd65 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -118,7 +118,7 @@ int get_dimlen(const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -154,7 +154,7 @@ bool has_dim (const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -181,7 +181,7 @@ bool has_variable (const std::string& filename, const std::string& varname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -212,7 +212,7 @@ bool has_attribute (const std::string& filename, const std::string& varname, con bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } // Get file id @@ -513,7 +513,7 @@ ekat::any get_any_attribute (const std::string& filename, const std::string& att } /* ----------------------------------------------------------------- */ ekat::any get_any_attribute (const std::string& filename, const std::string& var_name, const std::string& att_name) { - register_file(filename,Read,0); + register_file(filename,Read); auto ncid = get_file_ncid_c2f (filename.c_str()); EKAT_REQUIRE_MSG (ncid>=0, "[get_any_attribute] Error! Could not retrieve file ncid.\n" diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index acaf587d4470..c5ae69ea6c68 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -24,6 +24,51 @@ namespace scorpio { Append = 2, Write = 4 }; + + // I/O types supported + enum IOType { + // Default I/O type is used to let the code choose I/O type as needed (via CIME) + DefaultIOType = 0, + NetCDF, + PnetCDF, + Adios, + Hdf5 + }; + + inline int str2iotype(const std::string &str) + { + if(str == "default"){ + return static_cast(IOType::DefaultIOType); + } + else if(str == "netcdf"){ + return static_cast(IOType::NetCDF); + } + else if(str == "pnetcdf"){ + return static_cast(IOType::PnetCDF); + } + else if(str == "adios"){ + return static_cast(IOType::Adios); + } + else if(str == "hdf5"){ + return static_cast(IOType::Hdf5); + } + else{ + return static_cast(IOType::DefaultIOType); + } + } + + inline std::string iotype2str(int iotype) + { + switch(iotype){ + case static_cast(IOType::DefaultIOType): return "default"; + case static_cast(IOType::NetCDF): return "netcdf"; + case static_cast(IOType::PnetCDF): return "pnetcdf"; + case static_cast(IOType::Adios): return "adios"; + case static_cast(IOType::Hdf5): return "hdf5"; + default: return "default"; + } + } + /* All scorpio usage requires that the pio_subsystem is initialized. Happens only once per simulation */ void eam_init_pio_subsystem(const ekat::Comm& comm); void eam_init_pio_subsystem(const int mpicom, const int atm_id = 0); @@ -33,7 +78,7 @@ namespace scorpio { void eam_pio_closefile(const std::string& filename); void eam_flush_file(const std::string& filename); /* Register a new file to be used for input/output with the scorpio module */ - void register_file(const std::string& filename, const FileMode mode, int iotype); + void register_file(const std::string& filename, const FileMode mode, int iotype = IOType::DefaultIOType); /* Sets the IO decompostion for all variables in a particular filename. Required after all variables have been registered. Called once per file. */ int get_dimlen(const std::string& filename, const std::string& dimname); bool has_dim(const std::string& filename, const std::string& dimname); diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index 2d8c0aa2aef4..bfed937636e2 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -106,7 +106,7 @@ TEST_CASE("io_remap_test","io_remap_test") // Write remap data to file const std::string remap_filename = "remap_weights_np"+std::to_string(io_comm.size())+".nc"; - scorpio::register_file(remap_filename, scorpio::FileMode::Write,0); + scorpio::register_file(remap_filename, scorpio::FileMode::Write); scorpio::register_dimension(remap_filename,"n_a", "n_a", ncols_src, true); scorpio::register_dimension(remap_filename,"n_b", "n_b", ncols_tgt, true); diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index f7951a484454..13832c490441 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -235,7 +235,7 @@ void create_remap_file(const std::string& filename, const int ngdofs_tgt) const int ngdofs_src = ngdofs_tgt + 1; const int nnz = 2*ngdofs_tgt; - scorpio::register_file(filename, scorpio::FileMode::Write,0); + scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::register_dimension(filename,"n_a", "n_a", ngdofs_src, true); scorpio::register_dimension(filename,"n_b", "n_b", ngdofs_tgt, true); diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp index 0fc4d05a8b80..00ee58ef503c 100644 --- a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -122,7 +122,7 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { // Existing dofs are "copied", added dofs are averaged from neighbors const int nnz = ngdofs_src + 2*(ngdofs_src-1); - scorpio::register_file(filename, scorpio::FileMode::Write,0); + scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index db05558aa54a..6ec4a40c1236 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -97,7 +97,7 @@ Real data_func(const int col, const int vec, const Real pres) { void create_remap_file(const std::string& filename, const int nlevs, const std::vector& dofs_p, const std::vector& p_tgt) { - scorpio::register_file(filename, scorpio::FileMode::Write,0); + scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::register_dimension(filename,"lev","lev",nlevs, false); diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index c8cdccbc9148..3515bcdaac91 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -273,7 +273,7 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { if (ii==0) { ts_ref = ts_file_start; } - scorpio::register_file(filename,scorpio::Read,0); + scorpio::register_file(filename,scorpio::Read); const int ntime = scorpio::get_dimlen(filename,"time"); for (int tt=0; tt Date: Mon, 11 Mar 2024 21:19:38 -0500 Subject: [PATCH 008/476] Disable BP to NetCDF conversion Disable BP to NetCDF conversion by explicitly commenting it out from the workflow --- cime_config/machines/config_workflow.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_workflow.xml b/cime_config/machines/config_workflow.xml index 5dd81437a35d..b2eae349aff0 100644 --- a/cime_config/machines/config_workflow.xml +++ b/cime_config/machines/config_workflow.xml @@ -37,7 +37,7 @@ $BUILD_COMPLETE and $TEST - + - (case.run and case.post_run_io) or case.test + + case.run or case.test $DOUT_S 1 From 67aff454baed7280a7c05be26738a0ec456ea63a Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 19 Mar 2024 19:44:06 -0500 Subject: [PATCH 009/476] Add SCORPIO and GPTL includes Explicitly add SCORPIO and GPTL includes for EAMXX standalone builds. Note: Older versions (<= 1.4.*) of SCORPIO exported these internal paths by default. Newer versions of SCORPIO do not expose these paths, so standalone builds of EAMXX need to explicitly add them. --- components/eamxx/cmake/tpls/Scorpio.cmake | 11 +++++++++++ components/eamxx/src/share/CMakeLists.txt | 5 +++++ components/eamxx/src/share/io/CMakeLists.txt | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/components/eamxx/cmake/tpls/Scorpio.cmake b/components/eamxx/cmake/tpls/Scorpio.cmake index db1746298413..b75742fe1196 100644 --- a/components/eamxx/cmake/tpls/Scorpio.cmake +++ b/components/eamxx/cmake/tpls/Scorpio.cmake @@ -42,6 +42,17 @@ macro (CreateScorpioTargets) option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON) add_subdirectory (${E3SM_EXTERNALS_DIR}/scorpio ${CMAKE_BINARY_DIR}/externals/scorpio) + + set (SCORPIO_Fortran_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/externals/scorpio/src/flib CACHE INTERNAL "SCORPIO Fortran include dirs") + set (C_INCLUDE_DIRS "${E3SM_EXTERNALS_DIR}/scorpio/src/clib") + list(APPEND C_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/externals/scorpio/src/clib") + set (SCORPIO_C_INCLUDE_DIRS "${C_INCLUDE_DIRS}" CACHE INTERNAL "SCORPIO C include dirs") + + # Add GPTL from SCORPIO + if (NOT GPTL_PATH) + set (GPTL_PATH ${E3SM_EXTERNALS_DIR}/scorpio/src/gptl CACHE INTERNAL "Path to GPTL library") + endif () + EkatDisableAllWarning(pioc) EkatDisableAllWarning(piof) EkatDisableAllWarning(gptl) diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index 418d146d050a..701cf89bb03f 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -57,6 +57,11 @@ target_include_directories(scream_share PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/modules ) + +if (GPTL_PATH) + target_include_directories(scream_share PUBLIC ${GPTL_PATH}) +endif () + target_link_libraries(scream_share PUBLIC ekat pioc) target_compile_options(scream_share PUBLIC $<$:${SCREAM_Fortran_FLAGS}> diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index 7f9b64a829c3..8866e2944956 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -19,6 +19,14 @@ if (DEFINED ENV{ADIOS2_ROOT}) target_include_directories(scream_io PRIVATE $ENV{ADIOS2_ROOT}/include) endif () +if (SCORPIO_Fortran_INCLUDE_DIRS) + target_include_directories(scream_io PUBLIC ${SCORPIO_Fortran_INCLUDE_DIRS}) +endif () + +if (SCORPIO_C_INCLUDE_DIRS) + target_include_directories(scream_io PUBLIC ${SCORPIO_C_INCLUDE_DIRS}) +endif () + target_link_libraries(scream_io PUBLIC scream_share piof pioc) if (SCREAM_CIME_BUILD) From fa6c6e8794e383d917bf8ea0fceeec40ddcaeae8 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 21 Mar 2024 09:29:44 -0500 Subject: [PATCH 010/476] Updating SCORPIO to v1.6.2 alpha SCORPIO v1.6.2 includes the following fixes, * Support for standalone EAMXX/SCREAM builds * Fixes for ultra high res simulations using the SUBSET rearranger --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index 16e01694a011..5f2cd4b672bf 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 16e01694a011b50bd5ff68277aa2de204dfad36d +Subproject commit 5f2cd4b672bf96e09d819712e17003de3a505e9c From 6b407d0cb6cada94d2a004adbf21ce5a3baeaa97 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 27 Mar 2024 18:57:29 -0500 Subject: [PATCH 011/476] Ensure PIO calls are not optimized out At least on some machines (mappy + gcc 9.2.0) if the return value (ierr) from the PIO calls is ignored the PIO calls are optimized out (values/attributes not written out to the file etc). So adding a call to handle/touch the return value from the PIO calls to ensure that values/attributes are updated in the output file Thanks lbertag@sandia.gov for the fix. --- .../eamxx/src/share/io/scream_scorpio_interface.F90 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 4bb9c7efa543..bfd9ad68f2db 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -446,9 +446,11 @@ subroutine register_variable(filename,shortname,longname,units, & if (is_write(pio_atm_file%purpose)) then ierr = PIO_def_var(pio_atm_file%pioFileDesc, trim(shortname), hist_var%nc_dtype, hist_var%dimid(:numdims), hist_var%piovar) - call errorHandle("PIO ERROR: could not define variable "//trim(shortname),ierr) + call errorHandle("PIO ERROR: could not define variable "//trim(shortname)//" in file "//trim(filename),ierr) ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'units', hist_var%units ) + call errorHandle("PIO ERROR: could not set attribute 'units' for variable "//trim(shortname)//" in file "//trim(filename),ierr) ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'long_name', hist_var%long_name ) + call errorHandle("PIO ERROR: could not set attribute 'long_name' for variable "//trim(shortname)//" in file "//trim(filename),ierr) else ierr = PIO_inq_varid(pio_atm_file%pioFileDesc,trim(shortname),hist_var%piovar) call errorHandle("PIO ERROR: could not retrieve id for variable "//trim(shortname)//" from file "//trim(filename),ierr) @@ -778,7 +780,10 @@ subroutine eam_update_time(filename,time) pio_atm_file%numRecs = pio_atm_file%numRecs + 1 call get_var(pio_atm_file,'time',var) ! Only update time on the file if a valid time is provided - if (time>=0) ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,(/ pio_atm_file%numRecs /), (/ 1 /), (/ time /)) + if (time>=0) then + ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,(/ pio_atm_file%numRecs /), (/ 1 /), (/ time /)) + call errorHandle("PIO ERROR: something went wrong while writing time var on file="//trim(filename),ierr) + endif end subroutine eam_update_time !=====================================================================! ! Assign institutions to header metadata for a specific pio output file. @@ -1150,6 +1155,7 @@ subroutine eam_pio_finalize() #if !defined(SCREAM_CIME_BUILD) call PIO_finalize(pio_subsystem, ierr) + call errorHandle("PIO ERROR: something went wrong when calling PIO_finalize.",ierr) nullify(pio_subsystem) #endif @@ -1491,6 +1497,7 @@ function read_time_at_index(filename,time_index) result(val) ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim("time"),dim_id) call errorHandle("read_time_at_index ERROR: dimension 'time' not found in file "//trim(filename)//".",ierr) ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,dim_id,time_len) + call errorHandle("PIO Error! Something went wrong when inquiring time dim length on file file "//trim(filename)//".",ierr) if (present(time_index)) then timeidx = time_index From d132f13e7ac0e695f3586e0d1a5cdc9dd981ec2c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 11 Apr 2024 15:54:06 -0600 Subject: [PATCH 012/476] Update EKAT submodule Brings in new multi-sliced subfields capability. --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index 69a7b84e3963..0811e6cad49b 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 69a7b84e39639aacb63f45861033d698b1cf085a +Subproject commit 0811e6cad49b0f10d6c343bb6fed97e6e98fb794 From 0da40f5bfebaa47662e58a83e00086467886cce8 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 12 Apr 2024 15:14:00 -0600 Subject: [PATCH 013/476] Fixes to get valgrind working on mappy --- .../eamxx/scripts/jenkins/valgrind/mappy.supp | 313 +++++++++++++++++- components/eamxx/scripts/machines_specs.py | 5 +- 2 files changed, 304 insertions(+), 14 deletions(-) diff --git a/components/eamxx/scripts/jenkins/valgrind/mappy.supp b/components/eamxx/scripts/jenkins/valgrind/mappy.supp index 67d6ec76114c..b8d02e9906c3 100644 --- a/components/eamxx/scripts/jenkins/valgrind/mappy.supp +++ b/components/eamxx/scripts/jenkins/valgrind/mappy.supp @@ -75,6 +75,30 @@ fun:PMPI_Init fun:main } +{ + + Memcheck:Cond + fun:strlen + fun:PMI_KVS_Put + fun:kvs_put + fun:opal_pmix_base_commit_packed + fun:s1_commit + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} +{ + + Memcheck:Cond + fun:__strncpy_sse2_unaligned + obj:/usr/lib64/libpmi.so.0.0.0 + fun:kvs_put + fun:opal_pmix_base_commit_packed + fun:s1_commit + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} { Memcheck:Cond @@ -95,6 +119,238 @@ fun:PMPI_Init fun:main } +{ + + Memcheck:Cond + fun:strlen + obj:/usr/lib64/libslurm_pmi-20.11.9.so + obj:/usr/lib64/libslurm_pmi-20.11.9.so + fun:slurm_send_node_msg + fun:slurm_send_recv_msg + fun:slurm_send_recv_rc_msg_only_one + fun:slurm_pmi_send_kvs_comm_set + fun:PMI_KVS_Commit + fun:s1_commit + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} +{ + + Memcheck:Param + socketcall.sendto(msg) + fun:send + fun:slurm_send_timeout + fun:slurm_msg_sendto_timeout + fun:slurm_send_node_msg + fun:slurm_send_recv_msg + fun:slurm_send_recv_rc_msg_only_one + fun:slurm_pmi_send_kvs_comm_set + fun:PMI_KVS_Commit + fun:s1_commit + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} +{ + + Memcheck:Cond + fun:strlen + fun:PMI_KVS_Get + fun:kvs_get + fun:opal_pmix_base_get_packed + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:__strncpy_sse2_unaligned + fun:PMI_KVS_Get + fun:kvs_get + fun:opal_pmix_base_get_packed + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:strlen + fun:opal_pmix_base_get_packed + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:opal_pmix_base_get_packed + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:strlen + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:__GI_strlen + fun:strdup + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:__strcmp_sse42 + fun:opal_pmix_base_store + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:opal_pmix_base_store + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:__GI_strlen + fun:strdup + fun:opal_dss_copy_value + fun:opal_pmix_base_store + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:__strcmp_sse42 + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:pmi_encode + fun:opal_pmix_base_partial_commit_packed + fun:s1_put + fun:mca_btl_base_vader_modex_send + fun:mca_btl_vader_component_init + fun:mca_btl_base_select + fun:mca_bml_r2_component_init + fun:mca_bml_base_init + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} +{ + + Memcheck:Cond + fun:strlen + fun:opal_pmix_base_partial_commit_packed + fun:s1_put + fun:mca_btl_base_vader_modex_send + fun:mca_btl_vader_component_init + fun:mca_btl_base_select + fun:mca_bml_r2_component_init + fun:mca_bml_base_init + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} +{ + + Memcheck:Cond + fun:strlen + fun:PMI_KVS_Put + fun:kvs_put + fun:opal_pmix_base_partial_commit_packed + fun:s1_put + fun:mca_btl_base_vader_modex_send + fun:mca_btl_vader_component_init + fun:mca_btl_base_select + fun:mca_bml_r2_component_init + fun:mca_bml_base_init + fun:ompi_mpi_init + fun:PMPI_Init +} +{ + + Memcheck:Cond + fun:__strncpy_sse2_unaligned + obj:/usr/lib64/libpmi.so.0.0.0 + fun:kvs_put + fun:opal_pmix_base_partial_commit_packed + fun:s1_put + fun:mca_btl_base_vader_modex_send + fun:mca_btl_vader_component_init + fun:mca_btl_base_select + fun:mca_bml_r2_component_init + fun:mca_bml_base_init + fun:ompi_mpi_init + fun:PMPI_Init +} +{ + + Memcheck:Addr1 + fun:strlen + fun:opal_pmix_base_partial_commit_packed + fun:s1_put + fun:mca_btl_base_vader_modex_send + fun:mca_btl_vader_component_init + fun:mca_btl_base_select + fun:mca_bml_r2_component_init + fun:mca_bml_base_init + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} { Memcheck:Cond @@ -119,6 +375,26 @@ fun:PMPI_Init fun:main } +{ + + Memcheck:Cond + fun:pmi_encode + fun:opal_pmix_base_commit_packed + fun:s1_commit + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} +{ + + Memcheck:Cond + fun:strlen + fun:opal_pmix_base_commit_packed + fun:s1_commit + fun:ompi_mpi_init + fun:PMPI_Init + fun:main +} { Memcheck:Cond @@ -183,6 +459,7 @@ { Memcheck:Cond + fun:strlen fun:opal_pmix_base_get_packed fun:opal_pmix_base_cache_keys_locally fun:fencenb @@ -194,7 +471,6 @@ { Memcheck:Cond - fun:malloc fun:opal_pmix_base_get_packed fun:opal_pmix_base_cache_keys_locally fun:fencenb @@ -206,9 +482,7 @@ { Memcheck:Cond - fun:is_overlap - fun:memcpy@@GLIBC_2.14 - fun:opal_pmix_base_get_packed + fun:strlen fun:opal_pmix_base_cache_keys_locally fun:fencenb fun:opal_libevent2022_event_base_loop @@ -219,8 +493,8 @@ { Memcheck:Cond - fun:memcpy@@GLIBC_2.14 - fun:opal_pmix_base_get_packed + fun:__GI_strlen + fun:strdup fun:opal_pmix_base_cache_keys_locally fun:fencenb fun:opal_libevent2022_event_base_loop @@ -230,8 +504,9 @@ } { - Memcheck:Value8 - fun:opal_pmix_base_get_packed + Memcheck:Cond + fun:__strcmp_sse42 + fun:opal_pmix_base_store fun:opal_pmix_base_cache_keys_locally fun:fencenb fun:opal_libevent2022_event_base_loop @@ -242,8 +517,7 @@ { Memcheck:Cond - fun:realloc - fun:opal_pmix_base_get_packed + fun:opal_pmix_base_store fun:opal_pmix_base_cache_keys_locally fun:fencenb fun:opal_libevent2022_event_base_loop @@ -253,9 +527,22 @@ } { - Memcheck:Value8 - fun:memcpy@@GLIBC_2.14 - fun:opal_pmix_base_get_packed + Memcheck:Cond + fun:__GI_strlen + fun:strdup + fun:opal_dss_copy_value + fun:opal_pmix_base_store + fun:opal_pmix_base_cache_keys_locally + fun:fencenb + fun:opal_libevent2022_event_base_loop + fun:progress_engine + fun:start_thread + fun:clone +} +{ + + Memcheck:Cond + fun:__strcmp_sse42 fun:opal_pmix_base_cache_keys_locally fun:fencenb fun:opal_libevent2022_event_base_loop diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index df89ae7b5add..8070dacaa55c 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -28,7 +28,10 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 acme-gcc/11.2.0 sems-archive-git/2.10.1 acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme", "export GATOR_INITIAL_MB=4000MB"], + "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 acme-gcc/11.2.0 sems-archive-git/2.10.1 acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme", + "export GATOR_INITIAL_MB=4000MB", + "export PATH=/ascldap/users/jgfouca/packages/valgrind-3.22.0/bin:$PATH", + ], ["mpicxx","mpifort","mpicc"], "", "/sems-data-store/ACME/baselines/scream/master-baselines"), From ac6bcec685b9892b4737f17d3f5604bed3ec69cf Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 09:10:18 -0600 Subject: [PATCH 014/476] Check relative lat/lon error scorpio reads in floating precision, yaml reads in double precision, so using == might fail. --- .../src/control/intensive_observation_period.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index d925b21bfaa3..6caf48ff542c 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -307,10 +307,14 @@ initialize_iop_file(const util::TimeStamp& run_t0, Real iop_file_lat, iop_file_lon; read_variable_from_file(iop_file, "lat", "real", {"lat"}, -1, &iop_file_lat); read_variable_from_file(iop_file, "lon", "real", {"lon"}, -1, &iop_file_lon); - EKAT_REQUIRE_MSG(iop_file_lat == m_params.get("target_latitude"), - "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); - EKAT_REQUIRE_MSG(std::fmod(iop_file_lon + 360, 360) == m_params.get("target_longitude"), - "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); + const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ + m_params.get("target_latitude"); + const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ + m_params.get("target_longitude"); + EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), + "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); + EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), + "Error! IOP file variable \"lon\" does not match target_longitude from IOP parameters.\n"); // Store iop file pressure as helper field with dimension lev+1. // Load the first lev entries from iop file, the lev+1 entry will From 850ae9ade6a86186a6316aa9a86aaa72345ec2d1 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 09:15:55 -0600 Subject: [PATCH 015/476] Sync IOP file pres field to host IOP file pressure field will have been updated on device before calculating the surface value for IOP fields (if applicable) --- components/eamxx/src/control/intensive_observation_period.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 6caf48ff542c..90a985aa3618 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -697,6 +697,7 @@ read_iop_file_data (const util::TimeStamp& current_ts) const auto dx = iop_file_v_h(adjusted_file_levs-2) - iop_file_v_h(adjusted_file_levs-3); if (dx == 0) iop_file_v_h(adjusted_file_levs-1) = iop_file_v_h(adjusted_file_levs-2); else { + iop_file_pressure.sync_to_host(); const auto iop_file_pres_v_h = iop_file_pressure.get_view(); const auto dy = iop_file_pres_v_h(adjusted_file_levs-2) - iop_file_pres_v_h(adjusted_file_levs-3); const auto scale = dy/dx; From d7f5bcb0697d14857461a42d9eecc063623fb9d0 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 09:29:35 -0600 Subject: [PATCH 016/476] Use EKAT's new Pack::update() function --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 09e2c7a17b2d..a13922d610d8 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -138,26 +138,23 @@ advance_iop_subsidence(const KT::MemberType& team, const auto fac = (dt/2)/pdel(k); // Update u - auto& u_k = u(k); - u_k.set(at_top, u_k - fac*omega_int_kp1*delta_u_k); - u_k.set(at_bot, u_k - fac*omega_int_k*delta_u_km1); - u_k.set(at_mid, u_k - fac*(omega_int_kp1*delta_u_k + omega_int_k*delta_u_km1)); + u(k).update(at_top, fac*omega_int_kp1*delta_u_k, -1, 1); + u(k).update(at_bot, fac*omega_int_k*delta_u_km1, -1, 1); + u(k).update(at_mid, fac*(omega_int_kp1*delta_u_k + omega_int_k*delta_u_km1), -1, 1); // Update v - auto& v_k = v(k); - v_k.set(at_top, v_k - fac*omega_int_kp1*delta_v_k); - v_k.set(at_bot, v_k - fac*omega_int_k*delta_v_km1); - v_k.set(at_mid, v_k - fac*(omega_int_kp1*delta_v_k + omega_int_k*delta_v_km1)); + v(k).update(at_top, fac*omega_int_kp1*delta_v_k, -1, 1); + v(k).update(at_bot, fac*omega_int_k*delta_v_km1, -1, 1); + v(k).update(at_mid, fac*(omega_int_kp1*delta_v_k + omega_int_k*delta_v_km1), -1, 1); // Before updating T, first scale using thermal // expansion term due to LS vertical advection - auto& T_k = T(k); - T_k *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); + T(k) *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); // Update T - T_k.set(at_top, T_k - fac*omega_int_kp1*delta_T_k); - T_k.set(at_bot, T_k - fac*omega_int_k*delta_T_km1); - T_k.set(at_mid, T_k - fac*(omega_int_kp1*delta_T_k + omega_int_k*delta_T_km1)); + T(k).update(at_top, fac*omega_int_kp1*delta_T_k, -1, 1); + T(k).update(at_bot, fac*omega_int_k*delta_T_km1, -1, 1); + T(k).update(at_mid, fac*(omega_int_kp1*delta_T_k + omega_int_k*delta_T_km1), -1, 1); // Update Q Pack delta_tracer_k, delta_tracer_km1; @@ -167,10 +164,9 @@ advance_iop_subsidence(const KT::MemberType& team, if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); - auto& Q_k = Q(iq, k); - Q_k.set(at_top, Q_k - fac*omega_int_kp1*delta_tracer_k); - Q_k.set(at_bot, Q_k - fac*omega_int_k*delta_tracer_km1); - Q_k.set(at_mid, Q_k - fac*(omega_int_kp1*delta_tracer_k + omega_int_k*delta_tracer_km1)); + Q(iq, k).update(at_top, fac*omega_int_kp1*delta_tracer_k, -1, 1); + Q(iq, k).update(at_bot, fac*omega_int_k*delta_tracer_km1, -1, 1); + Q(iq, k).update(at_mid, fac*(omega_int_kp1*delta_tracer_k + omega_int_k*delta_tracer_km1), -1, 1); } }); @@ -192,8 +188,8 @@ advance_iop_forcing(const KT::MemberType& team, { const auto nlev_packs = ekat::npack(nlevs); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { - T(k) += dt*divT(k); - qv(k) += dt*divq(k); + T(k).update(divT(k), dt, 1.0); + qv(k).update(divq(k), dt, 1.0); }); } From 3ae438cd8dba6ddd40c131fee401522cf6eb84ed Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 10:01:13 -0600 Subject: [PATCH 017/476] Port iop_apply_coriolis --- .../control/intensive_observation_period.cpp | 25 ++++++--- .../src/dynamics/homme/eamxx_homme_iop.cpp | 54 ++++++++++++++++--- .../homme/eamxx_homme_process_interface.hpp | 11 ++++ .../src/physics/share/physics_constants.hpp | 1 + 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 90a985aa3618..5d617fcd4ced 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -238,7 +238,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, setup_iop_field({"Q2"}, fl_vector); setup_iop_field({"omega"}, fl_vector, "Ptend"); - // Require Ps, T, q, divT, divq are all defined in the iop file + // Certain fields are required from the iop file EKAT_REQUIRE_MSG(has_iop_field("Ps"), "Error! IOP file required to contain variable \"Ps\".\n"); EKAT_REQUIRE_MSG(has_iop_field("T"), @@ -250,6 +250,19 @@ initialize_iop_file(const util::TimeStamp& run_t0, EKAT_REQUIRE_MSG(has_iop_field("divq"), "Error! IOP file required to contain variable \"divq\".\n"); + // Check for large scale winds and enfore "all-or-nothing" for u and v component + const bool both_ls = (has_iop_field("u_ls") and has_iop_field("v_ls")); + const bool neither_ls = (not (has_iop_field("u_ls") or has_iop_field("v_ls"))); + EKAT_REQUIRE_MSG(both_ls or neither_ls, + "Error! Either u_ls and v_ls both defined in IOP file, or neither.\n"); + m_params.set("use_large_scale_wind", both_ls); + + // Require large scale winds if using Coriolis forcing + if (m_params.get("iop_coriolis")) { + EKAT_REQUIRE_MSG(m_params.get("use_large_scale_wind"), + "Error! Large scale winds required for coriolis forcing.\n"); + } + // If we have the vertical component of T/Q forcing, define 3d forcing as a computed field. if (has_iop_field("vertdivT")) { FieldIdentifier fid("divT3d", fl_vector, ekat::units::Units::nondimensional(), ""); @@ -268,12 +281,12 @@ initialize_iop_file(const util::TimeStamp& run_t0, m_iop_field_type.insert({"divq3d", IOPFieldType::Computed}); } - // Enforce that 3D forcing is all-or-nothing. - const bool both = (has_iop_field("divT3d") and has_iop_field("divq3d")); - const bool neither = (not (has_iop_field("divT3d") or has_iop_field("divq3d"))); - EKAT_REQUIRE_MSG(both or neither, + // Enforce that 3D forcing is all-or-nothing for T and q. + const bool both_3d_forcing = (has_iop_field("divT3d") and has_iop_field("divq3d")); + const bool neither_3d_forcing = (not (has_iop_field("divT3d") or has_iop_field("divq3d"))); + EKAT_REQUIRE_MSG(both_3d_forcing or neither_3d_forcing, "Error! Either T and q both have 3d forcing, or neither have 3d forcing.\n"); - m_params.set("use_3d_forcing", both); + m_params.set("use_3d_forcing", both_3d_forcing); // Initialize time information int bdate; diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index a13922d610d8..d184825254d2 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -193,6 +193,34 @@ advance_iop_forcing(const KT::MemberType& team, }); } +// Provide coriolis forcing to u and v winds, using large scale winds specified in IOP forcing file. +KOKKOS_FUNCTION +void HommeDynamics:: +iop_apply_coriolis(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real lat, + const view_1d& u_ls, + const view_1d& v_ls, + const view_1d& u, + const view_1d& v) +{ + using C = physics::Constants; + constexpr Real pi = C::Pi; + constexpr Real earth_rotation = C::omega; + + // Compute coriolis force + const auto fcor = 2*earth_rotation*std::sin(lat*pi/180); + + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + const auto u_cor = v(k) - v_ls(k); + const auto v_cor = u(k) - u_ls(k); + u(k).update(u_cor, dt*fcor, 1.0); + v(k).update(v_cor, -dt*fcor, 1.0); + }); +} + void HommeDynamics:: apply_iop_forcing(const Real dt) { @@ -256,14 +284,23 @@ apply_iop_forcing(const Real dt) // Load data from IOP files, if necessary m_iop->read_iop_file_data(timestamp()); - // Define local IOP param values and views + // Define local IOP param values const auto iop_dosubsidence = m_iop->get_params().get("iop_dosubsidence"); + const auto iop_coriolis = m_iop->get_params().get("iop_coriolis"); const auto use_3d_forcing = m_iop->get_params().get("use_3d_forcing"); - const auto omega = m_iop->get_iop_field("omega").get_view(); - const auto divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() - : m_iop->get_iop_field("divT").get_view(); - const auto divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() - : m_iop->get_iop_field("divq").get_view(); + const auto lat = m_iop->get_params().get("target_latitude"); + + // Define local IOP field views + view_1d omega, divT, divq, u_ls, v_ls; + if (iop_dosubsidence) omega = m_iop->get_iop_field("omega").get_view(); + divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() + : m_iop->get_iop_field("divT").get_view(); + divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() + : m_iop->get_iop_field("divq").get_view(); + if (iop_coriolis) { + u_ls = m_iop->get_iop_field("u_ls").get_view(); + v_ls = m_iop->get_iop_field("v_ls").get_view(); + } // Team policy and workspace manager for both homme and scream // related loops. We need separate policies since hommexx functions used here @@ -382,6 +419,11 @@ apply_iop_forcing(const Real dt) // Update T and qv according to large scale forcing as specified in IOP file. advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); + if (iop_coriolis) { + // Apply coriolis forcing to u and v winds + iop_apply_coriolis(team, total_levels, dt, lat, u_ls, v_ls, u_i, v_i); + } + // Release WS views ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index 2749d268c9a7..44b16153e839 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -135,6 +135,17 @@ class HommeDynamics : public AtmosphereProcess const view_1d& T, const view_1d& qv); + + KOKKOS_FUNCTION + static void iop_apply_coriolis(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real lat, + const view_1d& u_ls, + const view_1d& v_ls, + const view_1d& u, + const view_1d& v); + public: // Fast boolean function returning whether Physics PGN is being used. bool fv_phys_active() const; diff --git a/components/eamxx/src/physics/share/physics_constants.hpp b/components/eamxx/src/physics/share/physics_constants.hpp index fd78dff8c86a..9c6e8c339478 100644 --- a/components/eamxx/src/physics/share/physics_constants.hpp +++ b/components/eamxx/src/physics/share/physics_constants.hpp @@ -95,6 +95,7 @@ struct Constants static constexpr Scalar basetemp = 300.0; static constexpr Scalar r_earth = 6.376e6; // Radius of the earth in m static constexpr Scalar stebol = 5.67e-8; // Stefan-Boltzmann's constant (W/m^2/K^4) + static constexpr Scalar omega = 7.292e-5; // Earth's rotation (rad/sec) // Table dimension constants static constexpr int VTABLE_DIM0 = 300; From ca62824af816644d46b837b34c58b5301d249eff Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 10:07:23 -0600 Subject: [PATCH 018/476] EAM: move apply_iop_coriolis() out of nelemd loop apply_iop_coriolis() contains a loop over elements, in the current state coriolis forces were being applied multiple times per DoF, --- components/eam/src/dynamics/se/se_iop_intr_mod.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 b/components/eam/src/dynamics/se/se_iop_intr_mod.F90 index 02862980053c..0798a0a79f54 100644 --- a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 +++ b/components/eam/src/dynamics/se/se_iop_intr_mod.F90 @@ -423,10 +423,6 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete call outfld('QDIFF',qdiff_dyn,plon,begchunk) endif - if (iop_coriolis) then - call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt) - endif - call outfld('TOBS',tobs,plon,begchunk) call outfld('QOBS',qobs,plon,begchunk) call outfld('DIVQ',divq,plon,begchunk) @@ -442,6 +438,10 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete enddo + if (iop_coriolis) then + call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt) + endif + if ((iop_nudge_tq .or. iop_nudge_uv) .and. dp_crm) then ! If running in a doubly periodic CRM mode, then nudge the domain ! based on the domain mean and observed quantities of T, Q, u, and v From c9d1775f4a443a21587f6002c4be19188075711a Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 10:45:15 -0600 Subject: [PATCH 019/476] Define lambda for updating homme states for IOP functions Porting domain relaxation will need this exact code, so consolidaing into a lambda to avoid code duplication --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 139 +++++++++--------- 1 file changed, 67 insertions(+), 72 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index d184825254d2..f27b97f383e3 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -246,6 +246,7 @@ apply_iop_forcing(const Real dt) const auto nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const auto total_levels = m_dyn_grid->get_num_vertical_levels(); const auto qsize = params.qsize; + const auto theta_hydrostatic_mode = params.theta_hydrostatic_mode; // Sanity checks since we will be switching between ekat::Pack // and Homme::Scalar view types @@ -316,66 +317,68 @@ apply_iop_forcing(const Real dt) // TODO: Create a memory buffer for this class // and add the below WSM and views WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 32, policy_homme); + WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); view_Nd - temperature("temperature", nelem, NGP, NGP, NLEV), - exner("exner", nelem, NGP, NGP, NLEV); - - // Preprocess some homme states to get temperature and exner - Kokkos::parallel_for("compute_t_and_exner", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - auto rstar_slot = ws.take_macro_block("rstar", NGP*NGP); - uview_2d - pnh (reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV), - rstar(reinterpret_cast(rstar_slot.data()), NGP*NGP, NLEV); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, idx); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - - // Compute exner from EOS - if (params.theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } + rstar ("rstar", nelem, NGP, NGP, NLEV), + exner ("exner", nelem, NGP, NGP, NLEV), + temperature("temperature", nelem, NGP, NGP, NLEV); + + // Lambda for computing rstar, exner, and temperature from Hommexx + auto compute_homme_states = [&] () { + Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); + + // Get temp views from workspace + auto ws = homme_wsm.get_workspace(team); + auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); + uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto pnh_i = ekat::subview(pnh, idx); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); + Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); + Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); + Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); + + // Compute exner from EOS + if (theta_hydrostatic_mode) { + auto hydro_p_int = ws.take("hydro_p_int"); + Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); + elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); + eos.compute_exner(kv, pnh_scalar, exner_scalar); + ws.release(hydro_p_int); + } else { + eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); + } + + // Get the temperature from dynamics states + elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + }); - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + // Release WS views + ws.release_macro_block(pnh_slot, NGP*NGP); }); + }; - // Release WS views - ws.release_macro_block(rstar_slot, NGP*NGP); - ws.release_macro_block(pnh_slot, NGP*NGP); - }); + // Preprocess some homme states to get temperature and exner + compute_homme_states(); Kokkos::fence(); // Apply IOP forcing @@ -434,22 +437,16 @@ apply_iop_forcing(const Real dt) KV kv(team); const int ie = team.league_rank(); - // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto rstar_slot = ws.take_macro_block("rstar", NGP*NGP); - uview_2d - rstar(reinterpret_cast(rstar_slot.data()), NGP*NGP, NLEV); - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { const int igp = idx/NGP; const int jgp = idx%NGP; - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, idx); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); auto exner_i = ekat::subview(exner, ie, igp, jgp); auto temperature_i = ekat::subview(temperature, ie, igp, jgp); @@ -466,16 +463,14 @@ apply_iop_forcing(const Real dt) // For BFB on restarts, Q needs to be updated after we compute Qdp Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); }); + team.team_barrier(); - // Convert updated temperature back to potential temperature + // Recompute rstar with updated qv, and convert updated temperature back to potential temperature elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); }); }); - - // Release WS views - ws.release_macro_block(rstar_slot, NGP*NGP); }); } From 3f7eee0053194e4ca01df1fc1442caf3af70c988 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 12:01:27 -0600 Subject: [PATCH 020/476] Port domain relaxation --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 152 +++++++++++++++++- .../homme/eamxx_homme_process_interface.hpp | 1 + 2 files changed, 151 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index f27b97f383e3..5ec447028cc8 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -288,20 +288,39 @@ apply_iop_forcing(const Real dt) // Define local IOP param values const auto iop_dosubsidence = m_iop->get_params().get("iop_dosubsidence"); const auto iop_coriolis = m_iop->get_params().get("iop_coriolis"); + const auto iop_nudge_tq = m_iop->get_params().get("iop_nudge_tq"); + const auto iop_nudge_uv = m_iop->get_params().get("iop_nudge_uv"); + const auto use_large_scale_wind = m_iop->get_params().get("use_large_scale_wind"); const auto use_3d_forcing = m_iop->get_params().get("use_3d_forcing"); const auto lat = m_iop->get_params().get("target_latitude"); + const auto iop_nudge_tscale = m_iop->get_params().get("iop_nudge_tscale"); + const auto iop_nudge_tq_low = m_iop->get_params().get("iop_nudge_tq_low"); + const auto iop_nudge_tq_high = m_iop->get_params().get("iop_nudge_tq_high"); // Define local IOP field views - view_1d omega, divT, divq, u_ls, v_ls; - if (iop_dosubsidence) omega = m_iop->get_iop_field("omega").get_view(); + const Real ps_iop = m_iop->get_iop_field("Ps").get_view()(); + view_1d omega, divT, divq, u_ls, v_ls, qv_iop, t_iop, u_iop, v_iop; divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() : m_iop->get_iop_field("divT").get_view(); divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() : m_iop->get_iop_field("divq").get_view(); + if (iop_dosubsidence) { + omega = m_iop->get_iop_field("omega").get_view(); + } if (iop_coriolis) { u_ls = m_iop->get_iop_field("u_ls").get_view(); v_ls = m_iop->get_iop_field("v_ls").get_view(); } + if (iop_nudge_tq) { + qv_iop = m_iop->get_iop_field("q").get_view(); + t_iop = m_iop->get_iop_field("T").get_view(); + } + if (iop_nudge_uv) { + u_iop = use_large_scale_wind ? m_iop->get_iop_field("u_ls").get_view() + : m_iop->get_iop_field("u").get_view(); + v_iop = use_large_scale_wind ? m_iop->get_iop_field("v_ls").get_view() + : m_iop->get_iop_field("v").get_view(); + } // Team policy and workspace manager for both homme and scream // related loops. We need separate policies since hommexx functions used here @@ -472,6 +491,135 @@ apply_iop_forcing(const Real dt) }); }); }); + + if (iop_nudge_tq or iop_nudge_uv) { + // Nudge the domain based on the domain mean + // and observed quantities of T, Q, u, and + + if (iop_nudge_tq) { + // Compute rstar, exner and temperature from Hommexx + compute_homme_states(); + Kokkos::fence(); + } + + // Compute domain mean of qv, temperature, u, and v + + // TODO: add to local mem buffer + view_1d qv_mean, t_mean, u_mean, v_mean; + if (iop_nudge_tq) { + qv_mean = view_1d("u_mean", NLEV), + t_mean = view_1d("v_mean", NLEV); + } + if (iop_nudge_uv){ + u_mean = view_1d("u_mean", NLEV), + v_mean = view_1d("v_mean", NLEV); + } + + const auto qv_mean_h = Kokkos::create_mirror_view(qv_mean); + const auto t_mean_h = Kokkos::create_mirror_view(t_mean); + const auto u_mean_h = Kokkos::create_mirror_view(u_mean); + const auto v_mean_h = Kokkos::create_mirror_view(v_mean); + + for (int k=0; kget_num_global_dofs(); + t_mean_k /= m_dyn_grid->get_num_global_dofs(); + } + if (iop_nudge_uv){ + Real& u_mean_k = u_mean_h(k/Pack::n)[k%Pack::n]; + Real& v_mean_k = v_mean_h(k/Pack::n)[k%Pack::n]; + Kokkos::parallel_reduce("compute_domain_means_uv", + nelem*NGP*NGP, + KOKKOS_LAMBDA (const int idx, Real& u_sum, Real& v_sum) { + const int ie = idx/(NGP*NGP); + const int igp = (idx/NGP)%NGP; + const int jgp = idx%NGP; + + u_sum += v_dyn(ie, 0, igp, jgp, k/Pack::n)[k%Pack::n]; + v_sum += v_dyn(ie, 1, igp, jgp, k/Pack::n)[k%Pack::n]; + }, + u_mean_k, + v_mean_k); + + m_comm.all_reduce(&u_mean_k, 1, MPI_SUM); + m_comm.all_reduce(&v_mean_k, 1, MPI_SUM); + + u_mean_k /= m_dyn_grid->get_num_global_dofs(); + v_mean_k /= m_dyn_grid->get_num_global_dofs(); + } + } + Kokkos::deep_copy(qv_mean, qv_mean_h); + Kokkos::deep_copy(t_mean, t_mean_h); + Kokkos::deep_copy(u_mean, u_mean_h); + Kokkos::deep_copy(v_mean, v_mean_h); + + // Apply relaxation + const auto rtau = std::max(dt, iop_nudge_tscale); + Kokkos::parallel_for("apply_domain_relaxation", + policy_homme, + KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + if (iop_nudge_tq) { + // Restrict nudging of T and qv to certain levels if requested by user + // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high + // is in units of [hPa], thus convert iop_nudge_tq_low/high + Mask nudge_level; + for (int p=0; p= iop_nudge_tq_high*100); + } + + qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); + temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); + + // Convert updated temperature back to potential temperature + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + } + if (iop_nudge_uv) { + u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); + v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); + } + }); + }); + }); + } } } // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index 44b16153e839..c6fa150b10b3 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -26,6 +26,7 @@ class HommeDynamics : public AtmosphereProcess // Define some types needed by class using Pack = ekat::Pack; using IntPack = ekat::Pack; + using Mask = ekat::Mask; using KT = KokkosTypes; template From 0e04a506cec7be6b438c3cdd76dd57c6b952a9df Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 16 Apr 2024 12:05:09 -0600 Subject: [PATCH 021/476] Change standalone test to CASS case - New test allows for reading multiple IOP time slices during simulation - New case expects new forcing/nudging options added --- .../homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt | 4 ++-- .../homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt index ef2494304f58..c499ccf21875 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt @@ -19,7 +19,7 @@ CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_dp "homme_shoc_cld_spa_p3_rrtmgp_dp. # Set AD configurable options set (ATM_TIME_STEP 100) SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h -set (RUN_T0 1999-07-10-00000) +set (RUN_T0 2000-07-24-61100) # Determine num subcycles needed to keep shoc dt<=300s set (SHOC_MAX_DT 300) @@ -72,7 +72,7 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta-dp.nl set (TEST_INPUT_FILES scream/init/${EAMxx_tests_IC_FILE_128lev} cam/topo/${EAMxx_tests_TOPO_FILE} - cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc + cam/scam/iop/CASS_iopfile_4scam.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml index 950ef7e4db38..387f0670784b 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml @@ -6,10 +6,13 @@ driver_options: iop_options: doubly_periodic_mode: true - iop_file: ${IOP_DATA_DIR}/DYCOMSrf01_iopfile_4scam.nc - target_latitude: 31.5 - target_longitude: 238.5 + iop_file: ${IOP_DATA_DIR}/CASS_iopfile_4scam.nc + target_latitude: 36.605 + target_longitude: 262.515 iop_dosubsidence: true + iop_coriolis: true + iop_nudge_tq: true + iop_nudge_uv: true iop_srf_prop: true time_stepping: From 56b187abbe602c9f4750a38422529ae089839038 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 28 Mar 2024 12:02:36 -0500 Subject: [PATCH 022/476] Updating to SCORPIO 1.6.2 SCORPIO to 1.6.2 includes * Support for standalone EAMXX/SCREAM builds * A new rearranger, PIO_REARR_ANY --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index 5f2cd4b672bf..dc6f1eb24f3d 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 5f2cd4b672bf96e09d819712e17003de3a505e9c +Subproject commit dc6f1eb24f3dae1dcb0ebb13130c4941dc9bd445 From b65a1900104cef715294bd2caf1503ee3e03ba90 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 28 Mar 2024 12:01:58 -0500 Subject: [PATCH 023/476] Adding support for PIO_REARR_ANY Adding support for new SCORPIO rearranger, PIO_REARR_ANY --- driver-mct/cime_config/config_component.xml | 4 ++-- driver-mct/cime_config/namelist_definition_modelio.xml | 4 ++-- share/util/shr_pio_mod.F90 | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index a76311dbfa7a..d037770ed5c5 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -2692,10 +2692,10 @@ integer - 1,2 + 1,2,3 run_pio env_run.xml - pio rearranger choice box=1, subset=2 + pio rearranger choice box=1, subset=2, any=3 $PIO_VERSION $PIO_VERSION diff --git a/driver-mct/cime_config/namelist_definition_modelio.xml b/driver-mct/cime_config/namelist_definition_modelio.xml index ce0275e59bf3..2860ffd7106f 100644 --- a/driver-mct/cime_config/namelist_definition_modelio.xml +++ b/driver-mct/cime_config/namelist_definition_modelio.xml @@ -90,9 +90,9 @@ integer pio pio_inparm - -99,1,2 + -99,1,2,3 - Rearranger method for pio 1=box, 2=subset. + Rearranger method for pio 1=box, 2=subset, 3=any. $CPL_PIO_REARRANGER diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index ed7dfdaa2ed8..0935bce351a5 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -730,7 +730,8 @@ subroutine shr_pio_namelist_set(npes,mycomm, pio_stride, pio_root, pio_numiotask if(pio_stride == 1 .and. .not. pio_async_interface) then pio_root = 0 endif - if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX) then + if(pio_rearranger .ne. PIO_REARR_SUBSET .and. pio_rearranger .ne. PIO_REARR_BOX .and.& + pio_rearranger .ne. PIO_REARR_ANY) then write(shr_log_unit,*) 'pio_rearranger value, ',pio_rearranger,& ', not supported - using PIO_REARR_BOX' pio_rearranger = PIO_REARR_BOX From 9a41cceb725b15c3d7b9b9c46dcdc9d9ed4f5c79 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 17 Apr 2024 08:44:01 -0600 Subject: [PATCH 024/476] Consolidate update() calls --- .../src/dynamics/homme/eamxx_homme_iop.cpp | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 5ec447028cc8..9e04b4a00920 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -95,8 +95,9 @@ advance_iop_subsidence(const KT::MemberType& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { auto range_pack = ekat::range(k*Pack::n); const auto at_top = range_pack==0; + const auto not_at_top = not at_top; const auto at_bot = range_pack==nlevs-1; - const auto at_mid = not at_top and not at_bot; + const auto not_at_bot = not at_bot; const bool any_at_top = at_top.any(); const bool any_at_bot = at_bot.any(); @@ -138,23 +139,20 @@ advance_iop_subsidence(const KT::MemberType& team, const auto fac = (dt/2)/pdel(k); // Update u - u(k).update(at_top, fac*omega_int_kp1*delta_u_k, -1, 1); - u(k).update(at_bot, fac*omega_int_k*delta_u_km1, -1, 1); - u(k).update(at_mid, fac*(omega_int_kp1*delta_u_k + omega_int_k*delta_u_km1), -1, 1); + u(k).update(not_at_bot, fac*omega_int_kp1*delta_u_k, -1, 1); + u(k).update(not_at_top, fac*omega_int_k*delta_u_km1, -1, 1); // Update v - v(k).update(at_top, fac*omega_int_kp1*delta_v_k, -1, 1); - v(k).update(at_bot, fac*omega_int_k*delta_v_km1, -1, 1); - v(k).update(at_mid, fac*(omega_int_kp1*delta_v_k + omega_int_k*delta_v_km1), -1, 1); + v(k).update(not_at_bot, fac*omega_int_kp1*delta_v_k, -1, 1); + v(k).update(not_at_top, fac*omega_int_k*delta_v_km1, -1, 1); // Before updating T, first scale using thermal // expansion term due to LS vertical advection T(k) *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); // Update T - T(k).update(at_top, fac*omega_int_kp1*delta_T_k, -1, 1); - T(k).update(at_bot, fac*omega_int_k*delta_T_km1, -1, 1); - T(k).update(at_mid, fac*(omega_int_kp1*delta_T_k + omega_int_k*delta_T_km1), -1, 1); + T(k).update(not_at_bot, fac*omega_int_kp1*delta_T_k, -1, 1); + T(k).update(not_at_top, fac*omega_int_k*delta_T_km1, -1, 1); // Update Q Pack delta_tracer_k, delta_tracer_km1; @@ -164,9 +162,8 @@ advance_iop_subsidence(const KT::MemberType& team, if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); - Q(iq, k).update(at_top, fac*omega_int_kp1*delta_tracer_k, -1, 1); - Q(iq, k).update(at_bot, fac*omega_int_k*delta_tracer_km1, -1, 1); - Q(iq, k).update(at_mid, fac*(omega_int_kp1*delta_tracer_k + omega_int_k*delta_tracer_km1), -1, 1); + Q(iq, k).update(not_at_bot, fac*omega_int_kp1*delta_tracer_k, -1, 1); + Q(iq, k).update(not_at_top, fac*omega_int_k*delta_tracer_km1, -1, 1); } }); From bcb6d28378db903671a4b670c495cea45e532f18 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 18 Apr 2024 09:31:01 -0600 Subject: [PATCH 025/476] EAMxx: subfield of read-only fields must be read-only --- components/eamxx/src/share/field/field.cpp | 1 + components/eamxx/src/share/field/field.hpp | 4 ++-- components/eamxx/src/share/tests/field_tests.cpp | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index c1d03f5905e4..f754adc80f0c 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -99,6 +99,7 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, Field sf; sf.m_header = create_subfield_header(sf_id,m_header,idim,index,dynamic); sf.m_data = m_data; + sf.m_is_read_only = m_is_read_only; return sf; } diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index b8445bce02bf..4a4b20525a3e 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -237,9 +237,9 @@ class Field { // to store a stride for the slowest dimension. // - If dynamic = true, it is possible to "reset" the slice index (k) at runtime. Field subfield (const std::string& sf_name, const ekat::units::Units& sf_units, - const int idim, const int index, const bool dynamic = false) const; + const int idim, const int index, const bool dynamic = false) const; Field subfield (const std::string& sf_name, const int idim, - const int index, const bool dynamic = false) const; + const int index, const bool dynamic = false) const; Field subfield (const int idim, const int k, const bool dynamic = false) const; // If this field is a vector field, get a subfield for the ith component. diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index f15f5febbc83..9897bee08aee 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -166,6 +166,9 @@ TEST_CASE("field", "") { // Trying to reshape into something that the allocation cannot accommodate should throw REQUIRE_THROWS (f1.get_view()); + + // Can't get non-const data type view from a read-only field + REQUIRE_THROWS (f1.get_const().get_view()); } SECTION ("equivalent") { @@ -299,6 +302,10 @@ TEST_CASE("field", "") { for (int k=0; k Date: Fri, 5 Apr 2024 12:15:41 -0400 Subject: [PATCH 026/476] draft adding aodvis as a diag --- .../eamxx/src/diagnostics/CMakeLists.txt | 1 + components/eamxx/src/diagnostics/aodvis.cpp | 66 +++++++++++++++++++ components/eamxx/src/diagnostics/aodvis.hpp | 37 +++++++++++ .../src/diagnostics/register_diagnostics.hpp | 2 + 4 files changed, 106 insertions(+) create mode 100644 components/eamxx/src/diagnostics/aodvis.cpp create mode 100644 components/eamxx/src/diagnostics/aodvis.hpp diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index f34d5b99638f..7acd3662afd5 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -17,6 +17,7 @@ set(DIAGNOSTIC_SRCS virtual_temperature.cpp water_path.cpp wind_speed.cpp + aodvis.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp new file mode 100644 index 000000000000..6f512a602ec2 --- /dev/null +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -0,0 +1,66 @@ +#include "diagnostics/aodvis.hpp" + +#include + +namespace scream { + +AODVis::AODVis(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + // Nothing to do here +} + +void AODVis::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + auto grid = grids_manager->get_grid("Physics"); + const auto &grid_name = grid->name(); + + const auto nondim = Units::nondimensional(); + + m_ncols = grid->get_num_local_dofs(); + m_nlevs = grid->get_num_vertical_levels(); + + // Define layouts we need (both inputs and outputs) + // TODO: define this properly (dunno state of things) + auto scalar3d_swband_layout = grid->scalar3d_swband_layout(true); + auto scalar2d = grid->get_scalar2d(true); + + // The fields required for this diagnostic to be computed + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + + // Construct and allocate the aodvis field + // We are going to assume we have nondim units here for ease + FieldIdentifier fid("VisibileAerosolOpticalDepth", scalar2d, nondim, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); +} + +void AODVis::compute_diagnostic_impl() +{ + using KT = KokkosTypes; + using MT = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + const auto aod = m_diagnostic_output.get_view(); + const auto tau = get_field_in("aero_tau_sw").get_view(); + + const auto num_levs = m_num_levs; + const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); + Kokkos::parallel_for("Compute " + name(), policy, + KOKKOS_LAMBDA(const MT& team) { + const int icol = team.league_rank(); + // TODO: take a slice of tau at 9th band (vis band) + auto tau_vis = tau(:, 9, :) + auto tau_icol = ekat::subview(tau_vis,icol); + Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), + [&] (const int& ilev, Real& lsum) { + lsum += tau_icol(ilev); + },aod(icol)); + team.team_barrier(); + }); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/aodvis.hpp b/components/eamxx/src/diagnostics/aodvis.hpp new file mode 100644 index 000000000000..6498445ad0c9 --- /dev/null +++ b/components/eamxx/src/diagnostics/aodvis.hpp @@ -0,0 +1,37 @@ +#ifndef EAMXX_AODVIS_DIAG +#define EAMXX_AODVIS_DIAG + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream { + +/* + * This diagnostic will compute the visible aerosol optical depth. + */ + +class AODVis : public AtmosphereDiagnostic { + public: + // Constructors + AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const override { return "aodvis"; } + + // Set the grid + void set_grids( + const std::shared_ptr grids_manager) override; + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + int m_ncols; + int m_nlevs; + +}; + +} // namespace scream + +#endif // EAMXX_AODVIS_DIAG diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 1f0c3bd63e3f..9d2d1cf8a29c 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -20,6 +20,7 @@ #include "diagnostics/precip_surf_mass_flux.hpp" #include "diagnostics/surf_upward_latent_heat_flux.hpp" #include "diagnostics/wind_speed.hpp" +#include "diagnostics/aodvis.hpp" namespace scream { @@ -47,6 +48,7 @@ inline void register_diagnostics () { diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); + diag_factory.register_product("aodvis",&create_atmosphere_diagnostic); } } // namespace scream From b2cbf3bc7045e6ff1149ccb7d1050e931e778be6 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 5 Apr 2024 16:20:34 -0500 Subject: [PATCH 027/476] complete aodvis calc and testing --- components/eamxx/src/diagnostics/aodvis.cpp | 49 ++++---- components/eamxx/src/diagnostics/aodvis.hpp | 2 +- .../src/diagnostics/tests/CMakeLists.txt | 3 + .../src/diagnostics/tests/aodvis_test.cpp | 108 ++++++++++++++++++ 4 files changed, 137 insertions(+), 25 deletions(-) create mode 100644 components/eamxx/src/diagnostics/tests/aodvis_test.cpp diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 6f512a602ec2..846d3688d033 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -4,8 +4,7 @@ namespace scream { -AODVis::AODVis(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) +AODVis::AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereDiagnostic(comm, params) { // Nothing to do here } @@ -22,45 +21,47 @@ void AODVis::set_grids( m_ncols = grid->get_num_local_dofs(); m_nlevs = grid->get_num_vertical_levels(); + // TODO: don't hardcode this! + m_swbnd = 14; // Define layouts we need (both inputs and outputs) - // TODO: define this properly (dunno state of things) - auto scalar3d_swband_layout = grid->scalar3d_swband_layout(true); - auto scalar2d = grid->get_scalar2d(true); + FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, + {m_ncols, m_swbnd, m_nlevs}}; + FieldLayout scalar1d_layout{{COL}, {m_ncols}}; // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); // Construct and allocate the aodvis field // We are going to assume we have nondim units here for ease - FieldIdentifier fid("VisibileAerosolOpticalDepth", scalar2d, nondim, grid_name); + FieldIdentifier fid("aodvis", scalar1d_layout, nondim, + grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } -void AODVis::compute_diagnostic_impl() -{ +void AODVis::compute_diagnostic_impl() { using KT = KokkosTypes; using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; - const auto aod = m_diagnostic_output.get_view(); - const auto tau = get_field_in("aero_tau_sw").get_view(); + const auto aod = m_diagnostic_output.get_view(); + // get slice of tau at swbnd 9 + const auto tau_vis = + get_field_in("aero_tau_sw").subfield(1, 9).get_view(); - const auto num_levs = m_num_levs; - const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); - Kokkos::parallel_for("Compute " + name(), policy, - KOKKOS_LAMBDA(const MT& team) { - const int icol = team.league_rank(); - // TODO: take a slice of tau at 9th band (vis band) - auto tau_vis = tau(:, 9, :) - auto tau_icol = ekat::subview(tau_vis,icol); - Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), - [&] (const int& ilev, Real& lsum) { - lsum += tau_icol(ilev); - },aod(icol)); - team.team_barrier(); - }); + const auto num_levs = m_nlevs; + const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); + Kokkos::parallel_for( + "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { + const int icol = team.league_rank(); + auto tau_icol = ekat::subview(tau_vis, icol); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(team, num_levs), + [&](const int &ilev, Real &lsum) { lsum += tau_icol(ilev); }, + aod(icol)); + team.team_barrier(); + }); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/aodvis.hpp b/components/eamxx/src/diagnostics/aodvis.hpp index 6498445ad0c9..e101903da2b9 100644 --- a/components/eamxx/src/diagnostics/aodvis.hpp +++ b/components/eamxx/src/diagnostics/aodvis.hpp @@ -29,7 +29,7 @@ class AODVis : public AtmosphereDiagnostic { int m_ncols; int m_nlevs; - + int m_swbnd; }; } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index d413b013f444..3e17d1122a7a 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -61,4 +61,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test wind speed diagnostic CreateDiagTest(wind_speed "wind_speed_tests.cpp") + + # Test AODVIS + CreateDiagTest(aodvis "aodvis_test.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp new file mode 100644 index 000000000000..7766d2553bf6 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -0,0 +1,108 @@ +#include + +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_common_physics_functions.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_utils.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("aodvis") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2022, 1, 1}, {0, 0, 0}); + + const auto nondim = Units::nondimensional(); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 33; + const int ngcols = 2 * comm.size(); + + // TODO: Don't hardcode this! + constexpr int nbnds = 14; + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) tau + FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, {ngcols, nbnds, nlevs}}; + FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, + grid->name()); + Field tau(tau_fid); + tau.allocate_view(); + tau.get_header().get_tracking().update_time_stamp(t0); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(-1, 1); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + constexpr int ntests = 5; + for(int itest = 0; itest < ntests; ++itest) { + // Randomize tau + randomize(tau, engine, pdf); + + // Create and set up the diagnostic + ekat::ParameterList params; + auto diag = diag_factory.create("aodvis", comm, params); + diag->set_grids(gm); + diag->set_required_field(tau); + diag->initialize(t0, RunType::Initial); + + // Run diag + diag->compute_diagnostic(); + + // Check result + tau.sync_to_host(); + diag->get_diagnostic().sync_to_host(); + + auto tau_h = tau.get_view(); + auto aod_h = diag->get_diagnostic().get_view(); + + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + auto aod_temp = 0.0; + for(int ilev = 0; ilev < nlevs; ++ilev) { + // TODO: Don't hardcode the 9! + aod_temp += tau_h(icol, 9, ilev); + } + REQUIRE(aod_temp == aod_h(icol)); + } + } +} + +} // namespace scream From c3ff7007ea84851e4c2f9a8400b0d14aefd2665b Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 5 Apr 2024 16:43:24 -0700 Subject: [PATCH 028/476] move to swbnd 10 and fix tests on cuda --- components/eamxx/src/diagnostics/aodvis.cpp | 5 ++-- .../src/diagnostics/tests/aodvis_test.cpp | 28 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 846d3688d033..562886f3bcc7 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -46,9 +46,10 @@ void AODVis::compute_diagnostic_impl() { using ESU = ekat::ExeSpaceUtils; const auto aod = m_diagnostic_output.get_view(); - // get slice of tau at swbnd 9 + // TODO: don't hardcode swbnd 10 + // Get slice of tau at swbnd 10 const auto tau_vis = - get_field_in("aero_tau_sw").subfield(1, 9).get_view(); + get_field_in("aero_tau_sw").subfield(1, 10).get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 7766d2553bf6..52681aed25ba 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -1,15 +1,8 @@ -#include - #include "catch2/catch.hpp" #include "diagnostics/register_diagnostics.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" -#include "physics/share/physics_constants.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" -#include "share/util/scream_common_physics_functions.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_utils.hpp" namespace scream { @@ -64,7 +57,7 @@ TEST_CASE("aodvis") { // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(-1, 1); + RPDF pdf(0, 0.005); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -91,17 +84,26 @@ TEST_CASE("aodvis") { tau.sync_to_host(); diag->get_diagnostic().sync_to_host(); - auto tau_h = tau.get_view(); - auto aod_h = diag->get_diagnostic().get_view(); + auto tau_h = tau.get_view(); + auto aod_hf = diag->get_diagnostic(); + auto aod_h = aod_hf.get_view(); + + Field aod_tf = diag->get_diagnostic().clone(); + auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // Create aod_temp as scalar on host auto aod_temp = 0.0; for(int ilev = 0; ilev < nlevs; ++ilev) { - // TODO: Don't hardcode the 9! - aod_temp += tau_h(icol, 9, ilev); + // TODO: Don't hardcode the 10! + aod_temp += tau_h(icol, 10, ilev); + aod_t(icol) += tau_h(icol, 10, ilev); } - REQUIRE(aod_temp == aod_h(icol)); + // These should be identical, but not sure why they fail cuda + REQUIRE(std::abs(aod_temp - aod_h(icol)) < 1e-10); } + // Another way to test is to compare the views + REQUIRE(views_are_equal(aod_hf, aod_tf)); } } From 560ec8534b103dca97ade91431915abff8f44016 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 5 Apr 2024 18:12:56 -0700 Subject: [PATCH 029/476] relax tolerance to allow passing in sp --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 52681aed25ba..5f292b2c5b50 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -57,7 +57,7 @@ TEST_CASE("aodvis") { // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.005); + RPDF pdf(0, 0.05); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -92,15 +92,14 @@ TEST_CASE("aodvis") { auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - // Create aod_temp as scalar on host auto aod_temp = 0.0; for(int ilev = 0; ilev < nlevs; ++ilev) { // TODO: Don't hardcode the 10! aod_temp += tau_h(icol, 10, ilev); aod_t(icol) += tau_h(icol, 10, ilev); } - // These should be identical, but not sure why they fail cuda - REQUIRE(std::abs(aod_temp - aod_h(icol)) < 1e-10); + // Allow for sp tolerance? + REQUIRE(std::abs(aod_temp - aod_h(icol)) < 1e-6); } // Another way to test is to compare the views REQUIRE(views_are_equal(aod_hf, aod_tf)); From ffc05704a91789a3ea5de817482164b0630d89b7 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Mon, 8 Apr 2024 06:10:36 -0700 Subject: [PATCH 030/476] deep copy to zero on host --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 5f292b2c5b50..bfd8943f5e0c 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -89,7 +89,8 @@ TEST_CASE("aodvis") { auto aod_h = aod_hf.get_view(); Field aod_tf = diag->get_diagnostic().clone(); - auto aod_t = aod_tf.get_view(); + aod_tf.deep_copy(0.0); + auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { auto aod_temp = 0.0; From ea1bfa8b77eb67ff45f75cff7e0327b51d579081 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 17 Apr 2024 10:14:55 -0500 Subject: [PATCH 031/476] change name to AerosolOpticalDepth550nm --- components/eamxx/src/diagnostics/aodvis.cpp | 2 +- components/eamxx/src/diagnostics/aodvis.hpp | 2 +- components/eamxx/src/diagnostics/register_diagnostics.hpp | 2 +- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 562886f3bcc7..f966df1c2d11 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -34,7 +34,7 @@ void AODVis::set_grids( // Construct and allocate the aodvis field // We are going to assume we have nondim units here for ease - FieldIdentifier fid("aodvis", scalar1d_layout, nondim, + FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/aodvis.hpp b/components/eamxx/src/diagnostics/aodvis.hpp index e101903da2b9..f03d9a9e3405 100644 --- a/components/eamxx/src/diagnostics/aodvis.hpp +++ b/components/eamxx/src/diagnostics/aodvis.hpp @@ -15,7 +15,7 @@ class AODVis : public AtmosphereDiagnostic { AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The name of the diagnostic - std::string name() const override { return "aodvis"; } + std::string name() const override { return "AerosolOpticalDepth550nm"; } // Set the grid void set_grids( diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 9d2d1cf8a29c..b57a04bd3486 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -48,7 +48,7 @@ inline void register_diagnostics () { diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); - diag_factory.register_product("aodvis",&create_atmosphere_diagnostic); + diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index bfd8943f5e0c..befdd7705ecf 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -72,7 +72,7 @@ TEST_CASE("aodvis") { // Create and set up the diagnostic ekat::ParameterList params; - auto diag = diag_factory.create("aodvis", comm, params); + auto diag = diag_factory.create("AerosolOpticalDepth550nm", comm, params); diag->set_grids(gm); diag->set_required_field(tau); diag->initialize(t0, RunType::Initial); From 314e23c5c629d8bdbd4df734d202edb933a02367 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 18 Apr 2024 15:56:31 -0700 Subject: [PATCH 032/476] use ekat view red, fix const --- components/eamxx/src/diagnostics/aodvis.cpp | Bin 2301 -> 2301 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index f966df1c2d113db50c6e86eda23a0b413a13ef9b..ac249f8c853b43e86f62cbb58817a14c607e47b5 100644 GIT binary patch delta 41 Wcmew>_*ZblA&$xSIpky+@BskAgau3h delta 32 lcmew>_*ZblAr27*g_6|7Ts;sGpOjcsl$lzjF?lD4Isnwh3{U_7 From 4c8fa5ab3cff348c127058e2150c5990bc23c555 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 18 Apr 2024 16:03:16 -0700 Subject: [PATCH 033/476] use ekat view red, fix const --- components/eamxx/src/diagnostics/aodvis.cpp | Bin 2301 -> 2181 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index ac249f8c853b43e86f62cbb58817a14c607e47b5..ec9ec4f83e4989df63c249225e247f17351a0344 100644 GIT binary patch delta 100 zcmew>*ebZ;JR4hbeqM2j!shdAwv0ZB`6(Ki$@w{&3bqQV*@-1qR<0GP$-xDQ$*G|w znK{K)R%Mx~$IItm6l3VEfu@j0nw#X1TliKX!%GbZokPzL}k Ck|aL> delta 221 zcmZn_{42QOJlp20Y_^Q`-uckeOGapq2?#qN5O$nwX=Y zmQ!4stEo_}kW;Kznya8~t5A|y8lRb*pQ8a%rD?5Di)vJ2eu@T2T61z7hq^2SJ^%m+ C`#OmL From 12f931ebe80a017dca9f3e2fe6c8bb61c661164e Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 18 Apr 2024 16:47:50 -0700 Subject: [PATCH 034/476] use ESU shortcut instead --- components/eamxx/src/diagnostics/aodvis.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index ec9ec4f83e49..66b1a7532abb 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -57,7 +57,7 @@ void AODVis::compute_diagnostic_impl() { "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ekat::ExecSpaceUtils::view_reduction(team, 0, num_levs, tau_icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); }); } From 2b20ec556a8ef30b2bb581b3454cee5b20795ede Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 18 Apr 2024 16:56:35 -0700 Subject: [PATCH 035/476] opt for one test only, and sync to dev --- .../eamxx/src/diagnostics/tests/aodvis_test.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index befdd7705ecf..2abea562d8c1 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -84,25 +84,21 @@ TEST_CASE("aodvis") { tau.sync_to_host(); diag->get_diagnostic().sync_to_host(); - auto tau_h = tau.get_view(); - auto aod_hf = diag->get_diagnostic(); - auto aod_h = aod_hf.get_view(); + const auto tau_h = tau.get_view(); + const auto aod_hf = diag->get_diagnostic(); Field aod_tf = diag->get_diagnostic().clone(); aod_tf.deep_copy(0.0); auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - auto aod_temp = 0.0; for(int ilev = 0; ilev < nlevs; ++ilev) { // TODO: Don't hardcode the 10! - aod_temp += tau_h(icol, 10, ilev); aod_t(icol) += tau_h(icol, 10, ilev); } - // Allow for sp tolerance? - REQUIRE(std::abs(aod_temp - aod_h(icol)) < 1e-6); } - // Another way to test is to compare the views + aod_hf.sync_to_dev(); + aod_tf.sync_to_dev(); REQUIRE(views_are_equal(aod_hf, aod_tf)); } } From 71d3cd380f9cd1897ca9467f3e30c8b4dfdd6796 Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Fri, 19 Apr 2024 11:28:14 -0700 Subject: [PATCH 036/476] Add eamxx+tech overviews + bib file --- components/eamxx/docs/index.md | 17 ++-- components/eamxx/docs/refs.bib | 98 ++++++++++++++++++++++++ components/eamxx/docs/technical/index.md | 22 +++++- 3 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 components/eamxx/docs/refs.bib diff --git a/components/eamxx/docs/index.md b/components/eamxx/docs/index.md index 992b797671cf..d10d4d630fb4 100644 --- a/components/eamxx/docs/index.md +++ b/components/eamxx/docs/index.md @@ -1,10 +1,11 @@ -# The C++ E3SM Atmosphere Model (EAMxx) +# The E3SM Atmosphere Model in C++ (EAMxx) -Some nice introductory text goes here! Maybe some figures, too. Who knows?! +EAMxx is an entirely new atmosphere model for E3SM. It is written in C++ using the Kokkos performance portability library to enable it to run efficiently on CPUs, GPUs, and (hopefully) whatever comes next. Currently only the km-scale explicit-convection version called SCREAM (the Simple Cloud-Resolving E3SM Atmosphere Model) is available, but a low-resolution version is in the works. -* The [User Guide](user/index.md) explains how to run EAMxx, both in - its standalone configuration and within E3SM. -* The [Developer Guide](developer/index.md) contains all the information needed - to contribute to the development of EAMxx. -* The [Technical Guide](technical/index.md) contains all the technical - information about EAMxx. +Like the documentation for other component models, EAMxx documentation is divided into: + +* The [User Guide](user/index.md) which explains how to run EAMxx and describes all the compile-time and run-time options available for modifying a simulation +* The [Developer Guide](developer/index.md) which contains all the information needed to contribute to the development of EAMxx +* The [Technical Guide](technical/index.md) which describes the equations and numerical methods used by each of the specific process implementations and diagnostics available in the current version of EAMxx + +Put another way, all information about how to customize runs without changing code is included in the User Guide, general information about software design which is needed for intelligently modifying code goes in the Developer Guide, and details about the specific process implementations in the current model version are included in the Technical Guide. diff --git a/components/eamxx/docs/refs.bib b/components/eamxx/docs/refs.bib new file mode 100644 index 000000000000..3aed3d066dff --- /dev/null +++ b/components/eamxx/docs/refs.bib @@ -0,0 +1,98 @@ +@article{@Bogenschutz_Krueger13, +author = {Bogenschutz, Peter A. and Krueger, Steven K.}, +title = {A simplified PDF parameterization of subgrid-scale clouds and turbulence for cloud-resolving models}, +journal = {Journal of Advances in Modeling Earth Systems}, +volume = {5}, +number = {2}, +pages = {195-211}, +keywords = {cloud parameterization, turbulence, boundary layer clouds}, +doi = {https://doi.org/10.1002/jame.20018}, +url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1002/jame.20018}, +eprint = {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1002/jame.20018}, +year = {2013} +} + +@Article{Bradley_et22, +AUTHOR = {Bradley, A. M. and Bosler, P. A. and Guba, O.}, +TITLE = {Islet: interpolation semi-Lagrangian element-based transport}, +JOURNAL = {Geoscientific Model Development}, +VOLUME = {15}, +YEAR = {2022}, +NUMBER = {16}, +PAGES = {6285--6310}, +URL = {https://gmd.copernicus.org/articles/15/6285/2022/}, +DOI = {10.5194/gmd-15-6285-2022} +} + +@article{Caldwell_et21, +author = {Caldwell, P. M. and Terai, C. R. and Hillman, B. and Keen, N. D. and Bogenschutz, P. and Lin, W. and Beydoun, H. and Taylor, M. and Bertagna, L. and Bradley, A. M. and Clevenger, T. C. and Donahue, A. S. and Eldred, C. and Foucar, J. and Golaz, J.-C. and Guba, O. and Jacob, R. and Johnson, J. and Krishna, J. and Liu, W. and Pressel, K. and Salinger, A. G. and Singh, B. and Steyer, A. and Ullrich, P. and Wu, D. and Yuan, X. and Shpund, J. and Ma, H.-Y. and Zender, C. S.}, +title = {Convection-Permitting Simulations With the E3SM Global Atmosphere Model}, +journal = {Journal of Advances in Modeling Earth Systems}, +volume = {13}, +number = {11}, +pages = {e2021MS002544}, +keywords = {cloud resolving model, storm resolving model, general circulation model, convection permitting model, E3SM}, +doi = {https://doi.org/10.1029/2021MS002544}, +url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2021MS002544}, +eprint = {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2021MS002544}, +note = {e2021MS002544 2021MS002544}, +year = {2021} +} + +@article{Hannah_et21, +author = {Hannah, Walter M. and Bradley, Andrew M. and Guba, Oksana and Tang, Qi and Golaz, Jean-Christophe and Wolfe, Jon}, +title = {Separating Physics and Dynamics Grids for Improved Computational Efficiency in Spectral Element Earth System Models}, +journal = {Journal of Advances in Modeling Earth Systems}, +volume = {13}, +number = {7}, +pages = {e2020MS002419}, +keywords = {computational efficiency, E3SM, effective resolution, grid remap methods}, +doi = {https://doi.org/10.1029/2020MS002419}, +url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2020MS002419}, +eprint = {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2020MS002419}, +note = {e2020MS002419 2020MS002419}, +year = {2021} +} + +@article {Morrison_Milbrandt15, + author = "Hugh Morrison and Jason A. Milbrandt", + title = "Parameterization of Cloud Microphysics Based on the Prediction of Bulk Ice Particle Properties. Part I: Scheme Description and Idealized Tests", + journal = "Journal of the Atmospheric Sciences", + year = "2015", + publisher = "American Meteorological Society", + address = "Boston MA, USA", + volume = "72", + number = "1", + doi = "10.1175/JAS-D-14-0065.1", + pages= "287 - 311", + url = "https://journals.ametsoc.org/view/journals/atsc/72/1/jas-d-14-0065.1.xml" +} + +@article{Pincus_et19, +author = {Pincus, Robert and Mlawer, Eli J. and Delamere, Jennifer S.}, +title = {Balancing Accuracy, Efficiency, and Flexibility in Radiation Calculations for Dynamical Models}, +journal = {Journal of Advances in Modeling Earth Systems}, +volume = {11}, +number = {10}, +pages = {3074-3089}, +keywords = {radiation, atmospheric model, parameterization}, +doi = {https://doi.org/10.1029/2019MS001621}, +url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2019MS001621}, +eprint = {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2019MS001621}, +year = {2019} +} + +@article{Taylor_et20, +author = {Taylor, Mark A. and Guba, Oksana and Steyer, Andrew and Ullrich, Paul A. and Hall, David M. and Eldred, Christopher}, +title = {An Energy Consistent Discretization of the Nonhydrostatic Equations in Primitive Variables}, +journal = {Journal of Advances in Modeling Earth Systems}, +volume = {12}, +number = {1}, +pages = {e2019MS001783}, +keywords = {nonhydrostatic, hamiltonian, dynamical core, energy conservation, mimetic}, +doi = {https://doi.org/10.1029/2019MS001783}, +url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2019MS001783}, +eprint = {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2019MS001783}, +note = {e2019MS001783 10.1029/2019MS001783}, +year = {2020} +} diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 8b9a45be4fdf..0fb8cda1fece 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -1,3 +1,21 @@ -# SCREAM Technical Guide +# EAMxx Technical Guide -SCREAM contributors and maintainers will add detailed technical information about SCREAM here. +The goal of this document is to describe the specific equations, parameterizations, and numerical methods used in the current version of EAMxx. Because our master-branch implementation changes every time we make a new commit, this documentation will also evolve continuously. As such, documentation for master should always be considered to be preliminary and under construction. If you want trustworthy documentation, pull it from an official model release. + +## Overview + +Currently, EAMxx is only configured for km-scale convection-permitting runs. In order to provide scientifically-credible simulations at lower resolution, parameterizations for the following processes would be needed: + +1. deep convection +2. gravity-wave drag +3. energy fixer + +The only configuration of EAMxx that is currently implemented is the convection-permitting version, commonly known as the Simple Cloud-Resolving E3SM Atmosphere Model (SCREAM). Parameterizations in EAMxx-SCREAM are: + +1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions [@Taylor_et20] with semi-Lagrangian tracer advection as described by [@Bradley_et22] +2. the **Simple Higher-Order Closure (SHOC)** parameterization from [@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction +3. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. +4. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC +5. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance + +These parameterizations are described in more detail in Caldwell et al 2021. The CFMIP Observation Simulator Package (COSP) is also integrated into EAMxx, but currently only the ISCCP output is enabled. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. From ee61b1c7a020365a7eb47b4bcab6cb7b0f7aeffc Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Fri, 19 Apr 2024 11:59:11 -0700 Subject: [PATCH 037/476] Forgot TMS proc --- components/eamxx/docs/refs.bib | 15 +++++++++++++++ components/eamxx/docs/technical/index.md | 7 ++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/components/eamxx/docs/refs.bib b/components/eamxx/docs/refs.bib index 3aed3d066dff..d73c96995a51 100644 --- a/components/eamxx/docs/refs.bib +++ b/components/eamxx/docs/refs.bib @@ -39,6 +39,21 @@ @article{Caldwell_et21 year = {2021} } +@article{Fiedler_Panofsky72, +author = {Fiedler, F. and Panofsky, H. A.}, +title = {The geostrophic drag coefficient and the ‘effective’ roughness length}, +journal = {Quarterly Journal of the Royal Meteorological Society}, +volume = {98}, +number = {415}, +pages = {213-220}, +doi = {https://doi.org/10.1002/qj.49709841519}, +url = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1002/qj.49709841519}, +eprint = {https://rmets.onlinelibrary.wiley.com/doi/pdf/10.1002/qj.49709841519}, +abstract = {Abstract An ‘effective’ roughness length is defined for use over heterogeneous terrain as the roughness length which homogeneous terrain would have to give the correct surface stress over a given area. A method is suggested to compute geostrophic drag coefficient, wind-contour angle and surface heat flux, given this roughness length, latitude, geostrophic wind speed and insolation or ground-air temperature differences.}, +year = {1972} +} + + @article{Hannah_et21, author = {Hannah, Walter M. and Bradley, Andrew M. and Guba, Oksana and Tang, Qi and Golaz, Jean-Christophe and Wolfe, Jon}, title = {Separating Physics and Dynamics Grids for Improved Computational Efficiency in Spectral Element Earth System Models}, diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 0fb8cda1fece..80d35fd45949 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -14,8 +14,9 @@ The only configuration of EAMxx that is currently implemented is the convection- 1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions [@Taylor_et20] with semi-Lagrangian tracer advection as described by [@Bradley_et22] 2. the **Simple Higher-Order Closure (SHOC)** parameterization from [@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction -3. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. -4. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC -5. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance +3. turbulent mountain stress is crudely parameterized following [@Fiedler_Panofsky72] to reduce excessive winds around topography +4. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. +5. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC +6. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance These parameterizations are described in more detail in Caldwell et al 2021. The CFMIP Observation Simulator Package (COSP) is also integrated into EAMxx, but currently only the ISCCP output is enabled. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. From 1cfa1555006493776e9b020c094bc5a160be4fab Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Fri, 19 Apr 2024 12:05:33 -0700 Subject: [PATCH 038/476] Forgot SPA --- components/eamxx/docs/refs.bib | 12 ++++++++++++ components/eamxx/docs/technical/index.md | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/components/eamxx/docs/refs.bib b/components/eamxx/docs/refs.bib index d73c96995a51..6ab4e21c9173 100644 --- a/components/eamxx/docs/refs.bib +++ b/components/eamxx/docs/refs.bib @@ -97,6 +97,18 @@ @article{Pincus_et19 year = {2019} } +@Article{Stevens_et17, +AUTHOR = {Stevens, B. and Fiedler, S. and Kinne, S. and Peters, K. and Rast, S. and M\"usse, J. and Smith, S. J. and Mauritsen, T.}, +TITLE = {MACv2-SP: a parameterization of anthropogenic aerosol optical properties and an associated Twomey effect for use in CMIP6}, +JOURNAL = {Geoscientific Model Development}, +VOLUME = {10}, +YEAR = {2017}, +NUMBER = {1}, +PAGES = {433--452}, +URL = {https://gmd.copernicus.org/articles/10/433/2017/}, +DOI = {10.5194/gmd-10-433-2017} +} + @article{Taylor_et20, author = {Taylor, Mark A. and Guba, Oksana and Steyer, Andrew and Ullrich, Paul A. and Hall, David M. and Eldred, Christopher}, title = {An Energy Consistent Discretization of the Nonhydrostatic Equations in Primitive Variables}, diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 80d35fd45949..895df86b97c3 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -14,9 +14,10 @@ The only configuration of EAMxx that is currently implemented is the convection- 1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions [@Taylor_et20] with semi-Lagrangian tracer advection as described by [@Bradley_et22] 2. the **Simple Higher-Order Closure (SHOC)** parameterization from [@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction -3. turbulent mountain stress is crudely parameterized following [@Fiedler_Panofsky72] to reduce excessive winds around topography +3. **turbulent mountain stress** is crudely parameterized following [@Fiedler_Panofsky72] to reduce excessive winds around topography 4. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. +5. the effects of aerosol are prescribed via the Simple Prescribed Aerosol (SPA) scheme, which is very similar to MACv2-SP [@Stevens_et17] 5. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC 6. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance -These parameterizations are described in more detail in Caldwell et al 2021. The CFMIP Observation Simulator Package (COSP) is also integrated into EAMxx, but currently only the ISCCP output is enabled. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. +By default processes are called in this order, but which processes to include and in what order is modifiable at run time. These parameterizations are described in more detail in Caldwell et al 2021. The CFMIP Observation Simulator Package (COSP) is also integrated into EAMxx, but currently only the ISCCP output is enabled. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. From bcd19292977dc82a0a776d05649d19b036459421 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 19 Apr 2024 15:42:03 -0400 Subject: [PATCH 039/476] only test views are equal in non-release Co-authored-by: Thomas Conrad Clevenger --- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 2abea562d8c1..7846400287cc 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -99,7 +99,10 @@ TEST_CASE("aodvis") { } aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); - REQUIRE(views_are_equal(aod_hf, aod_tf)); + // Workaround for non-bfb behavior of view_reduction() in release builds + if (SCREAM_BFB_TESTING) { + REQUIRE(views_are_equal(aod_hf, aod_tf)); + } } } From 136ec08ffca518b7d2680e885bdf73080bb0db46 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 19 Apr 2024 14:00:13 -0600 Subject: [PATCH 040/476] progress --- components/eam/src/physics/rrtmgp/external | 2 +- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 14 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 6 + .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 108 +- .../src/physics/rrtmgp/mo_garand_atmos_io.cpp | 107 - .../src/physics/rrtmgp/mo_garand_atmos_io.h | 17 - .../rrtmgp/mo_load_cloud_coefficients.cpp | 107 - .../rrtmgp/mo_load_cloud_coefficients.h | 14 - .../physics/rrtmgp/mo_load_coefficients.cpp | 155 - .../src/physics/rrtmgp/mo_load_coefficients.h | 11 - .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 190 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 3201 +++++++++++------ .../rrtmgp/scream_rrtmgp_interface.hpp | 518 ++- 13 files changed, 2749 insertions(+), 1701 deletions(-) delete mode 100644 components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.cpp delete mode 100644 components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.h delete mode 100644 components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.cpp delete mode 100644 components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.h delete mode 100644 components/eamxx/src/physics/rrtmgp/mo_load_coefficients.cpp delete mode 100644 components/eamxx/src/physics/rrtmgp/mo_load_coefficients.h diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index ddf82a25a3a5..374f2bdfbe1e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit ddf82a25a3a59a8f0fe904b69181cb7bd99881fb +Subproject commit 374f2bdfbe1eea0489bd625114347a8463dee407 diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 5945189412c8..1c7736de76b4 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -2,6 +2,14 @@ include(EkatUtils) include(EkatSetCompilerFlags) include(ScreamUtils) +if (SCREAM_RRTMGP_ENABLE_YAKL) + add_definitions("-DRRTMGP_ENABLE_YAKL") +endif() + +if (SCREAM_RRTMGP_ENABLE_KOKKOS) + add_definitions("-DRRTMGP_ENABLE_KOKKOS") +endif() + # Copied from EKAT, YAKL is an interface target so requires special # handling. Get rid of this once RRTMGP is using kokkos. macro (SetCudaFlagsYakl targetName) @@ -116,6 +124,7 @@ set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_optical_props_kernels.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_rte_solver_kernels.cpp ${EAM_RRTMGP_DIR}/external/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp + ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_garand_atmos_io.cpp ) add_library(rrtmgp ${EXTERNAL_SRC}) target_compile_definitions(rrtmgp PUBLIC EAMXX_HAS_RRTMGP) @@ -129,6 +138,8 @@ target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL ${EAM_RRTMGP_DIR}/external/cpp + ${EAM_RRTMGP_DIR}/external/cpp/extensions/cloud_optics + ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky ${EAM_RRTMGP_DIR}/external/cpp/rte ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp @@ -151,9 +162,6 @@ target_include_directories(rrtmgp PUBLIC set(SCREAM_RRTMGP_SOURCES_YAKL scream_rrtmgp_interface.cpp - mo_garand_atmos_io.cpp - mo_load_coefficients.cpp - mo_load_cloud_coefficients.cpp ) add_library(scream_rrtmgp_yakl ${SCREAM_RRTMGP_SOURCES_YAKL}) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 16811885a3b9..3778e32138c9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -12,7 +12,9 @@ #include "ekat/ekat_assert.hpp" #include "cpp/rrtmgp/mo_gas_concentrations.h" +#ifdef RRTMGP_ENABLE_YAKL #include "YAKL.h" +#endif namespace scream { @@ -395,8 +397,10 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Whether or not to do MCICA subcolumn sampling m_do_subcol_sampling = m_params.get("do_subcol_sampling",true); +#ifdef RRTMGP_ENABLE_YAKL // Initialize yakl yakl_init(); +#endif // Names of active gases auto gas_names_yakl_offset = string1d("gas_names",m_ngas); @@ -1119,8 +1123,10 @@ void RRTMGPRadiation::finalize_impl () { m_gas_concs.reset(); rrtmgp::rrtmgp_finalize(); +#ifdef RRTMGP_ENABLE_YAKL // Finalize YAKL yakl_finalize(); +#endif } // ========================================================================================= diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index f1321fd08289..c0b29764ecae 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -77,7 +77,7 @@ class RRTMGPRadiation : public AtmosphereProcess { // This is only used if a positive value is supplied Real m_fixed_solar_zenith_angle; - // Need to hard-code some dimension sizes for now. + // Need to hard-code some dimension sizes for now. // TODO: find a better way of configuring this const int m_nswbands = 14; const int m_nlwbands = 16; @@ -88,7 +88,12 @@ class RRTMGPRadiation : public AtmosphereProcess { int m_ngas; std::vector m_gas_names; view_1d_real m_gas_mol_weights; +#ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + GasConcsK m_gas_concs_k; +#endif // Prescribed greenhouse gas surface concentrations in moles / moles air Real m_co2vmr; @@ -119,6 +124,7 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) +#ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; real1d sfc_alb_dir_nir; @@ -129,8 +135,22 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dir_nir; real1d sfc_flux_dif_vis; real1d sfc_flux_dif_nir; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real1dk mu0_k; + real1dk sfc_alb_dir_vis_k; + real1dk sfc_alb_dir_nir_k; + real1dk sfc_alb_dif_vis_k; + real1dk sfc_alb_dif_nir_k; + uview_1d cosine_zenith_k; + real1dk sfc_flux_dir_vis_k; + real1dk sfc_flux_dir_nir_k; + real1dk sfc_flux_dif_vis_k; + real1dk sfc_flux_dif_nir_k; +#endif // 2d size (ncol, nlay) +#ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; real2d z_del; @@ -147,8 +167,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sw_heating; real2d lw_heating; uview_2d d_dz; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real2dk p_lay_k; + real2dk t_lay_k; + real2dk z_del_k; + real2dk p_del_k; + real2dk qc_k; + real2dk nc_k; + real2dk qi_k; + real2dk cldfrac_tot_k; + real2dk eff_radius_qc_k; + real2dk eff_radius_qi_k; + real2dk tmp2d_k; + real2dk lwp_k; + real2dk iwp_k; + real2dk sw_heating_k; + real2dk lw_heating_k; + uview_2d d_dz_k; +#endif // 2d size (ncol, nlay+1) +#ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; real2d sw_flux_up; @@ -172,34 +212,100 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_up; real2d lw_clnsky_flux_dn; uview_2d d_tint; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real2dk p_lev_k; + real2dk t_lev_k; + real2dk sw_flux_up_k; + real2dk sw_flux_dn_k; + real2dk sw_flux_dn_dir_k; + real2dk lw_flux_up_k; + real2dk lw_flux_dn_k; + real2dk sw_clnclrsky_flux_up_k; + real2dk sw_clnclrsky_flux_dn_k; + real2dk sw_clnclrsky_flux_dn_dir_k; + real2dk sw_clrsky_flux_up_k; + real2dk sw_clrsky_flux_dn_k; + real2dk sw_clrsky_flux_dn_dir_k; + real2dk sw_clnsky_flux_up_k; + real2dk sw_clnsky_flux_dn_k; + real2dk sw_clnsky_flux_dn_dir_k; + real2dk lw_clnclrsky_flux_up_k; + real2dk lw_clnclrsky_flux_dn_k; + real2dk lw_clrsky_flux_up_k; + real2dk lw_clrsky_flux_dn_k; + real2dk lw_clnsky_flux_up_k; + real2dk lw_clnsky_flux_dn_k; + uview_2d d_tint_k; +#endif // 3d size (ncol, nlay+1, nswbands) +#ifdef RRTMGP_ENABLE_YAKL real3d sw_bnd_flux_up; real3d sw_bnd_flux_dn; real3d sw_bnd_flux_dir; real3d sw_bnd_flux_dif; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real3dk sw_bnd_flux_up_k; + real3dk sw_bnd_flux_dn_k; + real3dk sw_bnd_flux_dir_k; + real3dk sw_bnd_flux_dif_k; +#endif // 3d size (ncol, nlay+1, nlwbands) +#ifdef RRTMGP_ENABLE_YAKL real3d lw_bnd_flux_up; real3d lw_bnd_flux_dn; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real3dk lw_bnd_flux_up_k; + real3dk lw_bnd_flux_dn_k; +#endif // 2d size (ncol, nswbands) +#ifdef RRTMGP_ENABLE_YAKL real2d sfc_alb_dir; real2d sfc_alb_dif; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real2dk sfc_alb_dir_k; + real2dk sfc_alb_dif_k; +#endif // 3d size (ncol, nlay, n[sw,lw]bands) +#ifdef RRTMGP_ENABLE_YAKL real3d aero_tau_sw; real3d aero_ssa_sw; real3d aero_g_sw; real3d aero_tau_lw; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real3dk aero_tau_sw_k; + real3dk aero_ssa_sw_k; + real3dk aero_g_sw_k; + real3dk aero_tau_lw_k; +#endif // 3d size (ncol, nlay, n[sw,lw]bnds) +#ifdef RRTMGP_ENABLE_YAKL real3d cld_tau_sw_bnd; real3d cld_tau_lw_bnd; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real3dk cld_tau_sw_bnd_k; + real3dk cld_tau_lw_bnd_k; +#endif // 3d size (ncol, nlay, n[sw,lw]gpts) +#ifdef RRTMGP_ENABLE_YAKL real3d cld_tau_sw_gpt; real3d cld_tau_lw_gpt; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real3dk cld_tau_sw_gpt_k; + real3dk cld_tau_lw_gpt_k; +#endif }; diff --git a/components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.cpp b/components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.cpp deleted file mode 100644 index d8ea77994021..000000000000 --- a/components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "mo_garand_atmos_io.h" -#include "simple_netcdf.hpp" -#include - -using yakl::fortran::parallel_for; -using yakl::fortran::SimpleBounds; - -// Read in the data, then use only the first column, and copy it to all of the model columns -// In the end, all model columns will be identical -void read_atmos(std::string input_file, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, int ncol) { - simple_netcdf::SimpleNetCDF io; - io.open(input_file , NC_NOWRITE); - - int nlay = io.getDimSize("lay"); - int nlev = io.getDimSize("lev"); - - //p_lay = real2d("p_lay",ncol,nlay); - //t_lay = real2d("t_lay",ncol,nlay); - //p_lev = real2d("p_lev",ncol,nlev); - //t_lev = real2d("t_lev",ncol,nlev); - - real2d tmp2d; - // p_lay - io.read(tmp2d,"p_lay"); - // for (int ilay=1 ; ilay <= nlay ; ilay++) { - // for (int icol=1 ; icol <= ncol ; icol++) { - parallel_for( SimpleBounds<2>(nlay,ncol) , YAKL_LAMBDA (int ilay, int icol) { - p_lay(icol,ilay) = tmp2d(1,ilay); - }); - // t_lay - io.read(tmp2d,"t_lay"); - // for (int ilay=1 ; ilay <= nlay ; ilay++) { - // for (int icol=1 ; icol <= ncol ; icol++) { - parallel_for( SimpleBounds<2>(nlay,ncol) , YAKL_LAMBDA ( int ilay, int icol) { - t_lay(icol,ilay) = tmp2d(1,ilay); - }); - // p_lev - tmp2d = real2d(); // Reset tmp2d to avoid warnings about reallocating during file read - io.read(tmp2d,"p_lev"); - // for (int ilev=1 ; ilev <= nlev ; ilev++) { - // for (int icol=1 ; icol <= ncol ; icol++) { - parallel_for( SimpleBounds<2>(nlev,ncol) , YAKL_LAMBDA ( int ilev, int icol) { - p_lev(icol,ilev) = tmp2d(1,ilev); - }); - // t_lev - io.read(tmp2d,"t_lev"); - // for (int ilev=1 ; ilev <= nlev ; ilev++) { - // for (int icol=1 ; icol <= ncol ; icol++) { - parallel_for( SimpleBounds<2>(nlev,ncol) , YAKL_LAMBDA( int ilev, int icol) { - t_lev(icol,ilev) = tmp2d(1,ilev); - }); - - int ngas = 8; - string1d gas_names("gas_names",ngas); - gas_names(1) = std::string("h2o"); - gas_names(2) = std::string("co2"); - gas_names(3) = std::string("o3" ); - gas_names(4) = std::string("n2o"); - gas_names(5) = std::string("co" ); - gas_names(6) = std::string("ch4"); - gas_names(7) = std::string("o2" ); - gas_names(8) = std::string("n2" ); - - // Initialize GasConcs object with an "ncol" given from the calling program - gas_concs.init(gas_names,ncol,nlay); - - tmp2d = real2d(); // Reset the tmp2d variable - for (int igas=1 ; igas <= ngas ; igas++) { - std::string vmr_name = "vmr_"+gas_names(igas); - if ( ! io.varExists(vmr_name) ) { stoprun("ERROR: gas does not exist in input file"); } - // Read in 2-D varaible - io.read(tmp2d,vmr_name); - // Create 1-D variable with just the first column - real1d tmp1d("tmp1d",nlay); - // for (int i=1 ; i <= nlay ; i++) { - parallel_for( SimpleBounds<1>(nlay) , YAKL_LAMBDA (int i) { - tmp1d(i) = tmp2d(1,i); - }); - // Call set_vmr with only the first column from the data file copied among all of the model columns - gas_concs.set_vmr( gas_names(igas) , tmp1d ); - } - - io.close(); -} - - - -void write_sw_fluxes(std::string fileName, real2d const &flux_up, real2d const &flux_dn, real2d const &flux_dir, int ncol) { - simple_netcdf::SimpleNetCDF io; - io.open(fileName , NC_WRITE); - io.write(flux_up , "sw_flux_up_result" , {"col_new","lev"}); - io.write(flux_dn , "sw_flux_dn_result" , {"col_new","lev"}); - io.write(flux_dir , "sw_flux_dir_result" , {"col_new","lev"}); - io.close(); -} - - - -void write_lw_fluxes(std::string fileName, real2d const &flux_up, real2d const &flux_dn, int ncol) { - simple_netcdf::SimpleNetCDF io; - io.open(fileName , NC_WRITE); - io.write(flux_up , "lw_flux_up_result" , {"col_new","lev"}); - io.write(flux_dn , "lw_flux_dn_result" , {"col_new","lev"}); - io.close(); -} - diff --git a/components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.h b/components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.h deleted file mode 100644 index 719b360e740e..000000000000 --- a/components/eamxx/src/physics/rrtmgp/mo_garand_atmos_io.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef MO_GARAND_ATMOS_IO_HPP -#define MO_GARAND_ATMOS_IO_HPP - -#include "cpp/rrtmgp_const.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "YAKL.h" - -void read_atmos(std::string input_file, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, int ncol); - - -void write_sw_fluxes(std::string fileName, real2d const &flux_up, real2d const &flux_dn, real2d const &flux_dir, int ncol); - - -void write_lw_fluxes(std::string fileName, real2d const &flux_up, real2d const &flux_dn, int ncol); - -#endif diff --git a/components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.cpp b/components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.cpp deleted file mode 100644 index 1e9ead49f803..000000000000 --- a/components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "mo_load_cloud_coefficients.h" -#include "simple_netcdf.hpp" - -// read cloud optical property LUT coefficients from NetCDF file -void load_cld_lutcoeff(CloudOptics &cloud_spec, std::string cld_coeff_file) { - simple_netcdf::SimpleNetCDF io; - // Open cloud optical property coefficient file - io.open(cld_coeff_file , NC_NOWRITE); - - // Read LUT coefficient dimensions - int nband = io.getDimSize("nband"); - int nrghice = io.getDimSize("nrghice"); - int nsize_liq = io.getDimSize("nsize_liq"); - int nsize_ice = io.getDimSize("nsize_ice"); - - real2d band_lims_wvn("band_lims_wvn",2,nband); - io.read(band_lims_wvn,"bnd_limits_wavenumber"); - - // Read LUT constants - real radliq_lwr, radliq_upr, radliq_fac, radice_lwr, radice_upr, radice_fac; - io.read(radliq_lwr , "radliq_lwr"); - io.read(radliq_upr , "radliq_upr"); - io.read(radliq_fac , "radliq_fac"); - io.read(radice_lwr , "radice_lwr"); - io.read(radice_upr , "radice_upr"); - io.read(radice_fac , "radice_fac"); - - // Allocate cloud property lookup table input arrays - real2d lut_extliq("lut_extliq",nsize_liq, nband); - real2d lut_ssaliq("lut_ssaliq",nsize_liq, nband); - real2d lut_asyliq("lut_asyliq",nsize_liq, nband); - real3d lut_extice("lut_extice",nsize_ice, nband, nrghice); - real3d lut_ssaice("lut_ssaice",nsize_ice, nband, nrghice); - real3d lut_asyice("lut_asyice",nsize_ice, nband, nrghice); - // Read LUT coefficients - io.read(lut_extliq , "lut_extliq"); - io.read(lut_ssaliq , "lut_ssaliq"); - io.read(lut_asyliq , "lut_asyliq"); - io.read(lut_extice , "lut_extice"); - io.read(lut_ssaice , "lut_ssaice"); - io.read(lut_asyice , "lut_asyice"); - - io.close(); - - cloud_spec.load(band_lims_wvn, radliq_lwr, radliq_upr, radliq_fac, radice_lwr, radice_upr, radice_fac, - lut_extliq, lut_ssaliq, lut_asyliq, lut_extice, lut_ssaice, lut_asyice); -} - - - -// read cloud optical property Pade coefficients from NetCDF file -void load_cld_padecoeff(CloudOptics &cloud_spec, std::string cld_coeff_file) { - simple_netcdf::SimpleNetCDF io; - // Open cloud optical property coefficient file - io.open(cld_coeff_file , NC_NOWRITE); - - // Read Pade coefficient dimensions - int nband = io.getDimSize("nband"); - int nrghice = io.getDimSize("nrghice"); - int nsizereg = io.getDimSize("nsizereg"); - int ncoeff_ext = io.getDimSize("ncoeff_ext"); - int ncoeff_ssa_g = io.getDimSize("ncoeff_ssa_g"); - int nbound = io.getDimSize("nbound"); - - real2d band_lims_wvn("band_lims_wvn",2,nband); - io.read(band_lims_wvn, "bnd_limits_wavenumber"); - - // Allocate cloud property Pade coefficient input arrays - real3d pade_extliq("pade_extliq",nband, nsizereg, ncoeff_ext); - real3d pade_ssaliq("pade_ssaliq",nband, nsizereg, ncoeff_ssa_g); - real3d pade_asyliq("pade_asyliq",nband, nsizereg, ncoeff_ssa_g); - real4d pade_extice("pade_extice",nband, nsizereg, ncoeff_ext, nrghice); - real4d pade_ssaice("pade_ssaice",nband, nsizereg, ncoeff_ssa_g, nrghice); - real4d pade_asyice("pade_asyice",nband, nsizereg, ncoeff_ssa_g, nrghice); - io.read(pade_extliq, "pade_extliq"); - io.read(pade_ssaliq, "pade_ssaliq"); - io.read(pade_asyliq, "pade_asyliq"); - io.read(pade_extice, "pade_extice"); - io.read(pade_ssaice, "pade_ssaice"); - io.read(pade_asyice, "pade_asyice"); - - // Allocate cloud property Pade coefficient particle size boundary input arrays - real1d pade_sizreg_extliq("pade_sizreg_extliq",nbound); - real1d pade_sizreg_ssaliq("pade_sizreg_ssaliq",nbound); - real1d pade_sizreg_asyliq("pade_sizreg_asyliq",nbound); - real1d pade_sizreg_extice("pade_sizreg_extice",nbound); - real1d pade_sizreg_ssaice("pade_sizreg_ssaice",nbound); - real1d pade_sizreg_asyice("pade_sizreg_asyice",nbound); - - io.read(pade_sizreg_extliq, "pade_sizreg_extliq"); - io.read(pade_sizreg_ssaliq, "pade_sizreg_ssaliq"); - io.read(pade_sizreg_asyliq, "pade_sizreg_asyliq"); - io.read(pade_sizreg_extice, "pade_sizreg_extice"); - io.read(pade_sizreg_ssaice, "pade_sizreg_ssaice"); - io.read(pade_sizreg_asyice, "pade_sizreg_asyice"); - - io.close(); - - cloud_spec.load(band_lims_wvn, pade_extliq, pade_ssaliq, pade_asyliq, - pade_extice, pade_ssaice, pade_asyice, - pade_sizreg_extliq, pade_sizreg_ssaliq, pade_sizreg_asyliq, - pade_sizreg_extice, pade_sizreg_ssaice, pade_sizreg_asyice); -} - - - - diff --git a/components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.h b/components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.h deleted file mode 100644 index 9d3ee80c52aa..000000000000 --- a/components/eamxx/src/physics/rrtmgp/mo_load_cloud_coefficients.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef MO_LOAD_CLOUD_COEFFICIENTS_HPP -#define MO_LOAD_CLOUD_COEFFICIENTS_HPP - -#include "cpp/rrtmgp_const.h" -#include "cpp/rte/mo_optical_props.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" - -void load_cld_lutcoeff(CloudOptics &cloud_spec, std::string cld_coeff_file); - -void load_cld_padecoeff(CloudOptics &cloud_spec, std::string cld_coeff_file); - -#endif - - diff --git a/components/eamxx/src/physics/rrtmgp/mo_load_coefficients.cpp b/components/eamxx/src/physics/rrtmgp/mo_load_coefficients.cpp deleted file mode 100644 index 53626756554a..000000000000 --- a/components/eamxx/src/physics/rrtmgp/mo_load_coefficients.cpp +++ /dev/null @@ -1,155 +0,0 @@ -#include "mo_load_coefficients.h" -#include "simple_netcdf.hpp" - -// This code is part of RRTM for GCM Applications - Parallel (RRTMGP) -// -// Contacts: Robert Pincus and Eli Mlawer -// email: rrtmgp@aer.com -// -// Copyright 2015-2018, Atmospheric and Environmental Research and -// Regents of the University of Colorado. All right reserved. -// -// Use and duplication is permitted under the terms of the -// BSD 3-clause license, see http://opensource.org/licenses/BSD-3-Clause -// ------------------------------------------------------------------------------------------------- - -void load_and_init(GasOpticsRRTMGP &kdist, std::string filename, GasConcs const &available_gases) { - simple_netcdf::SimpleNetCDF io; - io.open(filename , NC_NOWRITE); - - // Read the many arrays - string1d gas_names; - string1d gas_minor; - string1d identifier_minor; - string1d minor_gases_lower; - string1d minor_gases_upper; - string1d scaling_gas_lower; - string1d scaling_gas_upper; - intHost3d key_species; - realHost2d band_lims; - intHost2d band2gpt; - real press_ref_trop; - real temp_ref_p; - real temp_ref_t; - realHost1d press_ref; - realHost1d temp_ref; - realHost3d vmr_ref; - realHost4d kmajor; - intHost2d minor_limits_gpt_lower; - intHost2d minor_limits_gpt_upper; - boolHost1d minor_scales_with_density_lower; - boolHost1d minor_scales_with_density_upper; - boolHost1d scale_by_complement_lower; - boolHost1d scale_by_complement_upper; - intHost1d kminor_start_lower; - intHost1d kminor_start_upper; - realHost3d kminor_lower; - realHost3d kminor_upper; - realHost3d rayl_lower; - realHost3d rayl_upper; - - // Read in strings - charHost2d tmp; - tmp = charHost2d(); io.read( tmp , "gas_names" ); gas_names = char2d_to_string1d(tmp); - tmp = charHost2d(); io.read( tmp , "gas_minor" ); gas_minor = char2d_to_string1d(tmp); - tmp = charHost2d(); io.read( tmp , "identifier_minor" ); identifier_minor = char2d_to_string1d(tmp); - tmp = charHost2d(); io.read( tmp , "minor_gases_lower" ); minor_gases_lower = char2d_to_string1d(tmp); - tmp = charHost2d(); io.read( tmp , "minor_gases_upper" ); minor_gases_upper = char2d_to_string1d(tmp); - tmp = charHost2d(); io.read( tmp , "scaling_gas_lower" ); scaling_gas_lower = char2d_to_string1d(tmp); - tmp = charHost2d(); io.read( tmp , "scaling_gas_upper" ); scaling_gas_upper = char2d_to_string1d(tmp); - - io.read( key_species , "key_species" ); - io.read( band_lims , "bnd_limits_wavenumber" ); - io.read( band2gpt , "bnd_limits_gpt" ); - io.read( press_ref , "press_ref" ); - io.read( temp_ref , "temp_ref" ); - io.read( temp_ref_p , "absorption_coefficient_ref_P" ); - io.read( temp_ref_t , "absorption_coefficient_ref_T" ); - io.read( press_ref_trop , "press_ref_trop" ); - io.read( kminor_lower , "kminor_lower" ); - io.read( kminor_upper , "kminor_upper" ); - io.read( minor_limits_gpt_lower , "minor_limits_gpt_lower" ); - io.read( minor_limits_gpt_upper , "minor_limits_gpt_upper" ); - io.read( minor_scales_with_density_lower , "minor_scales_with_density_lower" ); - io.read( minor_scales_with_density_upper , "minor_scales_with_density_upper" ); - io.read( scale_by_complement_lower , "scale_by_complement_lower" ); - io.read( scale_by_complement_upper , "scale_by_complement_upper" ); - io.read( kminor_start_lower , "kminor_start_lower" ); - io.read( kminor_start_upper , "kminor_start_upper" ); - io.read( vmr_ref , "vmr_ref" ); - io.read( kmajor , "kmajor" ); - - if (io.varExists("rayl_lower")) { - io.read( rayl_lower , "rayl_lower" ); - io.read( rayl_upper , "rayl_upper" ); - } - - // Initialize the gas optics class with data. The calls look slightly different depending - // on whether the radiation sources are internal to the atmosphere (longwave) or external (shortwave) - // gas_optics%load() returns a string; a non-empty string indicates an error. - if (io.varExists("totplnk")) { - // If there's a totplnk variable in the file, then it's a longwave (internal sources) type - realHost2d totplnk; - realHost4d planck_frac; - io.read( totplnk , "totplnk" ); - io.read( planck_frac , "plank_fraction" ); - kdist.load(available_gases, gas_names, key_species, band2gpt, band_lims, press_ref, press_ref_trop, - temp_ref, temp_ref_p, temp_ref_t, vmr_ref, kmajor, kminor_lower, kminor_upper, - gas_minor, identifier_minor, minor_gases_lower, minor_gases_upper, - minor_limits_gpt_lower, minor_limits_gpt_upper, minor_scales_with_density_lower, - minor_scales_with_density_upper, scaling_gas_lower, scaling_gas_upper, - scale_by_complement_lower, scale_by_complement_upper, kminor_start_lower, - kminor_start_upper, totplnk, planck_frac, rayl_lower, rayl_upper); - } else { - // Otherwise, it's a shortwave type - realHost1d solar_src; - try { - if (io.varExists("solar_source")) { - io.read( solar_src , "solar_source" ); - } else if (io.varExists("solar_source_quiet")) { - // Newer RRTMGP input data files include components of solar source function sufficient to - // compute solar variability; ignore this for now, and simply read in the baseline solar source. - // This seems to get us closer to the original average insolation from - // the full spectral resolution data, but there is disagreement between - // this and EAM RRTMG for some reason. - // TODO: Ultimately, we need to be able to use solar source data consistent with EAM (i.e., the - // input4MIPS data); this will require implementing a separate function - // here to read that data in and integrate the solar source over each - // wavelength/gpoint band - io.read( solar_src , "solar_source_quiet" ); - if (false) { - realHost1d solar_source_facular; - realHost1d solar_source_sunspot; - real mg_index; - real sb_index; - io.read( solar_source_facular , "solar_source_facular" ); - io.read( solar_source_sunspot , "solar_source_sunspot" ); - io.read( mg_index, "mg_default"); - io.read( sb_index, "sb_default"); - const real a_offset = 0.1495954; - const real b_offset = 0.00066696; - auto ngpt = solar_src.totElems(); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<1>(ngpt), YAKL_LAMBDA(int igpt) { - solar_src(igpt) = solar_src(igpt) - + (mg_index - a_offset) * solar_source_facular(igpt) - + (sb_index - b_offset) * solar_source_sunspot(igpt); - }); - } - } else { - throw 1; - } - } catch (int err) { - std::cout << "ERROR: no solar_source variable found in input data.\n"; - } - kdist.load(available_gases, gas_names, key_species, band2gpt, band_lims, press_ref, press_ref_trop, - temp_ref, temp_ref_p, temp_ref_t, vmr_ref, kmajor, kminor_lower, kminor_upper, - gas_minor, identifier_minor, minor_gases_lower, minor_gases_upper, - minor_limits_gpt_lower, minor_limits_gpt_upper, minor_scales_with_density_lower, - minor_scales_with_density_upper, scaling_gas_lower, scaling_gas_upper, - scale_by_complement_lower, scale_by_complement_upper, kminor_start_lower, - kminor_start_upper, solar_src, rayl_lower, rayl_upper); - } - io.close(); -} - - diff --git a/components/eamxx/src/physics/rrtmgp/mo_load_coefficients.h b/components/eamxx/src/physics/rrtmgp/mo_load_coefficients.h deleted file mode 100644 index 6e2c191cdbd7..000000000000 --- a/components/eamxx/src/physics/rrtmgp/mo_load_coefficients.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef MO_LOAD_COEFFICIENTS_HPP -#define MO_LOAD_COEFFICIENTS_HPP - -#include "cpp/rrtmgp_const.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" -#include - -void load_and_init(GasOpticsRRTMGP &kdist, std::string filename, GasConcs const &available_gases); - -#endif diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 887bf5416f49..6c7dd91f5d1b 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -3,83 +3,137 @@ #include "physics/share/physics_constants.hpp" #include "cpp/rrtmgp_const.h" + +#ifdef RRTMGP_ENABLE_YAKL #include "YAKL.h" #include "YAKL_Bounds_fortran.h" +#endif namespace scream { - namespace rrtmgp { +namespace rrtmgp { - // Things we need from YAKL - using yakl::intrinsics::maxval; - using yakl::intrinsics::minval; - using yakl::intrinsics::count; - using yakl::intrinsics::sum; - - // Provide a routine to compute heating due to radiative fluxes. This is - // computed as net flux into a layer, converted to a heating rate. It is - // the responsibility of the user to ensure fields are passed with the - // proper units. I.e., pressure at level interfaces should be in Pa, - // fluxes in W m-2, Cpair in J kg-1 K-1, gravit in m s-2. This will give - // heating in units of K s-1. - // TODO: we should probably update this to use the pseudo-density pdel instead - // of approximating pdel by differencing the level interface pressures. - // We are leaving this for the time being for consistency with SCREAMv0, - // from which this code was directly ported. - template void compute_heating_rate ( - yakl::Array const &flux_up, yakl::Array const &flux_dn, - yakl::Array const &pdel , yakl::Array &heating_rate - ) { - using physconst = scream::physics::Constants; - auto ncol = flux_up.dimension[0]; - auto nlay = flux_up.dimension[1]-1; - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { - heating_rate(icol,ilay) = ( - flux_up(icol,ilay+1) - flux_up(icol,ilay) - - flux_dn(icol,ilay+1) + flux_dn(icol,ilay) - ) * physconst::gravit / (physconst::Cpair * pdel(icol,ilay)); - }); - } +#ifdef RRTMGP_ENABLE_YAKL +// Things we need from YAKL +using yakl::intrinsics::maxval; +using yakl::intrinsics::minval; +using yakl::intrinsics::count; +using yakl::intrinsics::sum; +#endif - inline bool radiation_do(const int irad, const int nstep) { - // If irad == 0, then never do radiation; - // Otherwise, we always call radiation at the first step, - // and afterwards we do radiation if the timestep is divisible - // by irad - if (irad == 0) { - return false; - } else { - return ( (nstep == 0) || (nstep % irad == 0) ); - } - } +// Provide a routine to compute heating due to radiative fluxes. This is +// computed as net flux into a layer, converted to a heating rate. It is +// the responsibility of the user to ensure fields are passed with the +// proper units. I.e., pressure at level interfaces should be in Pa, +// fluxes in W m-2, Cpair in J kg-1 K-1, gravit in m s-2. This will give +// heating in units of K s-1. +// TODO: we should probably update this to use the pseudo-density pdel instead +// of approximating pdel by differencing the level interface pressures. +// We are leaving this for the time being for consistency with SCREAMv0, +// from which this code was directly ported. +#ifdef RRTMGP_ENABLE_YAKL +template +void compute_heating_rate ( + yakl::Array const &flux_up, yakl::Array const &flux_dn, + yakl::Array const &pdel , yakl::Array &heating_rate + ) { + using physconst = scream::physics::Constants; + auto ncol = flux_up.dimension[0]; + auto nlay = flux_up.dimension[1]-1; + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { + heating_rate(icol,ilay) = ( + flux_up(icol,ilay+1) - flux_up(icol,ilay) - + flux_dn(icol,ilay+1) + flux_dn(icol,ilay) + ) * physconst::gravit / (physconst::Cpair * pdel(icol,ilay)); + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +template +void compute_heating_rate ( + Kokkos::View const &flux_up, + Kokkos::View const &flux_dn, + Kokkos::View const &pdel , + Kokkos::View const &heating_rate) +{ + using physconst = scream::physics::Constants; + auto ncol = flux_up.dimension[0]; + auto nlay = flux_up.dimension[1]-1; + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + heating_rate(icol,ilay) = ( + flux_up(icol,ilay+1) - flux_up(icol,ilay) - + flux_dn(icol,ilay+1) + flux_dn(icol,ilay) + ) * physconst::gravit / (physconst::Cpair * pdel(icol,ilay)); + }); +} +#endif +inline bool radiation_do(const int irad, const int nstep) { + // If irad == 0, then never do radiation; + // Otherwise, we always call radiation at the first step, + // and afterwards we do radiation if the timestep is divisible + // by irad + if (irad == 0) { + return false; + } else { + return ( (nstep == 0) || (nstep % irad == 0) ); + } +} - // Verify that array only contains values within valid range, and if not - // report min and max of array - template bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { - bool pass = true; - auto _xmin = minval(x); - auto _xmax = maxval(x); - if (_xmin < xmin or _xmax > xmax) { - // How many outside range? - auto bad_mask = x.createDeviceCopy(); - memset(bad_mask, 0); - yakl::c::parallel_for(yakl::c::SimpleBounds<1>(x.totElems()), YAKL_LAMBDA (int i) { - if (x.data()[i] < xmin or x.data()[i] > xmax) { - bad_mask.data()[i] = 1; - } - }); - auto num_bad = sum(bad_mask); - pass = false; - out << msg << ": " - << num_bad << " values outside range " - << "[" << xmin << "," << xmax << "]" - << "; minval = " << _xmin - << "; maxval = " << _xmax << "\n"; - } - return pass; +// Verify that array only contains values within valid range, and if not +// report min and max of array +#ifdef RRTMGP_ENABLE_YAKL +template >::type* = nullptr> +bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = minval(x); + auto _xmax = maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + auto bad_mask = x.createDeviceCopy(); + memset(bad_mask, 0); + yakl::c::parallel_for(yakl::c::SimpleBounds<1>(x.totElems()), YAKL_LAMBDA (int i) { + if (x.data()[i] < xmin or x.data()[i] > xmax) { + bad_mask.data()[i] = 1; } + }); + auto num_bad = sum(bad_mask); + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + return pass; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +template >::type* = nullptr> +bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + bool1dk bad_mask("bad_mask", x.size()); + Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { + if (x.data()[i] < xmin or x.data()[i] > xmax) { + bad_mask.data()[i] = true; + } + }); + auto num_bad = conv::sum(bad_mask); + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + return pass; +} +#endif - } +} // namespace rrtmgp +} // namespace scream -} #endif diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index ec3460293951..51c415fc027f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,7 +1,7 @@ #include "scream_rrtmgp_interface.hpp" #include "rrtmgp_utils.hpp" -#include "mo_load_coefficients.h" -#include "mo_load_cloud_coefficients.h" +#include "cpp/examples/mo_load_coefficients.h" +#include "examples/all-sky/mo_load_cloud_coefficients.h" #include "cpp/rrtmgp/mo_gas_concentrations.h" #include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" @@ -9,1129 +9,2234 @@ #include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" #include "ekat/util/ekat_math_utils.hpp" +#ifdef RRTMGP_ENABLE_KOKKOS +#include "Kokkos_Random.hpp" +#endif namespace scream { - void yakl_init () - { - // Initialize yakl - if(!yakl::isInitialized()) { yakl::init(); } + +#ifdef RRTMGP_ENABLE_YAKL +void yakl_init () +{ + // Initialize yakl + if(!yakl::isInitialized()) { yakl::init(); } +} + +void yakl_finalize() +{ + // Finalize YAKL + yakl::finalize(); +} +#endif + +namespace rrtmgp { + +#ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::merge; +#endif + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +#ifdef RRTMGP_ENABLE_YAKL +extern GasOpticsRRTMGP k_dist_sw; +extern GasOpticsRRTMGP k_dist_lw; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern GasOpticsRRTMGPK k_dist_sw_k; +extern GasOpticsRRTMGPK k_dist_lw_k; +#endif + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +#ifdef RRTMGP_ENABLE_YAKL +extern CloudOptics cloud_optics_sw; +extern CloudOptics cloud_optics_lw; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern CloudOpticsK cloud_optics_sw_k; +extern CloudOpticsK cloud_optics_lw_k; +#endif + +bool initialized = false; + +// local functions +namespace { + +#ifdef RRTMGP_ENABLE_YAKL +OpticalProps2str get_cloud_optics_sw( + const int ncol, const int nlay, + CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei) { + + // Initialize optics + OpticalProps2str clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = real2d("rel_limited", ncol, nlay); + auto rei_limited = real2d("rei_limited", ncol, nlay); + limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +OpticalProps2strK get_cloud_optics_sw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { + + // Initialize optics + OpticalProps2strK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = real2dk("rel_limited", ncol, nlay); + auto rei_limited = real2dk("rei_limited", ncol, nlay); + limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} +#endif + +#ifdef RRTMGP_ENABLE_YAKL +OpticalProps1scl get_cloud_optics_lw( + const int ncol, const int nlay, + CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei) { + + // Initialize optics + OpticalProps1scl clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = real2d("rel_limited", ncol, nlay); + auto rei_limited = real2d("rei_limited", ncol, nlay); + limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +OpticalProps1sclK get_cloud_optics_lw( + const int ncol, const int nlay, + CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { + + // Initialize optics + OpticalProps1sclK clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = real2dk("rel_limited", ncol, nlay); + auto rei_limited = real2dk("rei_limited", ncol, nlay); + limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + // Return optics + return clouds; +} +#endif + +#ifdef RRTMGP_ENABLE_YAKL +OpticalProps2str get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { + // Initialized subsampled optics + OpticalProps2str subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = real2d("cldfrac_rad", ncol, nlay); + memset(cldfrac_rad, 0.0); // Start with all zeros + parallel_for(SimpleBounds<3>(nbnd,nlay,ncol), YAKL_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = int1d("seeds", ncol); + parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay) - int(p_lay(icol,nlay))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +OpticalProps2strK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { + // Initialized subsampled optics + OpticalProps2strK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = int1dk("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay) - int(p_lay(icol,nlay))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; } + }); + return subsampled_optics; +} +#endif - void yakl_finalize() - { - // Finalize YAKL - yakl::finalize(); +#ifdef RRTMGP_ENABLE_YAKL +OpticalProps1scl get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { + // Initialized subsampled optics + OpticalProps1scl subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = real2d("cldfrac_rad", ncol, nlay); + memset(cldfrac_rad, 0.0); // Start with all zeros + parallel_for(SimpleBounds<3>(nbnd,nlay,ncol), YAKL_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = int1d("seeds", ncol); + parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +OpticalProps1sclK get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { + // Initialized subsampled optics + OpticalProps1sclK subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); } - namespace rrtmgp { - - using yakl::fortran::parallel_for; - using yakl::fortran::SimpleBounds; - using yakl::intrinsics::merge; - - OpticalProps2str get_cloud_optics_sw(const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei); - OpticalProps1scl get_cloud_optics_lw(const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei); - OpticalProps2str get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay); - OpticalProps1scl get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay); - - /* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ - GasOpticsRRTMGP k_dist_sw; - GasOpticsRRTMGP k_dist_lw; - - /* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ - CloudOptics cloud_optics_sw; - CloudOptics cloud_optics_lw; - - /* - * Gas concentrations. We want to initialize this once, since it will - * determine the absorbing gases that we keep track of in the k-dist - * objects. So we will initialize once, and then just update values - * at runtime. - */ - //GasConcs gas_concs; - - bool initialized = false; - - /* - * The following routines provide a simple interface to RRTMGP. These - * can be used as-is, but are intended to be wrapped by the SCREAM AD - * interface to radiation. - */ - void rrtmgp_initialize(GasConcs &gas_concs, - const std::string coefficients_file_sw, const std::string coefficients_file_lw, - const std::string cloud_optics_file_sw, const std::string cloud_optics_file_lw, - const std::shared_ptr& logger) { - - // If we've already initialized, just exit - if (initialized) { - if (logger) - logger->info("RRTMGP is already initialized; skipping\n"); - return; - } - - // Initialize YAKL - if (!yakl::isInitialized()) { yakl::init(); } - - // Load and initialize absorption coefficient data - load_and_init(k_dist_sw, coefficients_file_sw, gas_concs); - load_and_init(k_dist_lw, coefficients_file_lw, gas_concs); - - // Load and initialize cloud optical property look-up table information - load_cld_lutcoeff(cloud_optics_sw, cloud_optics_file_sw); - load_cld_lutcoeff(cloud_optics_lw, cloud_optics_file_lw); - - // We are now initialized! - initialized = true; - } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = int1dk("seeds", ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + return subsampled_optics; +} +#endif - void rrtmgp_finalize() { - initialized = false; - k_dist_sw.finalize(); - k_dist_lw.finalize(); - cloud_optics_sw.finalize(); //~CloudOptics(); - cloud_optics_lw.finalize(); //~CloudOptics(); - } +} + +/* + * The following routines provide a simple interface to RRTMGP. These + * can be used as-is, but are intended to be wrapped by the SCREAM AD + * interface to radiation. + */ +#ifdef RRTMGP_ENABLE_YAKL +void rrtmgp_initialize(GasConcs &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) { + + // If we've already initialized, just exit + if (initialized) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize YAKL + if (!yakl::isInitialized()) { yakl::init(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw, cloud_optics_file_lw); + + // We are now initialized! + initialized = true; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void rrtmgp_initialize( + GasConcsK &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) +{ + // If we've already initialized, just exit + if (initialized) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize YAKL + if (!Kokkos::is_initialized()) { Kokkos::initialize(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + + // We are now initialized! + initialized = true; +} +#endif - void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, - real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, - real2d &sfc_alb_dir, real2d &sfc_alb_dif) { - EKAT_ASSERT_MSG(initialized, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); - auto wavenumber_limits = k_dist_sw.get_band_lims_wavenumber(); +void rrtmgp_finalize() { + initialized = false; +#ifdef RRTMGP_ENABLE_YAKL + k_dist_sw.finalize(); + k_dist_lw.finalize(); + cloud_optics_sw.finalize(); //~CloudOptics(); + cloud_optics_lw.finalize(); //~CloudOptics(); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + k_dist_sw_k.finalize(); + k_dist_lw_k.finalize(); + cloud_optics_sw_k.finalize(); //~CloudOptics(); + cloud_optics_lw_k.finalize(); //~CloudOptics(); +#endif +} - EKAT_ASSERT_MSG(yakl::intrinsics::size(wavenumber_limits, 1) == 2, - "Error! 1st dimension for wavenumber_limits should be 2."); - EKAT_ASSERT_MSG(yakl::intrinsics::size(wavenumber_limits, 2) == nswbands, - "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); +#ifdef RRTMGP_ENABLE_YAKL +void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, + real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, + real2d &sfc_alb_dir, real2d &sfc_alb_dif) { - // Loop over bands, and determine for each band whether it is broadly in the - // visible or infrared part of the spectrum (visible or "not visible") - parallel_for(SimpleBounds<2>(nswbands, ncol), YAKL_LAMBDA(const int ibnd, const int icol) { + EKAT_ASSERT_MSG(initialized, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw.get_band_lims_wavenumber(); - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; + EKAT_ASSERT_MSG(yakl::intrinsics::size(wavenumber_limits, 1) == 2, + "Error! 1st dimension for wavenumber_limits should be 2."); + EKAT_ASSERT_MSG(yakl::intrinsics::size(wavenumber_limits, 2) == nswbands, + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(2, ibnd) > visible_wavenumber_threshold ? true : false); + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + parallel_for(SimpleBounds<2>(nswbands, ncol), YAKL_LAMBDA(const int ibnd, const int icol) { - if (is_visible_wave1 && is_visible_wave2) { + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const real visible_wavenumber_threshold = 14286; - // Entire band is in the visible - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(2, ibnd) > visible_wavenumber_threshold ? true : false); - } else if (!is_visible_wave1 && !is_visible_wave2) { + if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); - } else { + } else if (!is_visible_wave1 && !is_visible_wave2) { - // Band straddles the visible to near-infrared transition, so we take - // the albedo to be the average of the visible and near-infrared - // broadband albedos - sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); - sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); - } - }); - } + } else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); - void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , - real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, - real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir) { - // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums - // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when - // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This - // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. - //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - - // Initialize sums over bands - memset(sfc_flux_dir_nir, 0); - memset(sfc_flux_dir_vis, 0); - memset(sfc_flux_dif_nir, 0); - memset(sfc_flux_dif_vis, 0); - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - auto wavenumber_limits = k_dist_sw.get_band_lims_wavenumber(); - parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(const int icol) { - for (int ibnd = 1; ibnd <= nswbands; ++ibnd) { - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(2, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - - // Entire band is in the visible - sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - - } else if (!is_visible_wave1 && !is_visible_wave2) { - - // Entire band is in the longwave (near-infrared) - sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - - } else { - - // Band straddles the visible to near-infrared transition, so put half - // the flux in visible and half in near-infrared fluxes - sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - } - } - }); + } + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, + real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, + real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { + + EKAT_ASSERT_MSG(initialized, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + + EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, + "Error! 1st dimension for wavenumber_limits should be 2."); + EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); + + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + Kokkos::parallel_for(conv::get_mdrp<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const real visible_wavenumber_threshold = 14286; + + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + } + else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + } + }); +} +#endif + +#ifdef RRTMGP_ENABLE_YAKL +void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , + real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, + real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir) { + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + memset(sfc_flux_dir_nir, 0); + memset(sfc_flux_dir_vis, 0); + memset(sfc_flux_dif_nir, 0); + memset(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const real visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw.get_band_lims_wavenumber(); + parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(const int icol) { + for (int ibnd = 1; ibnd <= nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(2, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + + } else if (!is_visible_wave1 && !is_visible_wave2) { + + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + + } else { + + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); } + } + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , + real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, + real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir) { + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + Kokkos::deep_copy(sfc_flux_dir_nir, 0); + Kokkos::deep_copy(sfc_flux_dir_vis, 0); + Kokkos::deep_copy(sfc_flux_dif_nir, 0); + Kokkos::deep_copy(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const real visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { + for (int ibnd = 0; ibnd < nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else { + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + } + } + }); +} +#endif - - void rrtmgp_main( - const int ncol, const int nlay, - real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, - real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cldfrac, - real3d &aer_tau_sw, real3d &aer_ssa_sw, real3d &aer_asm_sw, real3d &aer_tau_lw, - real3d &cld_tau_sw_bnd, real3d &cld_tau_lw_bnd, - real3d &cld_tau_sw_gpt, - real3d &cld_tau_lw_gpt, - real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dn_dir, - real2d &lw_flux_up, real2d &lw_flux_dn, - real2d &sw_clnclrsky_flux_up, real2d &sw_clnclrsky_flux_dn, real2d &sw_clnclrsky_flux_dn_dir, - real2d &sw_clrsky_flux_up, real2d &sw_clrsky_flux_dn, real2d &sw_clrsky_flux_dn_dir, - real2d &sw_clnsky_flux_up, real2d &sw_clnsky_flux_dn, real2d &sw_clnsky_flux_dn_dir, - real2d &lw_clnclrsky_flux_up, real2d &lw_clnclrsky_flux_dn, - real2d &lw_clrsky_flux_up, real2d &lw_clrsky_flux_dn, - real2d &lw_clnsky_flux_up, real2d &lw_clnsky_flux_dn, - real3d &sw_bnd_flux_up, real3d &sw_bnd_flux_dn, real3d &sw_bnd_flux_dn_dir, - real3d &lw_bnd_flux_up, real3d &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { +#ifdef RRTMGP_ENABLE_YAKL +void rrtmgp_main( + const int ncol, const int nlay, + real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, + GasConcs &gas_concs, + real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cldfrac, + real3d &aer_tau_sw, real3d &aer_ssa_sw, real3d &aer_asm_sw, real3d &aer_tau_lw, + real3d &cld_tau_sw_bnd, real3d &cld_tau_lw_bnd, + real3d &cld_tau_sw_gpt, + real3d &cld_tau_lw_gpt, + real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dn_dir, + real2d &lw_flux_up, real2d &lw_flux_dn, + real2d &sw_clnclrsky_flux_up, real2d &sw_clnclrsky_flux_dn, real2d &sw_clnclrsky_flux_dn_dir, + real2d &sw_clrsky_flux_up, real2d &sw_clrsky_flux_dn, real2d &sw_clrsky_flux_dn_dir, + real2d &sw_clnsky_flux_up, real2d &sw_clnsky_flux_dn, real2d &sw_clnsky_flux_dn_dir, + real2d &lw_clnclrsky_flux_up, real2d &lw_clnclrsky_flux_dn, + real2d &lw_clrsky_flux_up, real2d &lw_clrsky_flux_dn, + real2d &lw_clnsky_flux_up, real2d &lw_clnsky_flux_dn, + real3d &sw_bnd_flux_up, real3d &sw_bnd_flux_dn, real3d &sw_bnd_flux_dn_dir, + real3d &lw_bnd_flux_up, real3d &lw_bnd_flux_dn, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { #ifdef SCREAM_RRTMGP_DEBUG - // Sanity check inputs, and possibly repair - check_range(t_lay , k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), "rrtmgp_main::t_lay"); - check_range(t_lev , k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), "rrtmgp_main::t_lev"); - check_range(p_lay , k_dist_sw.get_press_min(), k_dist_sw.get_press_max(), "rrtmgp_main::p_lay"); - check_range(p_lev , k_dist_sw.get_press_min(), k_dist_sw.get_press_max(), "rrtmgp_main::p_lev"); - check_range(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); + // Sanity check inputs, and possibly repair + check_range(t_lay , k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), "rrtmgp_main::t_lay"); + check_range(t_lev , k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), "rrtmgp_main::t_lev"); + check_range(p_lay , k_dist_sw.get_press_min(), k_dist_sw.get_press_max(), "rrtmgp_main::p_lay"); + check_range(p_lev , k_dist_sw.get_press_min(), k_dist_sw.get_press_max(), "rrtmgp_main::p_lev"); + check_range(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); #endif - // Setup pointers to RRTMGP SW fluxes - FluxesByband fluxes_sw; - fluxes_sw.flux_up = sw_flux_up; - fluxes_sw.flux_dn = sw_flux_dn; - fluxes_sw.flux_dn_dir = sw_flux_dn_dir; - fluxes_sw.bnd_flux_up = sw_bnd_flux_up; - fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; - fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; - // Clean-clear-sky - FluxesBroadband clnclrsky_fluxes_sw; - clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; - clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; - clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; - // Clear-sky - FluxesBroadband clrsky_fluxes_sw; - clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; - clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; - clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; - // Clean-sky - FluxesBroadband clnsky_fluxes_sw; - clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; - clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; - clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; - - // Setup pointers to RRTMGP LW fluxes - FluxesByband fluxes_lw; - fluxes_lw.flux_up = lw_flux_up; - fluxes_lw.flux_dn = lw_flux_dn; - fluxes_lw.bnd_flux_up = lw_bnd_flux_up; - fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; - // Clean-clear-sky - FluxesBroadband clnclrsky_fluxes_lw; - clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; - clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; - // Clear-sky - FluxesBroadband clrsky_fluxes_lw; - clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; - clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; - // Clean-sky - FluxesBroadband clnsky_fluxes_lw; - clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; - clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; - - auto nswbands = k_dist_sw.get_nband(); - auto nlwbands = k_dist_lw.get_nband(); - - // Setup aerosol optical properties - OpticalProps2str aerosol_sw; - OpticalProps1scl aerosol_lw; - aerosol_sw.init(k_dist_sw.get_band_lims_wavenumber()); - aerosol_sw.alloc_2str(ncol, nlay); - parallel_for(SimpleBounds<3>(nswbands,nlay,ncol) , YAKL_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); - aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); - aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); - }); - aerosol_lw.init(k_dist_lw.get_band_lims_wavenumber()); - aerosol_lw.alloc_1scl(ncol, nlay); - parallel_for(SimpleBounds<3>(nlwbands,nlay,ncol) , YAKL_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); - }); + // Setup pointers to RRTMGP SW fluxes + FluxesByband fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadband clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + FluxesBroadband clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadband clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + FluxesByband fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadband clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + FluxesBroadband clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadband clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw.get_nband(); + auto nlwbands = k_dist_lw.get_nband(); + + // Setup aerosol optical properties + OpticalProps2str aerosol_sw; + OpticalProps1scl aerosol_lw; + aerosol_sw.init(k_dist_sw.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + parallel_for(SimpleBounds<3>(nswbands,nlay,ncol) , YAKL_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + parallel_for(SimpleBounds<3>(nlwbands,nlay,ncol) , YAKL_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); #ifdef SCREAM_RRTMGP_DEBUG - // Check aerosol optical properties - // NOTE: these should already have been checked by precondition checks, but someday we might have - // non-trivial aerosol optics, so this is still good to do here. - check_range(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); #endif - // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2str clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw, k_dist_sw, lwp, iwp, rel, rei); - OpticalProps1scl clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw, k_dist_lw, lwp, iwp, rel, rei); - clouds_sw.tau.deep_copy_to(cld_tau_sw_bnd); - clouds_lw.tau.deep_copy_to(cld_tau_lw_bnd); - - // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; - // This implements the Monte Carlo Independing Column Approximation by mapping only a single - // subcolumn (cloud state) to each gpoint. - auto nswgpts = k_dist_sw.get_ngpt(); - auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw, cldfrac, p_lay); - // Longwave - auto nlwgpts = k_dist_lw.get_ngpt(); - auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw, cldfrac, p_lay); - - // Copy cloud properties to outputs (is this needed, or can we just use pointers?) - // Alternatively, just compute and output a subcolumn cloud mask - parallel_for(SimpleBounds<3>(nswgpts, nlay, ncol), YAKL_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); - }); - parallel_for(SimpleBounds<3>(nlwgpts, nlay, ncol), YAKL_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); - }); + // Convert cloud physical properties to optical properties for input to RRTMGP + OpticalProps2str clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw, k_dist_sw, lwp, iwp, rel, rei); + OpticalProps1scl clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw, k_dist_lw, lwp, iwp, rel, rei); + clouds_sw.tau.deep_copy_to(cld_tau_sw_bnd); + clouds_lw.tau.deep_copy_to(cld_tau_lw_bnd); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + parallel_for(SimpleBounds<3>(nswgpts, nlay, ncol), YAKL_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + parallel_for(SimpleBounds<3>(nlwgpts, nlay, ncol), YAKL_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); #ifdef SCREAM_RRTMGP_DEBUG - // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, - // but we might want to provide additional debug info here. NOTE: we may actually want to move this - // up higher in the code, I think optical props should go up higher since optical props are kind of - // a parameterization of their own, and we might want to swap different choices. These checks go here - // only because we need to run them on computed optical props, so if the optical props themselves get - // computed up higher, then perform these checks higher as well - check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); #endif - // Do shortwave - rrtmgp_sw( - ncol, nlay, - k_dist_sw, p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, - tsi_scaling, logger, - extra_clnclrsky_diag, extra_clnsky_diag + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag ); - // Do longwave - rrtmgp_lw( - ncol, nlay, - k_dist_lw, p_lay, t_lay, p_lev, t_lev, gas_concs, - aerosol_lw, clouds_lw_gpt, - fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, - extra_clnclrsky_diag, extra_clnsky_diag + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag ); - - } - int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3d("cldx", ncol, nlay, ngpt); - - // Apply overlap assumption to set cldx - if (overlap_option == 0) { // Dummy mask, always cloudy - memset(cldx, 1); - } else { // Default case, maximum-random overlap - // Maximum-random overlap: - // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, - // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same - // algorithm used in RRTMG implementation of maximum-random overlap (see - // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) - // - // First, fill cldx with random numbers. Need to use a unique seed for each column! - parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - yakl::Random rand(seeds(icol)); - for (int igpt = 1; igpt <= ngpt; igpt++) { - for (int ilay = 1; ilay <= nlay; ilay++) { - cldx(icol,ilay,igpt) = rand.genFP(); - } - } - }); - // Step down columns and apply algorithm from eq (14) - parallel_for(SimpleBounds<2>(ngpt,ncol), YAKL_LAMBDA(int igpt, int icol) { - for (int ilay = 2; ilay <= nlay; ilay++) { - // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn - if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { - // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent - // layers are maximimally overlapped - cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); - } else { - // Cloud-less above, use new random number so that clouds are distributed - // randomly in this layer. Need to scale new random number to range - // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution - // of random numbers in this layer with the above branch of the conditional, - // which would otherwise inflate cloud fraction in this layer. - cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); - } - } - }); - } - - // Use cldx array to create subcolumn mask - parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { - if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { - subcolumn_mask(icol,ilay,igpt) = 1; - } else { - subcolumn_mask(icol,ilay,igpt) = 0; - } - }); - return subcolumn_mask; - } +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void rrtmgp_main( + const int ncol, const int nlay, + real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, + GasConcsK &gas_concs, + real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, + real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, + real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, + real3dk &cld_tau_sw_gpt, + real3dk &cld_tau_lw_gpt, + real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, + real2dk &lw_flux_up, real2dk &lw_flux_dn, + real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, + real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, + real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, + real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, + real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, + real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, + real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, + real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { +#ifdef SCREAM_RRTMGP_DEBUG + // Sanity check inputs, and possibly repair + check_range(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); +#endif - OpticalProps2str get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { - // Initialized subsampled optics - OpticalProps2str subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_2str(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2d("cldfrac_rad", ncol, nlay); - memset(cldfrac_rad, 0.0); // Start with all zeros - parallel_for(SimpleBounds<3>(nbnd,nlay,ncol), YAKL_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, - // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), - // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should - // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very - // high resolution. - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = int1d("seeds", ncol); - parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay) - int(p_lay(icol,nlay))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); - subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - subsampled_optics.ssa(icol,ilay,igpt) = 0; - subsampled_optics.g (icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; - } + // Setup pointers to RRTMGP SW fluxes + FluxesBybandK fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + FluxesBybandK fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + FluxesBroadbandK clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + FluxesBroadbandK clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + FluxesBroadbandK clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw_k.get_nband(); + auto nlwbands = k_dist_lw_k.get_nband(); + + // Setup aerosol optical properties + OpticalProps2strK aerosol_sw; + OpticalProps1sclK aerosol_lw; + aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); +#ifdef SCREAM_RRTMGP_DEBUG + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); +#endif + // Convert cloud physical properties to optical properties for input to RRTMGP + OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); + Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw_k.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw_k.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + Kokkos::parallel_for(conv::get_mdrp<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + Kokkos::parallel_for(conv::get_mdrp<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); - OpticalProps1scl get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { - // Initialized subsampled optics - OpticalProps1scl subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_1scl(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2d("cldfrac_rad", ncol, nlay); - memset(cldfrac_rad, 0.0); // Start with all zeros - parallel_for(SimpleBounds<3>(nbnd,nlay,ncol), YAKL_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM; use different - // seed values for longwave and shortwave - auto seeds = int1d("seeds", ncol); - parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; - } +#ifdef SCREAM_RRTMGP_DEBUG + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); +#endif + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag + ); + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag + ); - OpticalProps2str get_cloud_optics_sw( - const int ncol, const int nlay, - CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei) { - - // Initialize optics - OpticalProps2str clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_2str(ncol, nlay); +} +#endif - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2d("rel_limited", ncol, nlay); - auto rei_limited = real2d("rei_limited", ncol, nlay); - limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); +#ifdef RRTMGP_ENABLE_YAKL +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { + + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); + + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3d("cldx", ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + memset(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + yakl::Random rand(seeds(icol)); + for (int igpt = 1; igpt <= ngpt; igpt++) { + for (int ilay = 1; ilay <= nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + // Step down columns and apply algorithm from eq (14) + parallel_for(SimpleBounds<2>(ngpt,ncol), YAKL_LAMBDA(int igpt, int icol) { + for (int ilay = 2; ilay <= nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + return subcolumn_mask; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds) { + + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); + + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3dk("cldx", ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + Kokkos::deep_copy(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + auto seeds_host = Kokkos::create_mirror_view(seeds); + Kokkos::deep_copy(seeds_host, seeds); + for (int icol = 0; icol < ncol; ++icol) { + Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + auto generator = random_pool.get_state(); + cldx(icol,ilay,igpt) = generator.drand(0., 1.); + random_pool.free_state(generator); + }); + } + // Step down columns and apply algorithm from eq (14) + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + for (int ilay = 1; ilay < nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + return subcolumn_mask; +} +#endif - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); +#ifdef RRTMGP_ENABLE_YAKL +void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGP &k_dist, + real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, + GasConcs &gas_concs, + real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, + OpticalProps2str &aerosol, OpticalProps2str &clouds, + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { + + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + parallel_for(SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + parallel_for(SimpleBounds<3>(nbnd,nlay+1,ncol), YAKL_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = int1d("dayIndices", ncol); + memset(dayIndices, -1); + // Loop below has to be done on host, so create host copies + // TODO: there is probably a way to do this on the device + auto dayIndices_h = dayIndices.createHostCopy(); + auto mu0_h = mu0.createHostCopy(); + int nday = 0; + for (int icol = 1; icol <= ncol; icol++) { + if (mu0_h(icol) > 0) { + nday++; + dayIndices_h(nday) = icol; + } + } + // Copy data back to the device + dayIndices_h.deep_copy_to(dayIndices); + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + auto mu0_day = real1d("mu0_day", nday); + parallel_for(SimpleBounds<1>(nday), YAKL_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + auto p_lay_day = real2d("p_lay_day", nday, nlay); + auto t_lay_day = real2d("t_lay_day", nday, nlay); + parallel_for(SimpleBounds<2>(nlay,nday), YAKL_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + auto p_lev_day = real2d("p_lev_day", nday, nlay+1); + auto t_lev_day = real2d("t_lev_day", nday, nlay+1); + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + GasConcs gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 1; igas <= ngas; igas++) { + auto vmr_day = real2d("vmr_day", nday, nlay); + auto vmr = real2d("vmr" , ncol, nlay); + gas_concs.get_vmr(gas_names[igas], vmr); + parallel_for(SimpleBounds<2>(nlay,nday), YAKL_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + OpticalProps2str aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + parallel_for(SimpleBounds<3>(nbnd,nlay,nday), YAKL_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + OpticalProps2str clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + parallel_for(SimpleBounds<3>(ngpt,nlay,nday), YAKL_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + real2d sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); + real2d sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); + parallel_for(SimpleBounds<2>(nbnd,nday), YAKL_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + auto flux_up_day = real2d("flux_up_day", nday, nlay+1); + auto flux_dn_day = real2d("flux_dn_day", nday, nlay+1); + auto flux_dn_dir_day = real2d("flux_dn_dir_day", nday, nlay+1); + auto bnd_flux_up_day = real3d("bnd_flux_up_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_day = real3d("bnd_flux_dn_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_dir_day = real3d("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); + FluxesByband fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + OpticalProps2str optics; + optics.alloc_2str(nday, nlay, k_dist); + + OpticalProps2str optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = real2d("t_lay_limited", nday, nlay); + limit_to_bounds(t_lay_day, k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), t_lay_limited); + + // Do gas optics + real2d toa_flux("toa_flux", nday, ngpt); + auto p_lay_host = p_lay.createHostCopy(); + bool top_at_1 = p_lay_host(1, 1) < p_lay_host(1, nlay); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics_no_aerosols, toa_flux); + } - // Return optics - return clouds; - } +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif + + // Apply tsi_scaling + parallel_for(SimpleBounds<2>(ngpt,nday), YAKL_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + parallel_for(SimpleBounds<3>(nbnd,nlay+1,nday), YAKL_LAMBDA(int ibnd, int ilev, int iday) { + int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { + int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, + GasConcsK &gas_concs, + real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, + OpticalProps2strK &aerosol, OpticalProps2strK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { + + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = int1dk("dayIndices", ncol); + Kokkos::deep_copy(dayIndices, -1); + + int nday = 0; + // Serialized for now. + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { + for (int icol = 0; icol < ncol; ++icol) { + if (mu0(icol) > 0) { + ++nday_inner; + } + dayIndices(nday_inner) = icol; + } + }, Kokkos::Sum(nday)); + + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + auto mu0_day = real1dk("mu0_day", nday); + Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + auto p_lay_day = real2dk("p_lay_day", nday, nlay); + auto t_lay_day = real2dk("t_lay_day", nday, nlay); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + auto p_lev_day = real2dk("p_lev_day", nday, nlay+1); + auto t_lev_day = real2dk("t_lev_day", nday, nlay+1); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + GasConcsK gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 0; igas < ngas; igas++) { + auto vmr_day = real2dk("vmr_day", nday, nlay); + auto vmr = real2dk("vmr" , ncol, nlay); + gas_concs.get_vmr(gas_names[igas], vmr); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + OpticalProps2strK aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + OpticalProps2strK clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + real2dk sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); + real2dk sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); + Kokkos::parallel_for(conv::get_mdrp<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + auto flux_up_day = real2dk("flux_up_day", nday, nlay+1); + auto flux_dn_day = real2dk("flux_dn_day", nday, nlay+1); + auto flux_dn_dir_day = real2dk("flux_dn_dir_day", nday, nlay+1); + auto bnd_flux_up_day = real3dk("bnd_flux_up_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_day = real3dk("bnd_flux_dn_day", nday, nlay+1, nbnd); + auto bnd_flux_dn_dir_day = real3dk("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); + FluxesBybandK fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + OpticalProps2strK optics; + optics.alloc_2str(nday, nlay, k_dist); + + OpticalProps2strK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); + limit_to_bounds(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + + // Do gas optics + real2dk toa_flux("toa_flux", nday, ngpt); + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif - OpticalProps1scl get_cloud_optics_lw( - const int ncol, const int nlay, - CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei) { + // Apply tsi_scaling + Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { + const int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } +} +#endif - // Initialize optics - OpticalProps1scl clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! +#ifdef RRTMGP_ENABLE_YAKL +void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGP &k_dist, + real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, + GasConcs &gas_concs, + OpticalProps1scl &aerosol, + OpticalProps1scl &clouds, + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { + + // Problem size + int nbnd = k_dist.get_nband(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + parallel_for( + SimpleBounds<2>(nlay + 1, ncol), YAKL_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + parallel_for( + SimpleBounds<3>(nbnd, nlay + 1, ncol), + YAKL_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + OpticalProps1scl optics; + optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1scl optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + SourceFuncLW lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + real1d t_sfc ("t_sfc" ,ncol); + real2d emis_sfc("emis_sfc",nbnd,ncol); + + // Surface temperature + auto p_lay_host = p_lay.createHostCopy(); + bool top_at_1 = p_lay_host(1, 1) < p_lay_host(1, nlay); + parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, merge(nlay+1, 1, top_at_1)); + }); + memset(emis_sfc , 0.98_wp); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + int constexpr max_gauss_pts = 4; + realHost2d gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + gauss_Ds_host(1,1) = 1.66_wp ; gauss_Ds_host(2,1) = 0._wp; gauss_Ds_host(3,1) = 0._wp; gauss_Ds_host(4,1) = 0._wp; + gauss_Ds_host(1,2) = 1.18350343_wp; gauss_Ds_host(2,2) = 2.81649655_wp; gauss_Ds_host(3,2) = 0._wp; gauss_Ds_host(4,2) = 0._wp; + gauss_Ds_host(1,3) = 1.09719858_wp; gauss_Ds_host(2,3) = 1.69338507_wp; gauss_Ds_host(3,3) = 4.70941630_wp; gauss_Ds_host(4,3) = 0._wp; + gauss_Ds_host(1,4) = 1.06056257_wp; gauss_Ds_host(2,4) = 1.38282560_wp; gauss_Ds_host(3,4) = 2.40148179_wp; gauss_Ds_host(4,4) = 7.15513024_wp; + + realHost2d gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_wts_host(1,1) = 0.5_wp ; gauss_wts_host(2,1) = 0._wp ; gauss_wts_host(3,1) = 0._wp ; gauss_wts_host(4,1) = 0._wp ; + gauss_wts_host(1,2) = 0.3180413817_wp; gauss_wts_host(2,2) = 0.1819586183_wp; gauss_wts_host(3,2) = 0._wp ; gauss_wts_host(4,2) = 0._wp ; + gauss_wts_host(1,3) = 0.2009319137_wp; gauss_wts_host(2,3) = 0.2292411064_wp; gauss_wts_host(3,3) = 0.0698269799_wp; gauss_wts_host(4,3) = 0._wp ; + gauss_wts_host(1,4) = 0.1355069134_wp; gauss_wts_host(2,4) = 0.2034645680_wp; gauss_wts_host(3,4) = 0.1298475476_wp; gauss_wts_host(4,4) = 0.0311809710_wp; + + real2d gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + real2d gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_Ds_host .deep_copy_to(gauss_Ds ); + gauss_wts_host.deep_copy_to(gauss_wts); + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = real2d("t_lay_limited", ncol, nlay); + auto t_lev_limited = real2d("t_lev_limited", ncol, nlay+1); + limit_to_bounds(t_lay, k_dist_lw.get_temp_min(), k_dist_lw.get_temp_max(), t_lay_limited); + limit_to_bounds(t_lev, k_dist_lw.get_temp_min(), k_dist_lw.get_temp_max(), t_lev_limited); + + // Do gas optics + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, optics, lw_sources, real2d(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, optics_no_aerosols, lw_sources, real2d(), t_lev_limited); + } - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); +#endif - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2d("rel_limited", ncol, nlay); - auto rei_limited = real2d("rei_limited", ncol, nlay); - limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + // Combine gas and aerosol optics + aerosol.increment(optics); - // Return optics - return clouds; - } + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); + + // Combine gas and cloud optics + clouds.increment(optics); + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGP &k_dist, - real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, - real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, - OpticalProps2str &aerosol, OpticalProps2str &clouds, - FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &flux_dn_dir = fluxes.flux_dn_dir; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; - - // Reset fluxes to zero - parallel_for(SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilev, int icol) { - flux_up (icol,ilev) = 0; - flux_dn (icol,ilev) = 0; - flux_dn_dir(icol,ilev) = 0; - clnclrsky_flux_up (icol,ilev) = 0; - clnclrsky_flux_dn (icol,ilev) = 0; - clnclrsky_flux_dn_dir(icol,ilev) = 0; - clrsky_flux_up (icol,ilev) = 0; - clrsky_flux_dn (icol,ilev) = 0; - clrsky_flux_dn_dir(icol,ilev) = 0; - clnsky_flux_up (icol,ilev) = 0; - clnsky_flux_dn (icol,ilev) = 0; - clnsky_flux_dn_dir(icol,ilev) = 0; - }); - parallel_for(SimpleBounds<3>(nbnd,nlay+1,ncol), YAKL_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up (icol,ilev,ibnd) = 0; - bnd_flux_dn (icol,ilev,ibnd) = 0; - bnd_flux_dn_dir(icol,ilev,ibnd) = 0; - }); - - // Get daytime indices - auto dayIndices = int1d("dayIndices", ncol); - memset(dayIndices, -1); - // Loop below has to be done on host, so create host copies - // TODO: there is probably a way to do this on the device - auto dayIndices_h = dayIndices.createHostCopy(); - auto mu0_h = mu0.createHostCopy(); - int nday = 0; - for (int icol = 1; icol <= ncol; icol++) { - if (mu0_h(icol) > 0) { - nday++; - dayIndices_h(nday) = icol; - } - } - // Copy data back to the device - dayIndices_h.deep_copy_to(dayIndices); - if (nday == 0) { - // No daytime columns in this chunk, skip the rest of this routine - return; - } - - // Subset mu0 - auto mu0_day = real1d("mu0_day", nday); - parallel_for(SimpleBounds<1>(nday), YAKL_LAMBDA(int iday) { - mu0_day(iday) = mu0(dayIndices(iday)); - }); - - // subset state variables - auto p_lay_day = real2d("p_lay_day", nday, nlay); - auto t_lay_day = real2d("t_lay_day", nday, nlay); - parallel_for(SimpleBounds<2>(nlay,nday), YAKL_LAMBDA(int ilay, int iday) { - p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); - t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); - }); - auto p_lev_day = real2d("p_lev_day", nday, nlay+1); - auto t_lev_day = real2d("t_lev_day", nday, nlay+1); - parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { - p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); - t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); - }); - - // Subset gases - auto gas_names = gas_concs.get_gas_names(); - GasConcs gas_concs_day; - gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 1; igas <= ngas; igas++) { - auto vmr_day = real2d("vmr_day", nday, nlay); - auto vmr = real2d("vmr" , ncol, nlay); - gas_concs.get_vmr(gas_names(igas), vmr); - parallel_for(SimpleBounds<2>(nlay,nday), YAKL_LAMBDA(int ilay, int iday) { - vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); - }); - gas_concs_day.set_vmr(gas_names(igas), vmr_day); - } - - // Subset aerosol optics - OpticalProps2str aerosol_day; - aerosol_day.init(k_dist.get_band_lims_wavenumber()); - aerosol_day.alloc_2str(nday, nlay); - parallel_for(SimpleBounds<3>(nbnd,nlay,nday), YAKL_LAMBDA(int ibnd, int ilay, int iday) { - aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); - aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); - aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); - }); - - // Subset cloud optics - // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2str clouds_day; - clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); - clouds_day.alloc_2str(nday, nlay); - parallel_for(SimpleBounds<3>(ngpt,nlay,nday), YAKL_LAMBDA(int igpt, int ilay, int iday) { - clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); - clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); - clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); - }); - - // RRTMGP assumes surface albedos have a screwy dimension ordering - // for some strange reason, so we need to transpose these; also do - // daytime subsetting in the same kernel - real2d sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - real2d sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); - parallel_for(SimpleBounds<2>(nbnd,nday), YAKL_LAMBDA(int ibnd, int icol) { - sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); - sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); - }); - - // Temporaries we need for daytime-only fluxes - auto flux_up_day = real2d("flux_up_day", nday, nlay+1); - auto flux_dn_day = real2d("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = real2d("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = real3d("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = real3d("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = real3d("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesByband fluxes_day; - fluxes_day.flux_up = flux_up_day; - fluxes_day.flux_dn = flux_dn_day; - fluxes_day.flux_dn_dir = flux_dn_dir_day; - fluxes_day.bnd_flux_up = bnd_flux_up_day; - fluxes_day.bnd_flux_dn = bnd_flux_dn_day; - fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; - - // Allocate space for optical properties - OpticalProps2str optics; - optics.alloc_2str(nday, nlay, k_dist); - - OpticalProps2str optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_2str(nday, nlay, k_dist); - } - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2d("t_lay_limited", nday, nlay); - limit_to_bounds(t_lay_day, k_dist_sw.get_temp_min(), k_dist_sw.get_temp_max(), t_lay_limited); - - // Do gas optics - real2d toa_flux("toa_flux", nday, ngpt); - auto p_lay_host = p_lay.createHostCopy(); - bool top_at_1 = p_lay_host(1, 1) < p_lay_host(1, nlay); - - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics, toa_flux); - if (extra_clnsky_diag) { - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics_no_aerosols, toa_flux); - } + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, + GasConcsK &gas_concs, + OpticalProps1sclK &aerosol, + OpticalProps1sclK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { + + // Problem size + int nbnd = k_dist.get_nband(); + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + Kokkos::parallel_for( + conv::get_mdrp<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + Kokkos::parallel_for( + conv::get_mdrp<3>({nbnd, nlay + 1, ncol}), + KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + OpticalProps1sclK optics; + optics.alloc_1scl(ncol, nlay, k_dist); + OpticalProps1sclK optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + SourceFuncLWK lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + real1dk t_sfc ("t_sfc" ,ncol); + real2dk emis_sfc("emis_sfc",nbnd,ncol); + + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + // Surface temperature + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); + }); + Kokkos::deep_copy(emis_sfc , 0.98); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + int constexpr max_gauss_pts = 4; + realHost2dk gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; + gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; + gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; + gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; + + realHost2dk gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); + gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; + gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; + gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; + gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; + + real2dk gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); + real2dk gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); + Kokkos::deep_copy(gauss_wts, gauss_wts_host); + + // Limit temperatures for gas optics look-up tables + auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); + auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); + limit_to_bounds(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + + // Do gas optics + realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, real2dk(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, real2dk(), t_lev_limited); + } #ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); + // Check gas optics + check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); #endif - // Apply tsi_scaling - parallel_for(SimpleBounds<2>(ngpt,nday), YAKL_LAMBDA(int igpt, int iday) { - toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); - }); - - if (extra_clnclrsky_diag) { - // Compute clear-clean-sky (just gas) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { - int icol = dayIndices(iday); - clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } - - // Combine gas and aerosol optics - aerosol_day.delta_scale(); - aerosol_day.increment(optics); - - // Compute clearsky (gas + aerosol) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { - int icol = dayIndices(iday); - clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - - // Now merge in cloud optics and do allsky calculations - - // Combine gas and cloud optics - clouds_day.delta_scale(); - clouds_day.increment(optics); - // Compute fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { - int icol = dayIndices(iday); - flux_up (icol,ilev) = flux_up_day (iday,ilev); - flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - parallel_for(SimpleBounds<3>(nbnd,nlay+1,nday), YAKL_LAMBDA(int ibnd, int ilev, int iday) { - int icol = dayIndices(iday); - bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); - bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); - bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); - }); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds_day.increment(optics_no_aerosols); - // Compute cleansky (gas + clouds) fluxes on daytime columns - rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { - int icol = dayIndices(iday); - clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } - } + // Combine gas and aerosol optics + aerosol.increment(optics); - void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGP &k_dist, - real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, - OpticalProps1scl &aerosol, - OpticalProps1scl &clouds, - FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Problem size - int nbnd = k_dist.get_nband(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - - // Reset fluxes to zero - parallel_for( - SimpleBounds<2>(nlay + 1, ncol), YAKL_LAMBDA(int ilev, int icol) { - flux_up(icol, ilev) = 0; - flux_dn(icol, ilev) = 0; - clnclrsky_flux_up(icol, ilev) = 0; - clnclrsky_flux_dn(icol, ilev) = 0; - clrsky_flux_up(icol, ilev) = 0; - clrsky_flux_dn(icol, ilev) = 0; - clnsky_flux_up(icol, ilev) = 0; - clnsky_flux_dn(icol, ilev) = 0; - }); - parallel_for( - SimpleBounds<3>(nbnd, nlay + 1, ncol), - YAKL_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up(icol, ilev, ibnd) = 0; - bnd_flux_dn(icol, ilev, ibnd) = 0; - }); - - // Allocate space for optical properties - OpticalProps1scl optics; - optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1scl optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); - } - - // Boundary conditions - SourceFuncLW lw_sources; - lw_sources.alloc(ncol, nlay, k_dist); - real1d t_sfc ("t_sfc" ,ncol); - real2d emis_sfc("emis_sfc",nbnd,ncol); - - // Surface temperature - auto p_lay_host = p_lay.createHostCopy(); - bool top_at_1 = p_lay_host(1, 1) < p_lay_host(1, nlay); - parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - t_sfc(icol) = t_lev(icol, merge(nlay+1, 1, top_at_1)); - }); - memset(emis_sfc , 0.98_wp); - - // Get Gaussian quadrature weights - // TODO: move this crap out of userland! - // Weights and angle secants for first order (k=1) Gaussian quadrature. - // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 - // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - realHost2d gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(1,1) = 1.66_wp ; gauss_Ds_host(2,1) = 0._wp; gauss_Ds_host(3,1) = 0._wp; gauss_Ds_host(4,1) = 0._wp; - gauss_Ds_host(1,2) = 1.18350343_wp; gauss_Ds_host(2,2) = 2.81649655_wp; gauss_Ds_host(3,2) = 0._wp; gauss_Ds_host(4,2) = 0._wp; - gauss_Ds_host(1,3) = 1.09719858_wp; gauss_Ds_host(2,3) = 1.69338507_wp; gauss_Ds_host(3,3) = 4.70941630_wp; gauss_Ds_host(4,3) = 0._wp; - gauss_Ds_host(1,4) = 1.06056257_wp; gauss_Ds_host(2,4) = 1.38282560_wp; gauss_Ds_host(3,4) = 2.40148179_wp; gauss_Ds_host(4,4) = 7.15513024_wp; - - realHost2d gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(1,1) = 0.5_wp ; gauss_wts_host(2,1) = 0._wp ; gauss_wts_host(3,1) = 0._wp ; gauss_wts_host(4,1) = 0._wp ; - gauss_wts_host(1,2) = 0.3180413817_wp; gauss_wts_host(2,2) = 0.1819586183_wp; gauss_wts_host(3,2) = 0._wp ; gauss_wts_host(4,2) = 0._wp ; - gauss_wts_host(1,3) = 0.2009319137_wp; gauss_wts_host(2,3) = 0.2292411064_wp; gauss_wts_host(3,3) = 0.0698269799_wp; gauss_wts_host(4,3) = 0._wp ; - gauss_wts_host(1,4) = 0.1355069134_wp; gauss_wts_host(2,4) = 0.2034645680_wp; gauss_wts_host(3,4) = 0.1298475476_wp; gauss_wts_host(4,4) = 0.0311809710_wp; - - real2d gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - real2d gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_Ds_host .deep_copy_to(gauss_Ds ); - gauss_wts_host.deep_copy_to(gauss_wts); - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2d("t_lay_limited", ncol, nlay); - auto t_lev_limited = real2d("t_lev_limited", ncol, nlay+1); - limit_to_bounds(t_lay, k_dist_lw.get_temp_min(), k_dist_lw.get_temp_max(), t_lay_limited); - limit_to_bounds(t_lev, k_dist_lw.get_temp_min(), k_dist_lw.get_temp_max(), t_lev_limited); - - // Do gas optics - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, optics, lw_sources, real2d(), t_lev_limited); - if (extra_clnsky_diag) { - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, optics_no_aerosols, lw_sources, real2d(), t_lev_limited); - } + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); -#endif + // Combine gas and cloud optics + clouds.increment(optics); - if (extra_clnclrsky_diag) { - // Compute clean-clear-sky fluxes before we add in aerosols and clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); - } + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - // Combine gas and aerosol optics - aerosol.increment(optics); + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } - // Compute clear-sky fluxes before we add in clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); +} +#endif - // Combine gas and cloud optics - clouds.increment(optics); +#ifdef RRTMGP_ENABLE_YAKL +void compute_cloud_area( + int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, + const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = real2d("subcol_mask", ncol, ngpt); + memset(subcol_mask, 0); + yakl::fortran::parallel_for(SimpleBounds<3>(ngpt, nlay, ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + memset(cld_area, 0); + yakl::fortran::parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 1; igpt <= ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void compute_cloud_area( + int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, + const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area) { + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = real2dk("subcol_mask", ncol, ngpt); + Kokkos::parallel_for(conv::get_mdrp<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + Kokkos::deep_copy(cld_area, 0); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 0; igpt < ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); +} +#endif - // Compute allsky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); +#ifdef RRTMGP_ENABLE_YAKL +int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds.increment(optics_no_aerosols); - // Compute clean-sky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); - } +int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - } +int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } +#endif - void compute_cloud_area( - int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, - const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { - // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy - // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = real2d("subcol_mask", ncol, ngpt); - memset(subcol_mask, 0); - yakl::fortran::parallel_for(SimpleBounds<3>(ngpt, nlay, ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { - // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but - // using play/pmid does not - if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { - subcol_mask(icol,igpt) = 1; - } - }); - // Compute average over subcols to get cloud area - auto ngpt_inv = 1.0 / ngpt; - memset(cld_area, 0); - yakl::fortran::parallel_for(SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - // This loop needs to be serial because of the atomic reduction - for (int igpt = 1; igpt <= ngpt; ++igpt) { - cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; - } - }); +#ifdef RRTMGP_ENABLE_YAKL +int get_wavelength_index(OpticalProps &kdist, double wavelength) { + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + int nbnds = kdist.get_nband(); + yakl::ScalarLiveOut band_index(-1); + yakl::fortran::parallel_for(SimpleBounds<1>(nbnds), YAKL_LAMBDA(int ibnd) { + if (wavelength_bounds(1,ibnd) < wavelength_bounds(2,ibnd)) { + if (wavelength_bounds(1,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(2,ibnd)) { + band_index = ibnd; } - - int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } - - int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } - - int get_wavelength_index(OpticalProps &kdist, double wavelength) { - // Get wavelength bounds for all wavelength bands - auto wavelength_bounds = kdist.get_band_lims_wavelength(); - - // Find the band index for the specified wavelength - // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength - // in units of meters, we need a conversion factor of 10^2 - int nbnds = kdist.get_nband(); - yakl::ScalarLiveOut band_index(-1); - yakl::fortran::parallel_for(SimpleBounds<1>(nbnds), YAKL_LAMBDA(int ibnd) { - if (wavelength_bounds(1,ibnd) < wavelength_bounds(2,ibnd)) { - if (wavelength_bounds(1,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(2,ibnd)) { - band_index = ibnd; - } - } else { - if (wavelength_bounds(1,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(2,ibnd)) { - band_index = ibnd; - } - } - }); - return band_index.hostRead(); + } else { + if (wavelength_bounds(1,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(2,ibnd)) { + band_index = ibnd; } + } + }); + return band_index.hostRead(); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +int get_wavelength_index(OpticalPropsK &kdist, double wavelength) { + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + const int nbnds = kdist.get_nband(); + int band_index = -1; + Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { + if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { + if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } else { + if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } + }, Kokkos::Max(band_index)); + return band_index; +} +#endif - void compute_aerocom_cloudtop( - int ncol, int nlay, const real2d &tmid, const real2d &pmid, - const real2d &p_del, const real2d &z_del, const real2d &qc, - const real2d &qi, const real2d &rel, const real2d &rei, - const real2d &cldfrac_tot, const real2d &nc, - real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, - real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, - real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, - real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCOM recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - memset(T_mid_at_cldtop, 0.0); - memset(p_mid_at_cldtop, 0.0); - memset(cldfrac_ice_at_cldtop, 0.0); - memset(cldfrac_liq_at_cldtop, 0.0); - memset(cldfrac_tot_at_cldtop, 0.0); - memset(cdnc_at_cldtop, 0.0); - memset(eff_radius_qc_at_cldtop, 0.0); - memset(eff_radius_qi_at_cldtop, 0.0); - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1d("aerocom_clr", ncol); - memset(aerocom_clr, 1.0); - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - // Loop over all columns in parallel - yakl::fortran::parallel_for( - SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 2; ilay <= nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); +#ifdef RRTMGP_ENABLE_YAKL +void compute_aerocom_cloudtop( + int ncol, int nlay, const real2d &tmid, const real2d &pmid, + const real2d &p_del, const real2d &z_del, const real2d &qc, + const real2d &qi, const real2d &rel, const real2d &rei, + const real2d &cldfrac_tot, const real2d &nc, + real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, + real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, + real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, + real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop) { + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCOM recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + memset(T_mid_at_cldtop, 0.0); + memset(p_mid_at_cldtop, 0.0); + memset(cldfrac_ice_at_cldtop, 0.0); + memset(cldfrac_liq_at_cldtop, 0.0); + memset(cldfrac_tot_at_cldtop, 0.0); + memset(cdnc_at_cldtop, 0.0); + memset(eff_radius_qc_at_cldtop, 0.0); + memset(eff_radius_qi_at_cldtop, 0.0); + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = real1d("aerocom_clr", ncol); + memset(aerocom_clr, 1.0); + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + // TODO: move tunable constant to namelist + constexpr real q_threshold = 0.0; // BAD_CONSTANT! + // TODO: move tunable constant to namelist + constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + // Loop over all columns in parallel + yakl::fortran::parallel_for( + SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 2; ilay <= nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; } - } // namespace rrtmgp + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void compute_aerocom_cloudtop( + int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, + const real2dk &p_del, const real2dk &z_del, const real2dk &qc, + const real2dk &qi, const real2dk &rel, const real2dk &rei, + const real2dk &cldfrac_tot, const real2dk &nc, + real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, + real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, + real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, + real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCOM recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + Kokkos::deep_copy(T_mid_at_cldtop, 0.0); + Kokkos::deep_copy(p_mid_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); + Kokkos::deep_copy(cdnc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = real1dk("aerocom_clr", ncol); + Kokkos::deep_copy(aerocom_clr, 1.0); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + + // TODO: move tunable constant to namelist + constexpr real q_threshold = 0.0; // BAD_CONSTANT! + + // TODO: move tunable constant to namelist + constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + + // Loop over all columns in parallel + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 1; ilay < nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); +} +#endif + +} // namespace rrtmgp } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 65e165422865..102de74e71db 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -12,174 +12,354 @@ #include "ekat/logging/ekat_logger.hpp" namespace scream { - void yakl_init (); - void yakl_finalize(); - namespace rrtmgp { - /* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ - extern GasOpticsRRTMGP k_dist_sw; - extern GasOpticsRRTMGP k_dist_lw; - /* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ - extern CloudOptics cloud_optics_sw; - extern CloudOptics cloud_optics_lw; - /* - * Flag to indicate whether or not we have initialized RRTMGP - */ - extern bool initialized; - /* - * Initialize data for RRTMGP driver - */ - extern void rrtmgp_initialize(GasConcs &gas_concs, - const std::string coefficients_file_sw, const std::string coefficients_file_lw, - const std::string cloud_optics_file_sw, const std::string cloud_optics_file_lw, - const std::shared_ptr& logger); - /* - * Compute band-by-band surface albedos from broadband albedos. - */ - extern void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, - real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, - real2d &sfc_alb_dir, real2d &sfc_alb_dif); - /* - * Compute broadband visible/UV and near-infrared surface fluxes. - */ - extern void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , - real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, - real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); - /* - * Main driver code to run RRTMGP. - * The input logger is in charge of outputing info to - * screen and/or to file (or neither), depending on how it was set up. - */ - extern void rrtmgp_main( - const int ncol, const int nlay, - real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, - real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cldfrac, - real3d &aer_tau_sw, real3d &aer_ssa_sw, real3d &aer_asm_sw, real3d &aer_tau_lw, - real3d &cld_tau_sw_bnd, real3d &cld_tau_lw_bnd, - real3d &cld_tau_sw_gpt, real3d &cld_tau_lw_gpt, - real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dn_dir, - real2d &lw_flux_up, real2d &lw_flux_dn, - real2d &sw_clnclrsky_flux_up, real2d &sw_clnclrsky_flux_dn, real2d &sw_clnclrsky_flux_dn_dir, - real2d &sw_clrsky_flux_up, real2d &sw_clrsky_flux_dn, real2d &sw_clrsky_flux_dn_dir, - real2d &sw_clnsky_flux_up, real2d &sw_clnsky_flux_dn, real2d &sw_clnsky_flux_dn_dir, - real2d &lw_clnclrsky_flux_up, real2d &lw_clnclrsky_flux_dn, - real2d &lw_clrsky_flux_up, real2d &lw_clrsky_flux_dn, - real2d &lw_clnsky_flux_up, real2d &lw_clnsky_flux_dn, - real3d &sw_bnd_flux_up, real3d &sw_bnd_flux_dn, real3d &sw_bnd_flux_dn_dir, - real3d &lw_bnd_flux_up, real3d &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); - /* - * Perform any clean-up tasks - */ - extern void rrtmgp_finalize(); - /* - * Shortwave driver (called by rrtmgp_main) - */ - extern void rrtmgp_sw(const int ncol, const int nlay, - GasOpticsRRTMGP &k_dist, - real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, - real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, - OpticalProps2str &aerosol, OpticalProps2str &clouds, - FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); - /* - * Longwave driver (called by rrtmgp_main) - */ - extern void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGP &k_dist, - real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, - GasConcs &gas_concs, - OpticalProps1scl &aerosol, OpticalProps1scl &clouds, - FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); - /* - * Return a subcolumn mask consistent with a specified overlap assumption - */ - int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); - /* - * Compute cloud area from 3d subcol cloud property - */ - void compute_cloud_area( - int ncol, int nlay, int ngpt, Real pmin, Real pmax, - const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); - /* - * Return select cloud-top diagnostics following AeroCOM recommendation - */ - void compute_aerocom_cloudtop( - int ncol, int nlay, const real2d &tmid, const real2d &pmid, - const real2d &p_del, const real2d &z_del, const real2d &qc, - const real2d &qi, const real2d &rel, const real2d &rei, - const real2d &cldfrac_tot, const real2d &nc, - real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, - real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, - real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, - real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); - - /* - * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area - * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion - * with the standard practice of using "water path" to refer to the total column-integrated - * quantities). - */ - template void mixing_ratio_to_cloud_mass( - yakl::Array const &mixing_ratio, - yakl::Array const &cloud_fraction, - yakl::Array const &dp, - yakl::Array &cloud_mass) { - int ncol = mixing_ratio.dimension[0]; - int nlay = mixing_ratio.dimension[1]; - using physconst = scream::physics::Constants; - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay, ncol), YAKL_LAMBDA(int ilay, int icol) { - // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) - // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency - // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics - // in EAM, this is consistent with limits in MG2. Is this true for P3? - if (cloud_fraction(icol,ilay) > 0) { - // Compute layer-integrated cloud mass (per unit area) - auto incloud_mixing_ratio = std::min(mixing_ratio(icol,ilay) / std::max(0.0001, cloud_fraction(icol,ilay)), 0.005); - cloud_mass(icol,ilay) = incloud_mixing_ratio * dp(icol,ilay) / physconst::gravit; - } else { - cloud_mass(icol,ilay) = 0; - } - }); - } - - /* - * Routine to limit a quantity to set bounds. Used to make sure - * effective radii are within the bounds of the cloud optical - * property look-up tables, but could be used to limit other - * fields as well. - */ - template void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { - yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); - }); - } - - int get_wavelength_index(OpticalProps &kdist, double wavelength); - int get_wavelength_index_sw(double wavelength); - int get_wavelength_index_lw(double wavelength); - - } // namespace rrtmgp -} // namespace scream + +void yakl_init (); +void yakl_finalize(); + +namespace rrtmgp { + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +#ifdef RRTMGP_ENABLE_YAKL +extern GasOpticsRRTMGP k_dist_sw; +extern GasOpticsRRTMGP k_dist_lw; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern GasOpticsRRTMGPK k_dist_sw_k; +extern GasOpticsRRTMGPK k_dist_lw_k; +#endif + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +#ifdef RRTMGP_ENABLE_YAKL +extern CloudOptics cloud_optics_sw; +extern CloudOptics cloud_optics_lw; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern CloudOpticsK cloud_optics_sw_k; +extern CloudOpticsK cloud_optics_lw_k; +#endif + +/* + * Flag to indicate whether or not we have initialized RRTMGP + */ +extern bool initialized; + +/* + * Initialize data for RRTMGP driver + */ +#ifdef RRTMGP_ENABLE_YAKL +extern void rrtmgp_initialize( + GasConcs &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern void rrtmgp_initialize( + GasConcsK &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger); +#endif + +/* + * Compute band-by-band surface albedos from broadband albedos. + */ +#ifdef RRTMGP_ENABLE_YAKL +extern void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, + real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, + real2d &sfc_alb_dir, real2d &sfc_alb_dif); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, + real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, + real2dk &sfc_alb_dir, real2dk &sfc_alb_dif); +#endif + +/* + * Compute broadband visible/UV and near-infrared surface fluxes. + */ +#ifdef RRTMGP_ENABLE_YAKL +extern void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , + real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, + real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , + real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, + real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir); +#endif + +/* + * Main driver code to run RRTMGP. + * The input logger is in charge of outputing info to + * screen and/or to file (or neither), depending on how it was set up. + */ +#ifdef RRTMGP_ENABLE_YAKL +extern void rrtmgp_main( + const int ncol, const int nlay, + real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, + GasConcs &gas_concs, + real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cldfrac, + real3d &aer_tau_sw, real3d &aer_ssa_sw, real3d &aer_asm_sw, real3d &aer_tau_lw, + real3d &cld_tau_sw_bnd, real3d &cld_tau_lw_bnd, + real3d &cld_tau_sw_gpt, real3d &cld_tau_lw_gpt, + real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dn_dir, + real2d &lw_flux_up, real2d &lw_flux_dn, + real2d &sw_clnclrsky_flux_up, real2d &sw_clnclrsky_flux_dn, real2d &sw_clnclrsky_flux_dn_dir, + real2d &sw_clrsky_flux_up, real2d &sw_clrsky_flux_dn, real2d &sw_clrsky_flux_dn_dir, + real2d &sw_clnsky_flux_up, real2d &sw_clnsky_flux_dn, real2d &sw_clnsky_flux_dn_dir, + real2d &lw_clnclrsky_flux_up, real2d &lw_clnclrsky_flux_dn, + real2d &lw_clrsky_flux_up, real2d &lw_clrsky_flux_dn, + real2d &lw_clnsky_flux_up, real2d &lw_clnsky_flux_dn, + real3d &sw_bnd_flux_up, real3d &sw_bnd_flux_dn, real3d &sw_bnd_flux_dn_dir, + real3d &lw_bnd_flux_up, real3d &lw_bnd_flux_dn, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern void rrtmgp_main( + const int ncol, const int nlay, + real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, + GasConcsK &gas_concs, + real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, + real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, + real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, + real3dk &cld_tau_sw_gpt, real3dk &cld_tau_lw_gpt, + real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, + real2dk &lw_flux_up, real2dk &lw_flux_dn, + real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, + real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, + real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, + real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, + real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, + real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, + real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, + real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); +#endif + +/* + * Perform any clean-up tasks + */ +extern void rrtmgp_finalize(); + +/* + * Shortwave driver (called by rrtmgp_main) + */ +#ifdef RRTMGP_ENABLE_YAKL +extern void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGP &k_dist, + real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, + GasConcs &gas_concs, + real2d &sfc_alb_dir, real2d &sfc_alb_dif, real1d &mu0, + OpticalProps2str &aerosol, OpticalProps2str &clouds, + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern void rrtmgp_sw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, + GasConcsK &gas_concs, + real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, + OpticalProps2strK &aerosol, OpticalProps2strK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); +#endif + +/* + * Longwave driver (called by rrtmgp_main) + */ +#ifdef RRTMGP_ENABLE_YAKL +extern void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGP &k_dist, + real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, + GasConcs &gas_concs, + OpticalProps1scl &aerosol, OpticalProps1scl &clouds, + FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern void rrtmgp_lw( + const int ncol, const int nlay, + GasOpticsRRTMGPK &k_dist, + real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, + GasConcsK &gas_concs, + OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, + FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); +#endif + +/* + * Return a subcolumn mask consistent with a specified overlap assumption + */ +#ifdef RRTMGP_ENABLE_YAKL +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds); +#endif + +/* + * Compute cloud area from 3d subcol cloud property + */ +#ifdef RRTMGP_ENABLE_YAKL +void compute_cloud_area( + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void compute_cloud_area( + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area); +#endif + +/* + * Return select cloud-top diagnostics following AeroCOM recommendation + */ +#ifdef RRTMGP_ENABLE_YAKL +void compute_aerocom_cloudtop( + int ncol, int nlay, const real2d &tmid, const real2d &pmid, + const real2d &p_del, const real2d &z_del, const real2d &qc, + const real2d &qi, const real2d &rel, const real2d &rei, + const real2d &cldfrac_tot, const real2d &nc, + real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, + real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, + real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, + real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void compute_aerocom_cloudtop( + int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, + const real2dk &p_del, const real2dk &z_del, const real2dk &qc, + const real2dk &qi, const real2dk &rel, const real2dk &rei, + const real2dk &cldfrac_tot, const real2dk &nc, + real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, + real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, + real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, + real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop); +#endif + +/* + * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area + * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion + * with the standard practice of using "water path" to refer to the total column-integrated + * quantities). + */ +#ifdef RRTMGP_ENABLE_YAKL +template +void mixing_ratio_to_cloud_mass( + yakl::Array const &mixing_ratio, + yakl::Array const &cloud_fraction, + yakl::Array const &dp, + yakl::Array &cloud_mass) { + int ncol = mixing_ratio.dimension[0]; + int nlay = mixing_ratio.dimension[1]; + using physconst = scream::physics::Constants; + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay, ncol), YAKL_LAMBDA(int ilay, int icol) { + // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) + // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency + // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics + // in EAM, this is consistent with limits in MG2. Is this true for P3? + if (cloud_fraction(icol,ilay) > 0) { + // Compute layer-integrated cloud mass (per unit area) + auto incloud_mixing_ratio = std::min(mixing_ratio(icol,ilay) / std::max(0.0001, cloud_fraction(icol,ilay)), 0.005); + cloud_mass(icol,ilay) = incloud_mixing_ratio * dp(icol,ilay) / physconst::gravit; + } else { + cloud_mass(icol,ilay) = 0; + } + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +template +void mixing_ratio_to_cloud_mass( + Kokkos::View const& mixing_ratio, + Kokkos::View const& cloud_fraction, + Kokkos::View const& dp, + Kokkos::View const& cloud_mass) +{ + int ncol = mixing_ratio.extent(0); + int nlay = mixing_ratio.extent(1); + using physconst = scream::physics::Constants; + Kokkos::parallel_for(conv::get_mdrp<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) + // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency + // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics + // in EAM, this is consistent with limits in MG2. Is this true for P3? + if (cloud_fraction(icol,ilay) > 0) { + // Compute layer-integrated cloud mass (per unit area) + auto incloud_mixing_ratio = std::min(mixing_ratio(icol,ilay) / std::max(0.0001, cloud_fraction(icol,ilay)), 0.005); + cloud_mass(icol,ilay) = incloud_mixing_ratio * dp(icol,ilay) / physconst::gravit; + } else { + cloud_mass(icol,ilay) = 0; + } + }); +} +#endif + +/* + * Routine to limit a quantity to set bounds. Used to make sure + * effective radii are within the bounds of the cloud optical + * property look-up tables, but could be used to limit other + * fields as well. + */ +#ifdef RRTMGP_ENABLE_YAKL +template>::type* = nullptr> +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +template>::type* = nullptr> +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} +#endif + +#ifdef RRTMGP_ENABLE_YAKL +int get_wavelength_index(OpticalProps &kdist, double wavelength); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +int get_wavelength_index(OpticalPropsK &kdist, double wavelength); +#endif + +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); + +} // namespace rrtmgp +} // namespace scream #endif // SCREAM_RRTMGP_INTERFACE_HPP From a0f51bd278d4796f795faa9a0235484203cd96f0 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 19 Apr 2024 14:41:56 -0600 Subject: [PATCH 041/476] progress --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 169 +++++++++++++++++- 1 file changed, 165 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 3778e32138c9..40be50c39fbf 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -223,6 +223,7 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) Real* mem = reinterpret_cast(buffer_manager.get_memory()); +#ifdef RRTMGP_ENABLE_YAKL // 1d arrays m_buffer.mu0 = decltype(m_buffer.mu0)("mu0", mem, m_col_chunk_size); mem += m_buffer.mu0.totElems(); @@ -362,6 +363,151 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_sw_bnd.totElems(); m_buffer.cld_tau_lw_bnd = decltype(m_buffer.cld_tau_lw_bnd)("cld_tau_lw_bnd", mem, m_col_chunk_size, m_nlay, m_nlwbands); mem += m_buffer.cld_tau_lw_bnd.totElems(); +#endif + + mem = reinterpret_cast(buffer_manager.get_memory()); + +#ifdef RRTMGP_ENABLE_KOKKOS + // 1d arrays + m_buffer.mu0_k = decltype(m_buffer.mu0_k)(mem, m_col_chunk_size); + mem += m_buffer.mu0_k.size(); + m_buffer.sfc_alb_dir_vis_k = decltype(m_buffer.sfc_alb_dir_vis_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_alb_dir_vis_k.size(); + m_buffer.sfc_alb_dir_nir_k = decltype(m_buffer.sfc_alb_dir_nir_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_alb_dir_nir_k.size(); + m_buffer.sfc_alb_dif_vis_k = decltype(m_buffer.sfc_alb_dif_vis_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_alb_dif_vis_k.size(); + m_buffer.sfc_alb_dif_nir_k = decltype(m_buffer.sfc_alb_dif_nir_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_alb_dif_nir_k.size(); + m_buffer.sfc_flux_dir_vis_k = decltype(m_buffer.sfc_flux_dir_vis_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_flux_dir_vis_k.size(); + m_buffer.sfc_flux_dir_nir_k = decltype(m_buffer.sfc_flux_dir_nir_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_flux_dir_nir_k.size(); + m_buffer.sfc_flux_dif_vis_k = decltype(m_buffer.sfc_flux_dif_vis_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_flux_dif_vis_k.size(); + m_buffer.sfc_flux_dif_nir_k = decltype(m_buffer.sfc_flux_dif_nir_k)(mem, m_col_chunk_size); + mem += m_buffer.sfc_flux_dif_nir_k.size(); + m_buffer.cosine_zenith_k = decltype(m_buffer.cosine_zenith_k)(mem, m_col_chunk_size); + mem += m_buffer.cosine_zenith_k.size(); + + // 2d arrays + m_buffer.p_lay_k = decltype(m_buffer.p_lay_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.p_lay_k.size(); + m_buffer.t_lay_k = decltype(m_buffer.t_lay_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.t_lay_k.size(); + m_buffer.z_del_k = decltype(m_buffer.z_del_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.z_del_k.size(); + m_buffer.p_del_k = decltype(m_buffer.p_del_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.p_del_k.size(); + m_buffer.qc_k = decltype(m_buffer.qc_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.qc_k.size(); + m_buffer.nc_k = decltype(m_buffer.nc_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.nc_k.size(); + m_buffer.qi_k = decltype(m_buffer.qi_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.qi_k.size(); + m_buffer.cldfrac_tot_k = decltype(m_buffer.cldfrac_tot_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.cldfrac_tot_k.size(); + m_buffer.eff_radius_qc_k = decltype(m_buffer.eff_radius_qc_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.eff_radius_qc_k.size(); + m_buffer.eff_radius_qi_k = decltype(m_buffer.eff_radius_qi_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.eff_radius_qi_k.size(); + m_buffer.tmp2d_k = decltype(m_buffer.tmp2d_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.tmp2d_k.size(); + m_buffer.lwp_k = decltype(m_buffer.lwp_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.lwp_k.size(); + m_buffer.iwp_k = decltype(m_buffer.iwp_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.iwp_k.size(); + m_buffer.sw_heating_k = decltype(m_buffer.sw_heating_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.sw_heating_k.size(); + m_buffer.lw_heating_k = decltype(m_buffer.lw_heating_k)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.lw_heating_k.size(); + m_buffer.p_lev_k = decltype(m_buffer.p_lev_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.p_lev_k.size(); + m_buffer.t_lev_k = decltype(m_buffer.t_lev_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.t_lev_k.size(); + m_buffer.d_tint_k = decltype(m_buffer.d_tint_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.d_tint_k.size(); + m_buffer.d_dz_k = decltype(m_buffer.d_dz_k )(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.d_dz_k.size(); + // 3d arrays + m_buffer.sw_flux_up_k = decltype(m_buffer.sw_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_flux_up_k.size(); + m_buffer.sw_flux_dn_k = decltype(m_buffer.sw_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_flux_dn_k.size(); + m_buffer.sw_flux_dn_dir_k = decltype(m_buffer.sw_flux_dn_dir_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_flux_dn_dir_k.size(); + m_buffer.lw_flux_up_k = decltype(m_buffer.lw_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_flux_up_k.size(); + m_buffer.lw_flux_dn_k = decltype(m_buffer.lw_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_flux_dn_k.size(); + m_buffer.sw_clnclrsky_flux_up_k = decltype(m_buffer.sw_clnclrsky_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnclrsky_flux_up_k.size(); + m_buffer.sw_clnclrsky_flux_dn_k = decltype(m_buffer.sw_clnclrsky_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnclrsky_flux_dn_k.size(); + m_buffer.sw_clnclrsky_flux_dn_dir_k = decltype(m_buffer.sw_clnclrsky_flux_dn_dir_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnclrsky_flux_dn_dir_k.size(); + m_buffer.sw_clrsky_flux_up_k = decltype(m_buffer.sw_clrsky_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clrsky_flux_up_k.size(); + m_buffer.sw_clrsky_flux_dn_k = decltype(m_buffer.sw_clrsky_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clrsky_flux_dn_k.size(); + m_buffer.sw_clrsky_flux_dn_dir_k = decltype(m_buffer.sw_clrsky_flux_dn_dir_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clrsky_flux_dn_dir_k.size(); + m_buffer.sw_clnsky_flux_up_k = decltype(m_buffer.sw_clnsky_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnsky_flux_up_k.size(); + m_buffer.sw_clnsky_flux_dn_k = decltype(m_buffer.sw_clnsky_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnsky_flux_dn_k.size(); + m_buffer.sw_clnsky_flux_dn_dir_k = decltype(m_buffer.sw_clnsky_flux_dn_dir_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.sw_clnsky_flux_dn_dir_k.size(); + m_buffer.lw_clnclrsky_flux_up_k = decltype(m_buffer.lw_clnclrsky_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnclrsky_flux_up_k.size(); + m_buffer.lw_clnclrsky_flux_dn_k = decltype(m_buffer.lw_clnclrsky_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnclrsky_flux_dn_k.size(); + m_buffer.lw_clrsky_flux_up_k = decltype(m_buffer.lw_clrsky_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clrsky_flux_up_k.size(); + m_buffer.lw_clrsky_flux_dn_k = decltype(m_buffer.lw_clrsky_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clrsky_flux_dn_k.size(); + m_buffer.lw_clnsky_flux_up_k = decltype(m_buffer.lw_clnsky_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnsky_flux_up_k.size(); + m_buffer.lw_clnsky_flux_dn_k = decltype(m_buffer.lw_clnsky_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.lw_clnsky_flux_dn_k.size(); + // 3d arrays with nswbands dimension (shortwave fluxes by band) + m_buffer.sw_bnd_flux_up_k = decltype(m_buffer.sw_bnd_flux_up_k)(mem, m_col_chunk_size, m_nlay+1, m_nswbands); + mem += m_buffer.sw_bnd_flux_up_k.size(); + m_buffer.sw_bnd_flux_dn_k = decltype(m_buffer.sw_bnd_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1, m_nswbands); + mem += m_buffer.sw_bnd_flux_dn_k.size(); + m_buffer.sw_bnd_flux_dir_k = decltype(m_buffer.sw_bnd_flux_dir_k)(mem, m_col_chunk_size, m_nlay+1, m_nswbands); + mem += m_buffer.sw_bnd_flux_dir_k.size(); + m_buffer.sw_bnd_flux_dif_k = decltype(m_buffer.sw_bnd_flux_dif_k)(mem, m_col_chunk_size, m_nlay+1, m_nswbands); + mem += m_buffer.sw_bnd_flux_dif_k.size(); + // 3d arrays with nlwbands dimension (longwave fluxes by band) + m_buffer.lw_bnd_flux_up_k = decltype(m_buffer.lw_bnd_flux_up_k)(mem, m_col_chunk_size, m_nlay+1, m_nlwbands); + mem += m_buffer.lw_bnd_flux_up_k.size(); + m_buffer.lw_bnd_flux_dn_k = decltype(m_buffer.lw_bnd_flux_dn_k)(mem, m_col_chunk_size, m_nlay+1, m_nlwbands); + mem += m_buffer.lw_bnd_flux_dn_k.size(); + // 2d arrays with extra nswbands dimension (surface albedos by band) + m_buffer.sfc_alb_dir_k = decltype(m_buffer.sfc_alb_dir_k)(mem, m_col_chunk_size, m_nswbands); + mem += m_buffer.sfc_alb_dir_k.size(); + m_buffer.sfc_alb_dif_k = decltype(m_buffer.sfc_alb_dif_k)(mem, m_col_chunk_size, m_nswbands); + mem += m_buffer.sfc_alb_dif_k.size(); + // 3d arrays with extra band dimension (aerosol optics by band) + m_buffer.aero_tau_sw_k = decltype(m_buffer.aero_tau_sw_k)(mem, m_col_chunk_size, m_nlay, m_nswbands); + mem += m_buffer.aero_tau_sw_k.size(); + m_buffer.aero_ssa_sw_k = decltype(m_buffer.aero_ssa_sw_k)(mem, m_col_chunk_size, m_nlay, m_nswbands); + mem += m_buffer.aero_ssa_sw_k.size(); + m_buffer.aero_g_sw_k = decltype(m_buffer.aero_g_sw_k )(mem, m_col_chunk_size, m_nlay, m_nswbands); + mem += m_buffer.aero_g_sw_k.size(); + m_buffer.aero_tau_lw_k = decltype(m_buffer.aero_tau_lw_k)(mem, m_col_chunk_size, m_nlay, m_nlwbands); + mem += m_buffer.aero_tau_lw_k.size(); + // 3d arrays with extra ngpt dimension (cloud optics by gpoint; primarily for debugging) + m_buffer.cld_tau_sw_gpt_k = decltype(m_buffer.cld_tau_sw_gpt_k)(mem, m_col_chunk_size, m_nlay, m_nswgpts); + mem += m_buffer.cld_tau_sw_gpt_k.size(); + m_buffer.cld_tau_lw_gpt_k = decltype(m_buffer.cld_tau_lw_gpt_k)(mem, m_col_chunk_size, m_nlay, m_nlwgpts); + mem += m_buffer.cld_tau_lw_gpt_k.size(); + m_buffer.cld_tau_sw_bnd_k = decltype(m_buffer.cld_tau_sw_bnd_k)(mem, m_col_chunk_size, m_nlay, m_nswbands); + mem += m_buffer.cld_tau_sw_bnd_k.size(); + m_buffer.cld_tau_lw_bnd_k = decltype(m_buffer.cld_tau_lw_bnd_k)(mem, m_col_chunk_size, m_nlay, m_nlwbands); + mem += m_buffer.cld_tau_lw_bnd_k.size(); +#endif size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for RRTMGPRadiation."); @@ -403,16 +549,15 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { #endif // Names of active gases - auto gas_names_yakl_offset = string1d("gas_names",m_ngas); + auto gas_names_yakl_offset = string1dv(m_ngas); m_gas_mol_weights = view_1d_real("gas_mol_weights",m_ngas); // the lookup function for getting the gas mol weights doesn't work on device auto gas_mol_w_host = Kokkos::create_mirror_view(m_gas_mol_weights); for (int igas = 0; igas < m_ngas; igas++) { const auto& gas_name = m_gas_names[igas]; - /* Note: YAKL starts the index from 1 */ - gas_names_yakl_offset(igas+1) = gas_name; - gas_mol_w_host[igas] = PC::get_gas_mol_weight(gas_name); + gas_names_yakl_offset[igas] = gas_name; + gas_mol_w_host[igas] = PC::get_gas_mol_weight(gas_name); } Kokkos::deep_copy(m_gas_mol_weights,gas_mol_w_host); @@ -422,6 +567,7 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { std::string coefficients_file_lw = m_params.get("rrtmgp_coefficients_file_lw"); std::string cloud_optics_file_sw = m_params.get("rrtmgp_cloud_optics_file_sw"); std::string cloud_optics_file_lw = m_params.get("rrtmgp_cloud_optics_file_lw"); +#ifdef RRTMGP_ENABLE_YAKL m_gas_concs.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); rrtmgp::rrtmgp_initialize( m_gas_concs, @@ -429,6 +575,16 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + m_gas_concs_k.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); + rrtmgp::rrtmgp_initialize( + m_gas_concs_k, + coefficients_file_sw, coefficients_file_lw, + cloud_optics_file_sw, cloud_optics_file_lw, + m_atm_logger + ); +#endif // Set property checks for fields in this process add_invariant_check(get_field_out("T_mid"),m_grid,100.0, 500.0,false); @@ -555,7 +711,12 @@ void RRTMGPRadiation::run_impl (const double dt) { // On each chunk, we internally "reset" the GasConcs object to subview the concs 3d array // with the correct ncol dimension. So let's keep a copy of the original (ref-counted) // array, to restore at the end inside the m_gast_concs object. +#ifdef RRTMGP_ENABLE_YAKL auto gas_concs = m_gas_concs.concs; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + auto gas_concs_k = m_gas_concs_k.concs; +#endif // Compute orbital parameters; these are used both for computing // the solar zenith angle and also for computing total solar From c0ebec98820db978af9b0bcdb1fea209cbead193 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 19 Apr 2024 15:10:12 -0600 Subject: [PATCH 042/476] progress --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 40be50c39fbf..1b8d1aea72fd 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -797,6 +797,7 @@ void RRTMGPRadiation::run_impl (const double dt) { // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. +#ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); }; @@ -867,10 +868,89 @@ void RRTMGPRadiation::run_impl (const double dt) { auto d_tint = m_buffer.d_tint; auto d_dz = m_buffer.d_dz; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + auto subview_1dk = [&](const real1dk v) -> real1dk { + return real1dk(v.data(),ncol); + }; + auto subview_2dk = [&](const real2dk v) -> real2dk { + return real2dk(v.data(),ncol,v.extent(0)); + }; + auto subview_3dk = [&](const real3dk v) -> real3dk { + return real3dk(v.data(),ncol,v.extent(0),v.extent(1)); + }; + + auto p_lay_k = subview_2dk(m_buffer.p_lay_k); + auto t_lay_k = subview_2dk(m_buffer.t_lay_k); + auto p_lev_k = subview_2dk(m_buffer.p_lev_k); + auto z_del_k = subview_2dk(m_buffer.z_del_k); + auto p_del_k = subview_2dk(m_buffer.p_del_k); + auto t_lev_k = subview_2dk(m_buffer.t_lev_k); + auto mu0_k = subview_1dk(m_buffer.mu0_k); + auto sfc_alb_dir_k = subview_2dk(m_buffer.sfc_alb_dir_k); + auto sfc_alb_dif_k = subview_2dk(m_buffer.sfc_alb_dif_k); + auto sfc_alb_dir_vis_k = subview_1dk(m_buffer.sfc_alb_dir_vis_k); + auto sfc_alb_dir_nir_k = subview_1dk(m_buffer.sfc_alb_dir_nir_k); + auto sfc_alb_dif_vis_k = subview_1dk(m_buffer.sfc_alb_dif_vis_k); + auto sfc_alb_dif_nir_k = subview_1dk(m_buffer.sfc_alb_dif_nir_k); + auto qc_k = subview_2dk(m_buffer.qc_k); + auto nc_k = subview_2dk(m_buffer.nc_k); + auto qi_k = subview_2dk(m_buffer.qi_k); + auto cldfrac_tot_k = subview_2dk(m_buffer.cldfrac_tot_k); + auto rel_k = subview_2dk(m_buffer.eff_radius_qc_k); + auto rei_k = subview_2dk(m_buffer.eff_radius_qi_k); + auto sw_flux_up_k = subview_2dk(m_buffer.sw_flux_up_k); + auto sw_flux_dn_k = subview_2dk(m_buffer.sw_flux_dn_k); + auto sw_flux_dn_dir_k = subview_2dk(m_buffer.sw_flux_dn_dir_k); + auto lw_flux_up_k = subview_2dk(m_buffer.lw_flux_up_k); + auto lw_flux_dn_k = subview_2dk(m_buffer.lw_flux_dn_k); + auto sw_clnclrsky_flux_up_k = subview_2dk(m_buffer.sw_clnclrsky_flux_up_k); + auto sw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_k); + auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_dir_k); + auto sw_clrsky_flux_up_k = subview_2dk(m_buffer.sw_clrsky_flux_up_k); + auto sw_clrsky_flux_dn_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_k); + auto sw_clrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_dir_k); + auto sw_clnsky_flux_up_k = subview_2dk(m_buffer.sw_clnsky_flux_up_k); + auto sw_clnsky_flux_dn_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_k); + auto sw_clnsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_dir_k); + auto lw_clnclrsky_flux_up_k = subview_2dk(m_buffer.lw_clnclrsky_flux_up_k); + auto lw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.lw_clnclrsky_flux_dn_k); + auto lw_clrsky_flux_up_k = subview_2dk(m_buffer.lw_clrsky_flux_up_k); + auto lw_clrsky_flux_dn_k = subview_2dk(m_buffer.lw_clrsky_flux_dn_k); + auto lw_clnsky_flux_up_k = subview_2dk(m_buffer.lw_clnsky_flux_up_k); + auto lw_clnsky_flux_dn_k = subview_2dk(m_buffer.lw_clnsky_flux_dn_k); + auto sw_bnd_flux_up_k = subview_3dk(m_buffer.sw_bnd_flux_up_k); + auto sw_bnd_flux_dn_k = subview_3dk(m_buffer.sw_bnd_flux_dn_k); + auto sw_bnd_flux_dir_k = subview_3dk(m_buffer.sw_bnd_flux_dir_k); + auto sw_bnd_flux_dif_k = subview_3dk(m_buffer.sw_bnd_flux_dif_k); + auto lw_bnd_flux_up_k = subview_3dk(m_buffer.lw_bnd_flux_up_k); + auto lw_bnd_flux_dn_k = subview_3dk(m_buffer.lw_bnd_flux_dn_k); + auto sfc_flux_dir_vis_k = subview_1dk(m_buffer.sfc_flux_dir_vis_k); + auto sfc_flux_dir_nir_k = subview_1dk(m_buffer.sfc_flux_dir_nir_k); + auto sfc_flux_dif_vis_k = subview_1dk(m_buffer.sfc_flux_dif_vis_k); + auto sfc_flux_dif_nir_k = subview_1dk(m_buffer.sfc_flux_dif_nir_k); + auto aero_tau_sw_k = subview_3dk(m_buffer.aero_tau_sw_k); + auto aero_ssa_sw_k = subview_3dk(m_buffer.aero_ssa_sw_k); + auto aero_g_sw_k = subview_3dk(m_buffer.aero_g_sw_k); + auto aero_tau_lw_k = subview_3dk(m_buffer.aero_tau_lw_k); + auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); + auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); + auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); + auto cld_tau_lw_gpt_k = subview_3dk(m_buffer.cld_tau_lw_gpt_k); + + auto d_tint_k = m_buffer.d_tint_k; + auto d_dz_k = m_buffer.d_dz_k; +#endif // Set gas concs to "view" only the first ncol columns +#ifdef RRTMGP_ENABLE_YAKL m_gas_concs.ncol = ncol; m_gas_concs.concs = subview_3d(gas_concs); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + m_gas_concs_k.ncol = ncol; + m_gas_concs_k.concs = subview_3dk(gas_concs_k); +#endif // Copy data from the FieldManager to the YAKL arrays { From 8a5c7a3883c96be1c4a635cab464224ad8395b94 Mon Sep 17 00:00:00 2001 From: Oscar Date: Mon, 22 Apr 2024 12:38:47 -0600 Subject: [PATCH 043/476] mam4xx - updating the mam4xx and haero submodules; mam4xx's update includes a fix for compute-sanitizer fail in EAMxx. --- externals/haero | 2 +- externals/mam4xx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/externals/haero b/externals/haero index 77c4a45584f2..def15327bedb 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit 77c4a45584f2bd3bf38a18ea5bc24ef9ce3776a2 +Subproject commit def15327bedb978ee76889f34b326e6e60591760 diff --git a/externals/mam4xx b/externals/mam4xx index a19bc75891e2..168c5c07bf6f 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a19bc75891e235071426840742a2c530bbea9e52 +Subproject commit 168c5c07bf6f881269bf39089049ec8726f21ed8 From 3396bd93230b8cb128245ca0d44a979059a7f895 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 22 Apr 2024 15:46:38 -0600 Subject: [PATCH 044/476] progress --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 288 +++++- .../rrtmgp/eamxx_rrtmgp_process_interface.hpp | 9 +- .../src/physics/rrtmgp/rrtmgp_test_utils.cpp | 442 ++++++--- .../src/physics/rrtmgp/rrtmgp_test_utils.hpp | 88 +- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 14 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 17 +- .../rrtmgp/scream_rrtmgp_interface.hpp | 14 +- .../src/physics/rrtmgp/simple_netcdf.hpp | 300 ------- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 797 +++++++++++------ .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 840 +++++++++++++++++- 10 files changed, 2011 insertions(+), 798 deletions(-) delete mode 100644 components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1b8d1aea72fd..6552651658f1 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -387,8 +387,8 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.sfc_flux_dif_vis_k.size(); m_buffer.sfc_flux_dif_nir_k = decltype(m_buffer.sfc_flux_dif_nir_k)(mem, m_col_chunk_size); mem += m_buffer.sfc_flux_dif_nir_k.size(); - m_buffer.cosine_zenith_k = decltype(m_buffer.cosine_zenith_k)(mem, m_col_chunk_size); - mem += m_buffer.cosine_zenith_k.size(); + m_buffer.cosine_zenith = decltype(m_buffer.cosine_zenith)(mem, m_col_chunk_size); + mem += m_buffer.cosine_zenith.size(); // 2d arrays m_buffer.p_lay_k = decltype(m_buffer.p_lay_k)(mem, m_col_chunk_size, m_nlay); @@ -425,10 +425,10 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.p_lev_k.size(); m_buffer.t_lev_k = decltype(m_buffer.t_lev_k)(mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.t_lev_k.size(); - m_buffer.d_tint_k = decltype(m_buffer.d_tint_k)(mem, m_col_chunk_size, m_nlay+1); - mem += m_buffer.d_tint_k.size(); - m_buffer.d_dz_k = decltype(m_buffer.d_dz_k )(mem, m_col_chunk_size, m_nlay); - mem += m_buffer.d_dz_k.size(); + m_buffer.d_tint = decltype(m_buffer.d_tint)(mem, m_col_chunk_size, m_nlay+1); + mem += m_buffer.d_tint.size(); + m_buffer.d_dz = decltype(m_buffer.d_dz)(mem, m_col_chunk_size, m_nlay); + mem += m_buffer.d_dz.size(); // 3d arrays m_buffer.sw_flux_up_k = decltype(m_buffer.sw_flux_up_k)(mem, m_col_chunk_size, m_nlay+1); mem += m_buffer.sw_flux_up_k.size(); @@ -543,10 +543,8 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Whether or not to do MCICA subcolumn sampling m_do_subcol_sampling = m_params.get("do_subcol_sampling",true); -#ifdef RRTMGP_ENABLE_YAKL // Initialize yakl - yakl_init(); -#endif + init_kls(); // Names of active gases auto gas_names_yakl_offset = string1dv(m_ngas); @@ -865,9 +863,6 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_bnd = subview_3d(m_buffer.cld_tau_lw_bnd); auto cld_tau_sw_gpt = subview_3d(m_buffer.cld_tau_sw_gpt); auto cld_tau_lw_gpt = subview_3d(m_buffer.cld_tau_lw_gpt); - - auto d_tint = m_buffer.d_tint; - auto d_dz = m_buffer.d_dz; #endif #ifdef RRTMGP_ENABLE_KOKKOS auto subview_1dk = [&](const real1dk v) -> real1dk { @@ -937,10 +932,10 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); auto cld_tau_lw_gpt_k = subview_3dk(m_buffer.cld_tau_lw_gpt_k); - - auto d_tint_k = m_buffer.d_tint_k; - auto d_dz_k = m_buffer.d_dz_k; #endif + auto d_tint = m_buffer.d_tint; + auto d_dz = m_buffer.d_dz; + // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1002,12 +997,13 @@ void RRTMGPRadiation::run_impl (const double dt) { const Real bc_top = T_mid(itop); const Real bc_bot = sqrt(sqrt(d_surf_lw_flux_up(icol)/stebol)); if (itop == 0) { - CO::compute_interface_values_linear(team, nlay, T_mid, dz, bc_top, bc_bot, T_int); + CO::compute_interface_values_linear(team, nlay, T_mid, dz, bc_top, bc_bot, T_int); } else { - CO::compute_interface_values_linear(team, nlay, T_mid, dz, bc_bot, bc_top, T_int); + CO::compute_interface_values_linear(team, nlay, T_mid, dz, bc_bot, bc_top, T_int); } team.team_barrier(); +#ifdef RRTMGP_ENABLE_YAKL mu0(i+1) = d_mu0(i); sfc_alb_dir_vis(i+1) = d_sfc_alb_dir_vis(icol); sfc_alb_dir_nir(i+1) = d_sfc_alb_dir_nir(icol); @@ -1059,6 +1055,37 @@ void RRTMGPRadiation::run_impl (const double dt) { aero_tau_lw(i+1,k+1,b+1) = 0; }); } +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + // Note that RRTMGP expects ordering (col,lay,bnd) but the FM keeps things in (col,bnd,lay) order + if (do_aerosol_rad) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_sw_k(i,k,b) = d_aero_tau_sw(icol,b,k); + aero_ssa_sw_k(i,k,b) = d_aero_ssa_sw(icol,b,k); + aero_g_sw_k (i,k,b) = d_aero_g_sw (icol,b,k); + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_lw_k(i,k,b) = d_aero_tau_lw(icol,b,k); + }); + } else { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_sw_k(i,k,b) = 0; + aero_ssa_sw_k(i,k,b) = 0; + aero_g_sw_k (i,k,b) = 0; + }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlwbands*nlay), [&] (const int&idx) { + auto b = idx / nlay; + auto k = idx % nlay; + aero_tau_lw_k(i,k,b) = 0; + }); + } +#endif }); } Kokkos::fence(); @@ -1067,7 +1094,12 @@ void RRTMGPRadiation::run_impl (const double dt) { // set_vmr requires the input array size to have the correct size, // and the last chunk may have less columns, so create a temp of // correct size that uses m_buffer.tmp2d's pointer +#ifdef RRTMGP_ENABLE_YAKL real2d tmp2d = subview_2d(m_buffer.tmp2d); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real2dk tmp2d_k = subview_2dk(m_buffer.tmp2d_k); +#endif for (int igas = 0; igas < m_ngas; igas++) { auto name = m_gas_names[igas]; auto full_name = name + "_volume_mix_ratio"; @@ -1082,13 +1114,23 @@ void RRTMGPRadiation::run_impl (const double dt) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { +#ifdef RRTMGP_ENABLE_YAKL tmp2d(i+1,k+1) = d_vmr(icol,k); // Note that for YAKL arrays i and k start with index 1 +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + tmp2d_k(i,k) = d_vmr(icol,k); +#endif }); }); Kokkos::fence(); // Populate GasConcs object +#ifdef RRTMGP_ENABLE_YAKL m_gas_concs.set_vmr(name, tmp2d); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + m_gas_concs_k.set_vmr(name, tmp2d_k); +#endif } // Set layer cloud fraction. @@ -1102,20 +1144,36 @@ void RRTMGPRadiation::run_impl (const double dt) { // If we *are* doing subcolumn sampling for MCICA, then keep cloud fraction as input // from cloud fraction parameterization, wherever that is computed. auto do_subcol_sampling = m_do_subcol_sampling; +#ifdef RRTMGP_ENABLE_YAKL auto lwp = m_buffer.lwp; auto iwp = m_buffer.iwp; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + auto lwp_k = m_buffer.lwp_k; + auto iwp_k = m_buffer.iwp_k; +#endif if (not do_subcol_sampling) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { +#ifdef RRTMGP_ENABLE_YAKL if (d_cldfrac_tot(icol,k) > 0) { cldfrac_tot(i+1,k+1) = 1; } else { cldfrac_tot(i+1,k+1) = 0; } d_cldfrac_rad(icol,k) = cldfrac_tot(i+1,k+1); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + if (d_cldfrac_tot(icol,k) > 0) { + cldfrac_tot_k(i,k) = 1; + } else { + cldfrac_tot_k(i,k) = 0; + } + d_cldfrac_rad(icol,k) = cldfrac_tot_k(i,k); +#endif }); }); } else { @@ -1124,15 +1182,26 @@ void RRTMGPRadiation::run_impl (const double dt) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { +#ifdef RRTMGP_ENABLE_YAKL cldfrac_tot(i+1,k+1) = d_cldfrac_tot(icol,k); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + cldfrac_tot_k(i,k) = d_cldfrac_tot(icol,k); +#endif d_cldfrac_rad(icol,k) = d_cldfrac_tot(icol,k); }); }); } Kokkos::fence(); // Compute layer cloud mass (per unit area) +#ifdef RRTMGP_ENABLE_YAKL scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); scream::rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); + scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); +#endif // Convert to g/m2 (needed by RRTMGP) { const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); @@ -1140,8 +1209,14 @@ void RRTMGPRadiation::run_impl (const double dt) { const int i = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { // Note that for YAKL arrays i and k start with index 1 +#ifdef RRTMGP_ENABLE_YAKL lwp(i+1,k+1) *= 1e3; iwp(i+1,k+1) *= 1e3; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + lwp_k(i,k) *= 1e3; + iwp_k(i,k) *= 1e3; +#endif }); }); } @@ -1149,15 +1224,24 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute band-by-band surface_albedos. This is needed since // the AD passes broadband albedos, but rrtmgp require band-by-band. +#ifdef RRTMGP_ENABLE_YAKL rrtmgp::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, sfc_alb_dir, sfc_alb_dif); - +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + rrtmgp::compute_band_by_band_surface_albedos( + ncol, nswbands, + sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, + sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, + sfc_alb_dir_k, sfc_alb_dif_k); +#endif // Compute cloud optical properties here? // Run RRTMGP driver +#ifdef RRTMGP_ENABLE_YAKL rrtmgp::rrtmgp_main( ncol, m_nlay, p_lay, t_lay, p_lev, t_lev, @@ -1178,8 +1262,32 @@ void RRTMGPRadiation::run_impl (const double dt) { eccf, m_atm_logger, m_extra_clnclrsky_diag, m_extra_clnsky_diag ); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + rrtmgp::rrtmgp_main( + ncol, m_nlay, + p_lay_k, t_lay_k, p_lev_k, t_lev_k, + m_gas_concs_k, + sfc_alb_dir_k, sfc_alb_dif_k, mu0_k, + lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, + aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k, + cld_tau_sw_bnd_k, cld_tau_lw_bnd_k, + cld_tau_sw_gpt_k, cld_tau_lw_gpt_k, + sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, + sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, + sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, + sw_clnsky_flux_up_k, sw_clnsky_flux_dn_k, sw_clnsky_flux_dn_dir_k, + lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, + lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, + lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k, + sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k, + eccf, m_atm_logger, + m_extra_clnclrsky_diag, m_extra_clnsky_diag + ); +#endif // Update heating tendency +#ifdef RRTMGP_ENABLE_YAKL auto sw_heating = m_buffer.sw_heating; auto lw_heating = m_buffer.lw_heating; rrtmgp::compute_heating_rate( @@ -1201,8 +1309,33 @@ void RRTMGPRadiation::run_impl (const double dt) { }); } Kokkos::fence(); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + auto sw_heating_k = m_buffer.sw_heating_k; + auto lw_heating_k = m_buffer.lw_heating_k; + rrtmgp::compute_heating_rate( + sw_flux_up_k, sw_flux_dn_k, p_del_k, sw_heating_k + ); + rrtmgp::compute_heating_rate( + lw_flux_up_k, lw_flux_dn_k, p_del_k, lw_heating_k + ); + { + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int idx = team.league_rank(); + const int icol = idx+beg; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& ilay) { + // Combine SW and LW heating into a net heating tendency; use d_rad_heating_pdel temporarily + // Note that for YAKL arrays i and k start with index 1 + d_rad_heating_pdel(icol,ilay) = sw_heating_k(idx,ilay) + lw_heating_k(idx,ilay); + }); + }); + } + Kokkos::fence(); +#endif // Index to surface (bottom of model); used to get surface fluxes below +#ifdef RRTMGP_ENABLE_YAKL const int kbot = nlay+1; // Compute diffuse flux as difference between total and direct @@ -1221,8 +1354,30 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir ); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + const int kbot_k = nlay; + + // Compute diffuse flux as difference between total and direct + Kokkos::parallel_for(Kokkos::RangePolicy(0,nswbands*(nlay+1)*ncol), + KOKKOS_LAMBDA (const int idx) { + // CAREFUL: these are YAKL arrays, with "LayoutLeft". So make the indices stride accordingly, and add 1. + const int ibnd = (idx / ncol) / (nlay+1); + const int ilev = (idx / ncol) % (nlay+1); + const int icol = idx % ncol; + sw_bnd_flux_dif_k(icol,ilev,ibnd) = sw_bnd_flux_dn_k(icol,ilev,ibnd) - sw_bnd_flux_dir_k(icol,ilev,ibnd); + }); + // Compute surface fluxes + rrtmgp::compute_broadband_surface_fluxes( + ncol, kbot_k, nswbands, + sw_bnd_flux_dir_k, sw_bnd_flux_dif_k, + sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, + sfc_flux_dif_vis_k, sfc_flux_dif_nir_k + ); +#endif // Compute diagnostic total cloud area (vertically-projected cloud cover) +#ifdef RRTMGP_ENABLE_YAKL real1d cldlow ("cldlow", d_cldlow.data() + m_col_chunk_beg[ic], ncol); real1d cldmed ("cldmed", d_cldmed.data() + m_col_chunk_beg[ic], ncol); real1d cldhgh ("cldhgh", d_cldhgh.data() + m_col_chunk_beg[ic], ncol); @@ -1237,6 +1392,23 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay, cld_tau_lw_gpt, cldmed); rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay, cld_tau_lw_gpt, cldhgh); rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay, cld_tau_lw_gpt, cldtot); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real1dk cldlow_k (d_cldlow.data() + m_col_chunk_beg[ic], ncol); + real1dk cldmed_k (d_cldmed.data() + m_col_chunk_beg[ic], ncol); + real1dk cldhgh_k (d_cldhgh.data() + m_col_chunk_beg[ic], ncol); + real1dk cldtot_k (d_cldtot.data() + m_col_chunk_beg[ic], ncol); + // NOTE: limits for low, mid, and high clouds are mostly taken from EAM F90 source, with the + // exception that I removed the restriction on low clouds to be above (numerically lower pressures) + // 1200 hPa, and on high clouds to be below (numerically high pressures) 50 hPa. This probably + // does not matter in practice, as clouds probably should not be produced above 50 hPa and we + // should not be encountering surface pressure above 1200 hPa, but in the event that things go off + // the rails we might want to look at these still. + rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); + rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); + rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); + rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); +#endif // Get visible 0.67 micron band for COSP auto idx_067 = rrtmgp::get_wavelength_index_sw(0.67e-6); @@ -1244,6 +1416,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto idx_105 = rrtmgp::get_wavelength_index_lw(10.5e-6); // Compute cloud-top diagnostics following AeroCOM recommendation +#ifdef RRTMGP_ENABLE_YAKL real1d T_mid_at_cldtop ("T_mid_at_cldtop", d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1d p_mid_at_cldtop ("p_mid_at_cldtop", d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1d cldfrac_ice_at_cldtop ("cldfrac_ice_at_cldtop", d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); @@ -1258,9 +1431,27 @@ void RRTMGPRadiation::run_impl (const double dt) { nc, T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_liq_at_cldtop_k (d_cldfrac_liq_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cldfrac_tot_at_cldtop_k (d_cldfrac_tot_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk cdnc_at_cldtop_k (d_cdnc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); + + rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, + nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, + cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, + eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); +#endif // Copy output data back to FieldManager const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); +#ifdef RRTMGP_ENABLE_YAKL Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; @@ -1303,11 +1494,60 @@ void RRTMGPRadiation::run_impl (const double dt) { d_sunlit(icol) = 0.0; } }); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + const int icol = i + beg; + d_sfc_flux_dir_nir(icol) = sfc_flux_dir_nir_k(i); + d_sfc_flux_dir_vis(icol) = sfc_flux_dir_vis_k(i); + d_sfc_flux_dif_nir(icol) = sfc_flux_dif_nir_k(i); + d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis_k(i); + d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); + d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { + d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); + d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); + d_sw_flux_dn_dir(icol,k) = sw_flux_dn_dir_k(i,k); + d_lw_flux_up(icol,k) = lw_flux_up_k(i,k); + d_lw_flux_dn(icol,k) = lw_flux_dn_k(i,k); + d_sw_clnclrsky_flux_up(icol,k) = sw_clnclrsky_flux_up_k(i,k); + d_sw_clnclrsky_flux_dn(icol,k) = sw_clnclrsky_flux_dn_k(i,k); + d_sw_clnclrsky_flux_dn_dir(icol,k) = sw_clnclrsky_flux_dn_dir_k(i,k); + d_sw_clrsky_flux_up(icol,k) = sw_clrsky_flux_up_k(i,k); + d_sw_clrsky_flux_dn(icol,k) = sw_clrsky_flux_dn_k(i,k); + d_sw_clrsky_flux_dn_dir(icol,k) = sw_clrsky_flux_dn_dir_k(i,k); + d_sw_clnsky_flux_up(icol,k) = sw_clnsky_flux_up_k(i,k); + d_sw_clnsky_flux_dn(icol,k) = sw_clnsky_flux_dn_k(i,k); + d_sw_clnsky_flux_dn_dir(icol,k) = sw_clnsky_flux_dn_dir_k(i,k); + d_lw_clnclrsky_flux_up(icol,k) = lw_clnclrsky_flux_up_k(i,k); + d_lw_clnclrsky_flux_dn(icol,k) = lw_clnclrsky_flux_dn_k(i,k); + d_lw_clrsky_flux_up(icol,k) = lw_clrsky_flux_up_k(i,k); + d_lw_clrsky_flux_dn(icol,k) = lw_clrsky_flux_dn_k(i,k); + d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); + d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); + }); + // Extract optical properties for COSP + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067); + d_dtau105(icol,k) = cld_tau_lw_bnd_k(i,k,idx_105); + }); + if (d_sw_clrsky_flux_dn(icol,0) > 0) { + d_sunlit(icol) = 1.0; + } else { + d_sunlit(icol) = 0.0; + } + }); +#endif } // loop over chunk // Restore the refCounted array. +#ifdef RRTMGP_ENABLE_YAKL m_gas_concs.concs = gas_concs; - +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + m_gas_concs_k.concs = gas_concs_k; +#endif } // update_rad // Apply temperature tendency; if we updated radiation this timestep, then d_rad_heating_pdel should @@ -1361,13 +1601,15 @@ void RRTMGPRadiation::run_impl (const double dt) { // ========================================================================================= void RRTMGPRadiation::finalize_impl () { +#ifdef RRTMGP_ENABLE_YAKL m_gas_concs.reset(); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + m_gas_concs_k.reset(); +#endif rrtmgp::rrtmgp_finalize(); -#ifdef RRTMGP_ENABLE_YAKL - // Finalize YAKL - yakl_finalize(); -#endif + finalize_kls(); } // ========================================================================================= diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index c0b29764ecae..66c89476028b 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -124,13 +124,13 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) + uview_1d cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; real1d sfc_alb_dir_nir; real1d sfc_alb_dif_vis; real1d sfc_alb_dif_nir; - uview_1d cosine_zenith; real1d sfc_flux_dir_vis; real1d sfc_flux_dir_nir; real1d sfc_flux_dif_vis; @@ -142,7 +142,6 @@ class RRTMGPRadiation : public AtmosphereProcess { real1dk sfc_alb_dir_nir_k; real1dk sfc_alb_dif_vis_k; real1dk sfc_alb_dif_nir_k; - uview_1d cosine_zenith_k; real1dk sfc_flux_dir_vis_k; real1dk sfc_flux_dir_nir_k; real1dk sfc_flux_dif_vis_k; @@ -150,6 +149,7 @@ class RRTMGPRadiation : public AtmosphereProcess { #endif // 2d size (ncol, nlay) + uview_2d d_dz; #ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; @@ -166,7 +166,6 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d iwp; real2d sw_heating; real2d lw_heating; - uview_2d d_dz; #endif #ifdef RRTMGP_ENABLE_KOKKOS real2dk p_lay_k; @@ -184,10 +183,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real2dk iwp_k; real2dk sw_heating_k; real2dk lw_heating_k; - uview_2d d_dz_k; #endif // 2d size (ncol, nlay+1) + uview_2d d_tint; #ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; @@ -211,7 +210,6 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clrsky_flux_dn; real2d lw_clnsky_flux_up; real2d lw_clnsky_flux_dn; - uview_2d d_tint; #endif #ifdef RRTMGP_ENABLE_KOKKOS real2dk p_lev_k; @@ -236,7 +234,6 @@ class RRTMGPRadiation : public AtmosphereProcess { real2dk lw_clrsky_flux_dn_k; real2dk lw_clnsky_flux_up_k; real2dk lw_clnsky_flux_dn_k; - uview_2d d_tint_k; #endif // 3d size (ncol, nlay+1, nswbands) diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index b36949e934f0..ea3280a3e035 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -1,7 +1,8 @@ #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" -#include "physics/rrtmgp/mo_garand_atmos_io.h" -#include "physics/rrtmgp/simple_netcdf.hpp" +#ifdef RRTMGP_ENABLE_YAKL +#include "YAKL_netcdf.h" +#endif #include "cpp/rrtmgp/mo_gas_concentrations.h" #include "cpp/rte/mo_fluxes.h" @@ -12,161 +13,316 @@ namespace rrtmgpTest { - using yakl::fortran::parallel_for; - using yakl::fortran::SimpleBounds; - using yakl::intrinsics::mod; - using yakl::intrinsics::merge; - - bool file_exists(const char *filename) { - if (auto file = fopen(filename, "r")) { - fclose(file); - return true; - } else { - return false; - } - } +#ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::mod; +using yakl::intrinsics::merge; +#endif + +bool file_exists(const char *filename) { + if (auto file = fopen(filename, "r")) { + fclose(file); + return true; + } else { + return false; + } +} - // TODO: use YAKL intrinsics for this to avoid needing to make host copies - bool all_close(real2d &arr1, real2d &arr2, double tolerance) { - int nx = arr1.dimension[0]; - int ny = arr2.dimension[1]; - auto arr1_h = arr1.createHostCopy(); - auto arr2_h = arr2.createHostCopy(); - for (int i=1; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { - printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); - return false; - } - } - } - return true; +// TODO: use YAKL intrinsics for this to avoid needing to make host copies +#ifdef RRTMGP_ENABLE_YAKL +bool all_close(real2d &arr1, real2d &arr2, double tolerance) { + int nx = arr1.dimension[0]; + int ny = arr2.dimension[1]; + auto arr1_h = arr1.createHostCopy(); + auto arr2_h = arr2.createHostCopy(); + for (int i=1; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } + } + } + return true; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { + int nx = arr1.extent(0); + int ny = arr2.extent(1); + auto arr1_h = Kokkos::create_mirror_view(arr1); + auto arr2_h = Kokkos::create_mirror_view(arr2); + Kokkos::deep_copy(arr1_h, arr1); + Kokkos::deep_copy(arr2_h, arr2); + for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } } + } + return true; +} +#endif +#ifdef RRTMGP_ENABLE_YAKL +void dummy_atmos( + std::string inputfile, + int ncol, real2d &p_lay, real2d &t_lay, + real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, + real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, + real1d &mu0, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cld) { - void dummy_atmos( - std::string inputfile, - int ncol, real2d &p_lay, real2d &t_lay, - real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, - real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, - real1d &mu0, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cld) { + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run - // Setup boundary conditions, solar zenith angle, etc - // NOTE: this stuff would come from the model in a real run + // Ocean-ish values for surface albedos, just for example + memset(sfc_alb_dir_vis , 0.06_wp ); + memset(sfc_alb_dir_nir , 0.06_wp ); + memset(sfc_alb_dif_vis , 0.06_wp ); + memset(sfc_alb_dif_nir , 0.06_wp ); - // Ocean-ish values for surface albedos, just for example - memset(sfc_alb_dir_vis , 0.06_wp ); - memset(sfc_alb_dir_nir , 0.06_wp ); - memset(sfc_alb_dif_vis , 0.06_wp ); - memset(sfc_alb_dif_nir , 0.06_wp ); + // Pick a solar zenith angle; this should come from the model + memset(mu0, 0.86_wp ); + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void dummy_atmos( + std::string inputfile, + int ncol, real2dk &p_lay, real2dk &t_lay, + real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, + real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, + real1dk &mu0, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld) { - // Pick a solar zenith angle; this should come from the model - memset(mu0, 0.86_wp ); + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run - // Get dummy cloud PHYSICAL properties. Note that this function call - // needs the CloudOptics object only because it uses the min and max - // valid values from the lookup tables for liquid and ice water path to - // create a dummy atmosphere. - dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); - } + // Ocean-ish values for surface albedos, just for example + Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); - void dummy_clouds( - CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { - - // Problem sizes - int ncol = t_lay.dimension[0]; - int nlay = t_lay.dimension[1]; - - // Generate some fake liquid and ice water data. We pick values to be midway between - // the min and max of the valid lookup table values for effective radii - real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); - real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - - // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and - // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. - // Set sane values for liquid and ice water path. - // NOTE: these "sane" values are in g/m2! - parallel_for( SimpleBounds<2>(nlay,ncol) , YAKL_LAMBDA (int ilay, int icol) { - cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100._wp * 100._wp && p_lay(icol,ilay) < 900._wp * 100._wp && mod(icol, 3) != 0; - // Ice and liquid will overlap in a few layers - lwp(icol,ilay) = merge(10._wp, 0._wp, cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263._wp); - iwp(icol,ilay) = merge(10._wp, 0._wp, cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273._wp); - rel(icol,ilay) = merge(rel_val, 0._wp, lwp(icol,ilay) > 0._wp); - rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); - }); - } + // Pick a solar zenith angle; this should come from the model + Kokkos::deep_copy(mu0, 0.86 ); - // Function to read fluxes from input file so we can compare our answers against the reference - void read_fluxes( - std::string inputfile, - real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, - real2d &lw_flux_up, real2d &lw_flux_dn) { - - // Initialize netcdf reader - simple_netcdf::SimpleNetCDF io; - io.open(inputfile, NC_NOWRITE); - - // Initialize arrays to hold fluxes - int nlev = io.getDimSize("lev"); - int ncol = io.getDimSize("col_flx"); - sw_flux_up = real2d("sw_flux_up" , ncol, nlev); - sw_flux_dn = real2d("sw_flux_dn" , ncol, nlev); - sw_flux_dir = real2d("sw_flux_dir", ncol, nlev); - lw_flux_up = real2d("lw_flux_up" , ncol, nlev); - lw_flux_dn = real2d("lw_flux_dn" , ncol, nlev); - - // Read data - io.read(sw_flux_up, "sw_flux_up" ); - io.read(sw_flux_dn, "sw_flux_dn" ); - io.read(sw_flux_dir, "sw_flux_dir"); - io.read(lw_flux_up, "lw_flux_up" ); - io.read(lw_flux_dn, "lw_flux_dn" ); - } + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(scream::rrtmgp::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} +#endif - void write_fluxes( - std::string outputfile, - real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, - real2d &lw_flux_up, real2d &lw_flux_dn) { +#ifdef RRTMGP_ENABLE_YAKL +void dummy_clouds( + CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { - simple_netcdf::SimpleNetCDF io; - io.create(outputfile); - io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); - io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); - io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); - io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); - io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); - io.close(); + // Problem sizes + int ncol = t_lay.dimension[0]; + int nlay = t_lay.dimension[1]; - } + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - // TODO: This function should instead take values, not file names, - // because for the test we do not want to write to file - int compare(std::string file1, std::string file2) { - // Read data from baseline and test file - real2d sw_flux_up_1; - real2d sw_flux_dn_1; - real2d sw_flux_dir_1; - real2d lw_flux_up_1; - real2d lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2d sw_flux_up_2; - real2d sw_flux_dn_2; - real2d sw_flux_dir_2; - real2d lw_flux_up_2; - real2d lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; - } + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + parallel_for( SimpleBounds<2>(nlay,ncol) , YAKL_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100._wp * 100._wp && p_lay(icol,ilay) < 900._wp * 100._wp && mod(icol, 3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = merge(10._wp, 0._wp, cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263._wp); + iwp(icol,ilay) = merge(10._wp, 0._wp, cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273._wp); + rel(icol,ilay) = merge(rel_val, 0._wp, lwp(icol,ilay) > 0._wp); + rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); + }); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask) { + + // Problem sizes + int ncol = t_lay.extent(0); + int nlay = t_lay.extent(1); + + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); + + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + Kokkos::parallel_for( conv::get_mdrp<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); + iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); + rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); + rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); + }); +} +#endif + +// Function to read fluxes from input file so we can compare our answers against the reference +#ifdef RRTMGP_ENABLE_YAKL +void read_fluxes( + std::string inputfile, + real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, + real2d &lw_flux_up, real2d &lw_flux_dn) { + + // Initialize netcdf reader + yakl::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); + + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2d("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2d("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2d("sw_flux_dir", ncol, nlev); + lw_flux_up = real2d("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2d("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void read_fluxes( + std::string inputfile, + real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, + real2dk &lw_flux_up, real2dk &lw_flux_dn) { + + // Initialize netcdf reader + conv::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); + + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); + lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} +#endif + +#ifdef RRTMGP_ENABLE_YAKL +void write_fluxes( + std::string outputfile, + real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, + real2d &lw_flux_up, real2d &lw_flux_dn) { + + yakl::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +void write_fluxes( + std::string outputfile, + real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, + real2dk &lw_flux_up, real2dk &lw_flux_dn) { + + conv::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} +#endif + +// TODO: This function should instead take values, not file names, +// because for the test we do not want to write to file +#ifdef RRTMGP_ENABLE_YAKL +int compare(std::string file1, std::string file2) { + // Read data from baseline and test file + real2d sw_flux_up_1; + real2d sw_flux_dn_1; + real2d sw_flux_dir_1; + real2d lw_flux_up_1; + real2d lw_flux_dn_1; + read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); + real2d sw_flux_up_2; + real2d sw_flux_dn_2; + real2d sw_flux_dir_2; + real2d lw_flux_up_2; + real2d lw_flux_dn_2; + read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); + + // Check values + int nerr = 0; + if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; + return nerr; +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +int compare(std::string file1, std::string file2) { + // Read data from baseline and test file + real2dk sw_flux_up_1; + real2dk sw_flux_dn_1; + real2dk sw_flux_dir_1; + real2dk lw_flux_up_1; + real2dk lw_flux_dn_1; + read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); + real2dk sw_flux_up_2; + real2dk sw_flux_dn_2; + real2dk sw_flux_dir_2; + real2dk lw_flux_up_2; + real2dk lw_flux_dn_2; + read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); + + // Check values + int nerr = 0; + if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; + return nerr; +} +#endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index cc1135f864ae..b9498a9a34f9 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -1,30 +1,70 @@ #ifndef RRTMGP_TEST_UTILS_HPP #define RRTMGP_TEST_UTILS_HPP #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" + namespace rrtmgpTest { - bool file_exists(const char *filename); - bool all_close(real2d &arr1, real2d &arr2, double tolerance); - void dummy_clouds( - CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cld - ); - void dummy_atmos( - std::string inputfile, - int ncol, real2d &p_lay, real2d &t_lay, - real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, - real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, - real1d &mu0, - real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cld - ); - void read_fluxes( - std::string inputfile, - real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, - real2d &lw_flux_up, real2d &lw_flux_dn - ); - void write_fluxes( - std::string outputfile, - real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, - real2d &lw_flux_up, real2d &lw_flux_dn - ); + +bool file_exists(const char *filename); + +#ifdef RRTMGP_ENABLE_YAKL +bool all_close(real2d &arr1, real2d &arr2, double tolerance); + +void dummy_clouds( + CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cld +); + +void dummy_atmos( + std::string inputfile, + int ncol, real2d &p_lay, real2d &t_lay, + real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, + real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, + real1d &mu0, + real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cld +); + +void read_fluxes( + std::string inputfile, + real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, + real2d &lw_flux_up, real2d &lw_flux_dn +); + +void write_fluxes( + std::string outputfile, + real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, + real2d &lw_flux_up, real2d &lw_flux_dn +); +#endif + +#ifdef RRTMGP_ENABLE_KOKKOS +bool all_close(real2dk &arr1, real2dk &arr2, double tolerance); + +void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld +); + +void dummy_atmos( + std::string inputfile, + int ncol, real2dk &p_lay, real2dk &t_lay, + real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, + real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, + real1dk &mu0, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld +); + +void read_fluxes( + std::string inputfile, + real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, + real2dk &lw_flux_up, real2dk &lw_flux_dn +); + +void write_fluxes( + std::string outputfile, + real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, + real2dk &lw_flux_up, real2dk &lw_flux_dn +); +#endif + } #endif diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 6c7dd91f5d1b..947485705927 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -48,16 +48,16 @@ void compute_heating_rate ( } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template void compute_heating_rate ( - Kokkos::View const &flux_up, - Kokkos::View const &flux_dn, - Kokkos::View const &pdel , - Kokkos::View const &heating_rate) + View1 const &flux_up, + View2 const &flux_dn, + View3 const &pdel , + View4 const &heating_rate) { using physconst = scream::physics::Constants; - auto ncol = flux_up.dimension[0]; - auto nlay = flux_up.dimension[1]-1; + auto ncol = flux_up.extent(0); + auto nlay = flux_up.extent(1)-1; Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { heating_rate(icol,ilay) = ( flux_up(icol,ilay+1) - flux_up(icol,ilay) - diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 51c415fc027f..20d1c1564d79 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -15,19 +15,28 @@ namespace scream { -#ifdef RRTMGP_ENABLE_YAKL -void yakl_init () +void init_kls () { +#ifdef RRTMGP_ENABLE_YAKL // Initialize yakl if(!yakl::isInitialized()) { yakl::init(); } +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + // Initialize kokkos + if(!Kokkos::is_initialized()) { Kokkos::initialize(); } +#endif } -void yakl_finalize() +void finalize_kls() { +#ifdef RRTMGP_ENABLE_YAKL // Finalize YAKL yakl::finalize(); -} #endif +#ifdef RRTMGP_ENABLE_KOKKOS + Kokkos::finalize(); +#endif +} namespace rrtmgp { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 102de74e71db..df700264ec4a 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -13,8 +13,8 @@ namespace scream { -void yakl_init (); -void yakl_finalize(); +void init_kls (); +void finalize_kls(); namespace rrtmgp { @@ -300,12 +300,12 @@ void mixing_ratio_to_cloud_mass( } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template +template void mixing_ratio_to_cloud_mass( - Kokkos::View const& mixing_ratio, - Kokkos::View const& cloud_fraction, - Kokkos::View const& dp, - Kokkos::View const& cloud_mass) + View1 const& mixing_ratio, + View2 const& cloud_fraction, + View3 const& dp, + View4 const& cloud_mass) { int ncol = mixing_ratio.extent(0); int nlay = mixing_ratio.extent(1); diff --git a/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp b/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp deleted file mode 100644 index fb965623bc9c..000000000000 --- a/components/eamxx/src/physics/rrtmgp/simple_netcdf.hpp +++ /dev/null @@ -1,300 +0,0 @@ -#ifndef SIMPLE_NETCDF_HPP -#define SIMPLE_NETCDF_HPP -// Provide a simple netCDF interface for RRTMGP that will read data in as YAKL -// arrays. This replaces the simple netCDF interface that is included with YAKL -// that requires netcdf-cxx4. We use it here instead of SCORPIO because we use -// the I/O routines from RRTMGP++ mostly unchanged, and they used the YAKL I/O -// for convenience. This small library mimics the functionality of the YAKL I/O -// and provides a nearly 100% compatible API, but uses netCDF-C directly rather -// than netcdf-cxx4. This library is provided specifically to work with -// existing RRTMGP I/O routines; all other codes developed for SCREAM should -// use SCORPIO for I/O. -#include -#include "YAKL.h" - -using namespace yakl; -namespace simple_netcdf { - - class SimpleNetCDF { - - protected: - - int ncid; - - public: - - // Constructor - SimpleNetCDF() {}; - - // Destructor - ~SimpleNetCDF() { - //close(); - }; - - void close() { - handle_error(nc_close(ncid)); - } - - void create(std::string filename, int mode=NC_CLOBBER) { - handle_error(nc_create(filename.c_str(), mode, &ncid)); - }; - - void open(std::string filename, int mode=NC_NOWRITE) { - handle_error(nc_open(filename.c_str(), mode, &ncid)); - }; - - void open(char *filename) { - handle_error(nc_open(filename, NC_NOWRITE, &ncid)); - } - - // NetCDF routines return an integer error code. Define a function - // here to abort program execution and throw an error code if we - // encounter a non-zero NetCDF return code. We will wrap our - // NetCDF calls with this function to handle these errors in a - // consistent way - void handle_error(int err) { - if (err) { - std::cout << "ERROR: " << nc_strerror(err) << std::endl; - abort(); - } - } - - void handle_error(int err, const char *file, int line) { - if (err) { - std::cout << "ERROR: " << nc_strerror(err) << " at line " << line << " in " << file << std::endl; - abort(); - } - } - - // Read a netCDF array to a YAKL array - template void read(Array &arr, std::string varName) { - - // Get variable ID - int varid; - handle_error(nc_inq_varid(ncid, varName.c_str(), &varid), __FILE__, __LINE__); - - // Get variable dimension sizes - int ndims; - int dimids[NC_MAX_VAR_DIMS]; - nc_type vtype; - handle_error(nc_inq_var(ncid, varid, NULL, &vtype, &ndims, dimids, NULL), __FILE__, __LINE__); - std::vector dimSizes(ndims); - size_t dimsize; - for (int i = 0; i < ndims; i++) { - handle_error(nc_inq_dimlen(ncid, dimids[i], &dimsize), __FILE__, __LINE__); - dimSizes[i] = dimsize; - } - - // If style is fortran, we need to reverse array dims - if (myStyle == styleFortran) { - std::reverse(dimSizes.begin(), dimSizes.end()); - } - - // Allocate (or reshape) the yakl array - arr = Array(varName.c_str(),dimSizes); - - // Read variable data - if (myMem == memDevice) { - auto arrHost = arr.createHostObject(); - if (std::is_same::value) { - // Create boolean array from integer arrays - Array tmp("tmp",dimSizes); - handle_error(nc_get_var(ncid, varid, tmp.data()), __FILE__, __LINE__); - for (int i=0; i < arr.totElems(); i++) { arrHost.myData[i] = tmp.myData[i] == 1; } - } else { - // Need to be careful with floats; nc_get_var is overloaded on type, but we need - // to make sure we read floats from file with the float procedure, and doubles - // with that for doubles. The danger is if the user passes a yakl array here - // with type double, but tries to read type float from file. - // TODO: why does the YAKL implementation for this work fine, but this version - // calling nc_get_var directly does not? - if (vtype == NC_FLOAT) { - Array tmp("tmp",dimSizes); - handle_error(nc_get_var(ncid, varid, tmp.data()), __FILE__, __LINE__); - for (size_t i=0; i < arr.totElems(); i++) { arrHost.myData[i] = tmp.myData[i]; } - } else { - handle_error(nc_get_var(ncid, varid, arrHost.data()), __FILE__, __LINE__); - } - } - arrHost.deep_copy_to(arr); - } else { - if (std::is_same::value) { - // Create boolean array from integer arrays - Array tmp("tmp",dimSizes); - handle_error(nc_get_var(ncid, varid, tmp.data()), __FILE__, __LINE__); - for (size_t i=0; i < arr.totElems(); i++) { arr.myData[i] = tmp.myData[i] == 1; } - } else { - if (vtype == NC_FLOAT) { - Array tmp("tmp",dimSizes); - handle_error(nc_get_var(ncid, varid, tmp.data()), __FILE__, __LINE__); - for (size_t i=0; i < arr.totElems(); i++) { arr.myData[i] = tmp.myData[i]; } - } else { - handle_error(nc_get_var(ncid, varid, arr.data()), __FILE__, __LINE__); - } - } - } - - } - - // Read a scalar type - template void read(T &arr , std::string varName) { - // Get variable ID - int varid; - handle_error(nc_inq_varid(ncid, varName.c_str(), &varid), __FILE__, __LINE__); - - // Read data - handle_error(nc_get_var(ncid, varid, &arr), __FILE__, __LINE__); - } - - // Check if variable exists in file - bool varExists (std::string varName) { - int varid; - int ncerr = nc_inq_varid(ncid, varName.c_str(), &varid); - if (ncerr == 0) { - return true; - } else { - return false; - } - } - - bool dimExists (std::string dimName) { - int dimid; - int ncerr = nc_inq_dimid(ncid, dimName.c_str(), &dimid); - if (ncerr == 0) { - return true; - } else { - return false; - } - } - - size_t getDimSize(std::string dimName) { - // Get dimension ID - int dimid; - handle_error(nc_inq_dimid(ncid, dimName.c_str(), &dimid)); - - // Get dimension size - size_t dimSize; - handle_error(nc_inq_dimlen(ncid, dimid, &dimSize)); - - return dimSize; - } - - void addDim(std::string dimName, int dimSize, int *dimid) { - // Put file into define mode - int ncerr = nc_redef(ncid); - if ((ncerr != NC_NOERR) and (ncerr != NC_EINDEFINE)) { - handle_error(ncerr, __FILE__, __LINE__); - } - - // Define dimension - handle_error(nc_def_dim(ncid, dimName.c_str(), dimSize, dimid), __FILE__, __LINE__); - - // End define mode - handle_error(nc_enddef(ncid), __FILE__, __LINE__); - } - - void addVar(std::string varName, nc_type varType, int ndims, int dimids[], int *varid) { - // Put file into define mode - int ncerr = nc_redef(ncid); - if ((ncerr != NC_NOERR) and (ncerr != NC_EINDEFINE)) { - handle_error(ncerr, __FILE__, __LINE__); - } - - // Define variable - handle_error(nc_def_var(ncid, varName.c_str(), varType, ndims, dimids, varid), __FILE__, __LINE__); - - // End define mode - handle_error(nc_enddef(ncid), __FILE__, __LINE__); - } - - template void putVar(T const &arr, std::string varName) { - // Make sure file is not in define mode - int ncerr = nc_enddef(ncid); - if ((ncerr != NC_NOERR) and (ncerr != NC_ENOTINDEFINE)) { - handle_error(ncerr, __FILE__, __LINE__); - } - - // Get variable Id - int varid; - handle_error(nc_inq_varid(ncid, varName.c_str(), &varid), __FILE__, __LINE__); - - // Write variable data - handle_error(nc_put_var(ncid, varid, arr), __FILE__, __LINE__); - } - - template - void write(Array const &arr, std::string varName, std::vector dimNames) { - - // Make sure length of dimension names is equal to rank of array - if (rank != dimNames.size()) { yakl_throw("dimNames.size() != Array rank"); } - - // Get dimension sizes - // Define dimensions if they do not exist and get dimension IDs - int dimids[rank]; - size_t dimSize; - size_t idim; - for (size_t i = 0; i < dimNames.size(); i++) { - // If style is Fortran, dimension ordering is reversed - if (myStyle == styleC) { - idim = i; - } else { - idim = rank - 1 - i; - } - int ncerr = nc_inq_dimid(ncid, dimNames[i].c_str(), &dimids[idim]); - if (ncerr == NC_NOERR) { - // check that size is correct - handle_error(nc_inq_dimlen(ncid, dimids[idim], &dimSize), __FILE__, __LINE__); - if (dimSize != arr.dimension[i]) { - yakl_throw("dimSize != arr.dimension[i]"); - } - } else { - addDim(dimNames[i], arr.dimension[i], &dimids[idim]); - } - } - - // Add variable if it does not exist - if (!varExists(varName)) { - int varid; - addVar(varName, getType(), rank, dimids, &varid); - } - - // Write data to file - if (myMem == memDevice) { - putVar(arr.createHostCopy().data(), varName); - } else { - putVar(arr.data(), varName); - } - } - - template void write(T arr, std::string varName) { - // If variable does not exist, try to add it - if (!varExists(varName)) { - int dimids[1] = {0}; - int varid; - addVar(varName, getType(), 0, dimids, &varid); - } - // Write to file - putVar(&arr, varName); - } - - // Determine nc_type corresponding to intrinsic type - template nc_type getType() const { - if ( std::is_same::value ) { return NC_CHAR; } - else if ( std::is_same::value ) { return NC_UBYTE; } - else if ( std::is_same::value ) { return NC_SHORT; } - else if ( std::is_same::value ) { return NC_USHORT; } - else if ( std::is_same::value ) { return NC_INT; } - else if ( std::is_same::value ) { return NC_UINT; } - else if ( std::is_same::value ) { return NC_INT64; } - else if ( std::is_same::value ) { return NC_UINT64; } - else if ( std::is_same::value ) { return NC_FLOAT; } - else if ( std::is_same::value ) { return NC_DOUBLE; } - else if ( std::is_same::value ) { return NC_STRING; } - else { yakl_throw("Invalid type"); } - return -1; - } - - }; // class SimpleNetCDF - -} // namespace simple_netcdf -#endif diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 8b2ddbf373a7..2526a5401fda 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -1,13 +1,15 @@ #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" -#include "physics/rrtmgp/mo_garand_atmos_io.h" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #include "share/scream_types.hpp" #include "share/scream_session.hpp" #include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "examples/all-sky/mo_garand_atmos_io.h" +#ifdef RRTMGP_ENABLE_YAKL #include "YAKL.h" +#endif #include "ekat/util/ekat_test_utils.hpp" #include @@ -15,6 +17,8 @@ using namespace scream; +namespace { + // Names of input files we will need. // NOTE: using old (full gpoint resolution) coefficient data for consistency with reference problem std::string coefficients_file_sw = SCREAM_DATA_DIR "/init/rrtmgp-data-sw-g224-2018-12-04.nc"; @@ -26,297 +30,532 @@ void expect_another_arg (int i, int argc) { EKAT_REQUIRE_MSG(i != argc-1, "Expected another cmd-line arg."); } -int run(int argc, char** argv) { - using namespace ekat::logger; - using logger_t = Logger; - - - ekat::Comm comm(MPI_COMM_WORLD); - auto logger = std::make_shared("",LogLevel::info,comm); - - // Parse command line arguments - if (argc < 3) { - std::string msg = "Missing required inputs. Usage:\n"; - msg += argv[0]; - msg += " -i -b [options]\n"; - logger->error(msg); - return 1; +#ifdef RRTMGP_ENABLE_YAKL +int run_yakl(int argc, char** argv) { + using namespace ekat::logger; + using logger_t = Logger; + + ekat::Comm comm(MPI_COMM_WORLD); + auto logger = std::make_shared("",LogLevel::info,comm); + + // Parse command line arguments + if (argc < 3) { + std::string msg = "Missing required inputs. Usage:\n"; + msg += argv[0]; + msg += " -i -b [options]\n"; + logger->error(msg); + return 1; + } + std::string inputfile, baseline, device; + + for (int i = 1; i < argc-1; ++i) { + if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { + expect_another_arg(i, argc); + ++i; + baseline = argv[i]; } - std::string inputfile, baseline, device; - - for (int i = 1; i < argc-1; ++i) { - if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { - expect_another_arg(i, argc); - ++i; - baseline = argv[i]; - } - if (ekat::argv_matches(argv[i], "-i", "--input-file")) { - expect_another_arg(i, argc); - ++i; - inputfile = argv[i]; - } - // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; - } + if (ekat::argv_matches(argv[i], "-i", "--input-file")) { + expect_another_arg(i, argc); + ++i; + inputfile = argv[i]; } + // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it + if (std::string(argv[i])=="--ekat-kokkos-device") { + expect_another_arg(i, argc); + ++i; + device = argv[i]; + } + } + + // Check to see that inputfiles exist + if (!rrtmgpTest::file_exists(inputfile.c_str())) { + logger->error("Inputfile " + inputfile + " does not exist.\n"); + return -1; + } + if (!rrtmgpTest::file_exists(baseline.c_str())) { + logger->error("Baseline " + baseline + " does not exist.\n"); + return -1; + } + + // Initialize yakl + logger->info("Initialize yakl...\n"); + yakl::init(); + + // Get reference fluxes from input file; do this here so we can get ncol dimension + logger->info("Read fluxes...\n"); + real2d sw_flux_up_ref; + real2d sw_flux_dn_ref; + real2d sw_flux_dir_ref; + real2d lw_flux_up_ref; + real2d lw_flux_dn_ref; + rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + + // Get dimension sizes + int ncol = sw_flux_up_ref.dimension[0]; + int nlev = sw_flux_up_ref.dimension[1]; + int nlay = nlev - 1; + + // Read in dummy Garand atmosphere; if this were an actual model simulation, + // these would be passed as inputs to the driver + // NOTE: set ncol to size of col_flx dimension in the input file. This is so + // that we can compare to the reference data provided in that file. Note that + // this will copy the first column of the input data (the first profile) ncol + // times. We will then fill some fraction of these columns with clouds for + // the test problem. + logger->info("Read dummy atmos...\n"); + real2d p_lay("p_lay", ncol, nlay); + real2d t_lay("t_lay", ncol, nlay); + real2d p_lev("p_lev", ncol, nlay+1); + real2d t_lev("t_lev", ncol, nlay+1); + GasConcs gas_concs; + read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, ncol); - // Check to see that inputfiles exist - if (!rrtmgpTest::file_exists(inputfile.c_str())) { - logger->error("Inputfile " + inputfile + " does not exist.\n"); - return -1; + // Initialize absorption coefficients + logger->info("Initialize RRTMGP...\n"); + scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + + // Setup our dummy atmosphere based on the input data we read in + logger->info("Setup dummy atmos...\n"); + real1d sfc_alb_dir_vis("sfc_alb_dir_vis", ncol); + real1d sfc_alb_dir_nir("sfc_alb_dir_nir", ncol); + real1d sfc_alb_dif_vis("sfc_alb_dif_vis", ncol); + real1d sfc_alb_dif_nir("sfc_alb_dif_nir", ncol); + real1d mu0("mu0", ncol); + real2d lwp("lwp", ncol, nlay); + real2d iwp("iwp", ncol, nlay); + real2d rel("rel", ncol, nlay); + real2d rei("rei", ncol, nlay); + real2d cld("cld", ncol, nlay); + rrtmgpTest::dummy_atmos( + inputfile, ncol, p_lay, t_lay, + sfc_alb_dir_vis, sfc_alb_dir_nir, + sfc_alb_dif_vis, sfc_alb_dif_nir, + mu0, + lwp, iwp, rel, rei, cld + ); + + // Setup flux outputs; In a real model run, the fluxes would be + // input/outputs into the driver (persisting between calls), and + // we would just have to setup the pointers to them in the + // FluxesBroadband object + logger->info("Setup fluxes...\n"); + const auto nswbands = scream::rrtmgp::k_dist_sw.get_nband(); + const auto nlwbands = scream::rrtmgp::k_dist_lw.get_nband(); + real2d sw_flux_up ("sw_flux_up" , ncol, nlay+1); + real2d sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); + real2d sw_flux_dir("sw_flux_dir", ncol, nlay+1); + real2d lw_flux_up ("lw_flux_up" , ncol, nlay+1); + real2d lw_flux_dn ("lw_flux_dn" , ncol, nlay+1); + real2d sw_clnclrsky_flux_up ("sw_clnclrsky_flux_up" , ncol, nlay+1); + real2d sw_clnclrsky_flux_dn ("sw_clnclrsky_flux_dn" , ncol, nlay+1); + real2d sw_clnclrsky_flux_dir("sw_clnclrsky_flux_dir", ncol, nlay+1); + real2d sw_clrsky_flux_up ("sw_clrsky_flux_up" , ncol, nlay+1); + real2d sw_clrsky_flux_dn ("sw_clrsky_flux_dn" , ncol, nlay+1); + real2d sw_clrsky_flux_dir("sw_clrsky_flux_dir", ncol, nlay+1); + real2d sw_clnsky_flux_up ("sw_clnsky_flux_up" , ncol, nlay+1); + real2d sw_clnsky_flux_dn ("sw_clnsky_flux_dn" , ncol, nlay+1); + real2d sw_clnsky_flux_dir("sw_clnsky_flux_dir", ncol, nlay+1); + real2d lw_clnclrsky_flux_up ("lw_clnclrsky_flux_up" , ncol, nlay+1); + real2d lw_clnclrsky_flux_dn ("lw_clnclrsky_flux_dn" , ncol, nlay+1); + real2d lw_clrsky_flux_up ("lw_clrsky_flux_up" , ncol, nlay+1); + real2d lw_clrsky_flux_dn ("lw_clrsky_flux_dn" , ncol, nlay+1); + real2d lw_clnsky_flux_up ("lw_clnsky_flux_up" , ncol, nlay+1); + real2d lw_clnsky_flux_dn ("lw_clnsky_flux_dn" , ncol, nlay+1); + real3d sw_bnd_flux_up ("sw_bnd_flux_up" , ncol, nlay+1, nswbands); + real3d sw_bnd_flux_dn ("sw_bnd_flux_dn" , ncol, nlay+1, nswbands); + real3d sw_bnd_flux_dir("sw_bnd_flux_dir", ncol, nlay+1, nswbands); + real3d lw_bnd_flux_up ("lw_bnd_flux_up" , ncol, nlay+1, nlwbands); + real3d lw_bnd_flux_dn ("lw_bnd_flux_dn" , ncol, nlay+1, nlwbands); + + // Compute band-by-band surface_albedos. + real2d sfc_alb_dir("sfc_alb_dir", ncol, nswbands); + real2d sfc_alb_dif("sfc_alb_dif", ncol, nswbands); + rrtmgp::compute_band_by_band_surface_albedos( + ncol, nswbands, + sfc_alb_dir_vis, sfc_alb_dir_nir, + sfc_alb_dif_vis, sfc_alb_dif_nir, + sfc_alb_dir, sfc_alb_dif); + + // Setup some dummy aerosol optical properties + auto aer_tau_sw = real3d("aer_tau_sw", ncol, nlay, nswbands); + auto aer_ssa_sw = real3d("aer_ssa_sw", ncol, nlay, nswbands); + auto aer_asm_sw = real3d("aer_asm_sw", ncol, nlay, nswbands); + auto aer_tau_lw = real3d("aer_tau_lw", ncol, nlay, nlwbands); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<3>(nswbands,nlay,ncol), YAKL_LAMBDA(int ibnd, int ilay, int icol) { + aer_tau_sw(icol,ilay,ibnd) = 0; + aer_ssa_sw(icol,ilay,ibnd) = 0; + aer_asm_sw(icol,ilay,ibnd) = 0; + }); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<3>(nlwbands,nlay,ncol), YAKL_LAMBDA(int ibnd, int ilay, int icol) { + aer_tau_lw(icol,ilay,ibnd) = 0; + }); + + // These are returned as outputs now from rrtmgp_main + // TODO: provide as inputs consistent with how aerosol is treated? + const auto nswgpts = scream::rrtmgp::k_dist_sw.get_ngpt(); + const auto nlwgpts = scream::rrtmgp::k_dist_lw.get_ngpt(); + auto cld_tau_sw_bnd = real3d("cld_tau_sw_bnd", ncol, nlay, nswbands); + auto cld_tau_lw_bnd = real3d("cld_tau_lw_bnd", ncol, nlay, nlwbands); + auto cld_tau_sw = real3d("cld_tau_sw", ncol, nlay, nswgpts); + auto cld_tau_lw = real3d("cld_tau_lw", ncol, nlay, nlwgpts); + + // Run RRTMGP code on dummy atmosphere + logger->info("Run RRTMGP...\n"); + const Real tsi_scaling = 1; + scream::rrtmgp::rrtmgp_main( + ncol, nlay, + p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, + lwp, iwp, rel, rei, cld, + aer_tau_sw, aer_ssa_sw, aer_asm_sw, aer_tau_lw, + cld_tau_sw_bnd, cld_tau_lw_bnd, // outputs + cld_tau_sw, cld_tau_lw, // outputs + sw_flux_up, sw_flux_dn, sw_flux_dir, + lw_flux_up, lw_flux_dn, + sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dir, + sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dir, + sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dir, + lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, + lw_clrsky_flux_up, lw_clrsky_flux_dn, + lw_clnsky_flux_up, lw_clnsky_flux_dn, + sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, + lw_bnd_flux_up, lw_bnd_flux_dn, tsi_scaling, logger, + true, true // extra_clnclrsky_diag, extra_clnsky_diag + // set them both to true because we are testing them below + ); + + // Check values against baseline + logger->info("Check values...\n"); + rrtmgpTest::read_fluxes( + baseline, + sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, + lw_flux_up_ref, lw_flux_dn_ref + ); + int nerr = 0; + if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + + // Because the aerosol optical properties are all set to zero, these fluxes must be equal + if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + + logger->info("Cleaning up...\n"); + // Clean up or else YAKL will throw errors + scream::rrtmgp::rrtmgp_finalize(); + sw_flux_up_ref.deallocate(); + sw_flux_dn_ref.deallocate(); + sw_flux_dir_ref.deallocate(); + lw_flux_up_ref.deallocate(); + lw_flux_dn_ref.deallocate(); + sw_flux_up.deallocate(); + sw_flux_dn.deallocate(); + sw_flux_dir.deallocate(); + lw_flux_up.deallocate(); + lw_flux_dn.deallocate(); + sw_clnclrsky_flux_up.deallocate(); + sw_clnclrsky_flux_dn.deallocate(); + sw_clnclrsky_flux_dir.deallocate(); + sw_clrsky_flux_up.deallocate(); + sw_clrsky_flux_dn.deallocate(); + sw_clrsky_flux_dir.deallocate(); + sw_clnsky_flux_up.deallocate(); + sw_clnsky_flux_dn.deallocate(); + sw_clnsky_flux_dir.deallocate(); + lw_clnclrsky_flux_up.deallocate(); + lw_clnclrsky_flux_dn.deallocate(); + lw_clrsky_flux_up.deallocate(); + lw_clrsky_flux_dn.deallocate(); + lw_clnsky_flux_up.deallocate(); + lw_clnsky_flux_dn.deallocate(); + sw_bnd_flux_up.deallocate(); + sw_bnd_flux_dn.deallocate(); + sw_bnd_flux_dir.deallocate(); + lw_bnd_flux_up.deallocate(); + lw_bnd_flux_dn.deallocate(); + p_lay.deallocate(); + t_lay.deallocate(); + p_lev.deallocate(); + t_lev.deallocate(); + gas_concs.reset(); + sfc_alb_dir_vis.deallocate(); + sfc_alb_dir_nir.deallocate(); + sfc_alb_dif_vis.deallocate(); + sfc_alb_dif_nir.deallocate(); + sfc_alb_dir.deallocate(); + sfc_alb_dif.deallocate(); + mu0.deallocate(); + lwp.deallocate(); + iwp.deallocate(); + rel.deallocate(); + rei.deallocate(); + cld.deallocate(); + aer_tau_sw.deallocate(); + aer_ssa_sw.deallocate(); + aer_asm_sw.deallocate(); + aer_tau_lw.deallocate(); + cld_tau_sw.deallocate(); + cld_tau_lw.deallocate(); + cld_tau_sw_bnd.deallocate(); + cld_tau_lw_bnd.deallocate(); + yakl::finalize(); + + return nerr != 0 ? 1 : 0; +} // end of main driver code +#endif + +#ifdef RRTMGP_ENABLE_KOKKOS +int run_kokkos(int argc, char** argv) { + using namespace ekat::logger; + using logger_t = Logger; + + ekat::Comm comm(MPI_COMM_WORLD); + auto logger = std::make_shared("",LogLevel::info,comm); + + // Parse command line arguments + if (argc < 3) { + std::string msg = "Missing required inputs. Usage:\n"; + msg += argv[0]; + msg += " -i -b [options]\n"; + logger->error(msg); + return 1; + } + std::string inputfile, baseline, device; + + for (int i = 1; i < argc-1; ++i) { + if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { + expect_another_arg(i, argc); + ++i; + baseline = argv[i]; } - if (!rrtmgpTest::file_exists(baseline.c_str())) { - logger->error("Baseline " + baseline + " does not exist.\n"); - return -1; + if (ekat::argv_matches(argv[i], "-i", "--input-file")) { + expect_another_arg(i, argc); + ++i; + inputfile = argv[i]; } - - // Initialize yakl - logger->info("Initialize yakl...\n"); - yakl::init(); - - // Get reference fluxes from input file; do this here so we can get ncol dimension - logger->info("Read fluxes...\n"); - real2d sw_flux_up_ref; - real2d sw_flux_dn_ref; - real2d sw_flux_dir_ref; - real2d lw_flux_up_ref; - real2d lw_flux_dn_ref; - rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); - - // Get dimension sizes - int ncol = sw_flux_up_ref.dimension[0]; - int nlev = sw_flux_up_ref.dimension[1]; - int nlay = nlev - 1; - - // Read in dummy Garand atmosphere; if this were an actual model simulation, - // these would be passed as inputs to the driver - // NOTE: set ncol to size of col_flx dimension in the input file. This is so - // that we can compare to the reference data provided in that file. Note that - // this will copy the first column of the input data (the first profile) ncol - // times. We will then fill some fraction of these columns with clouds for - // the test problem. - logger->info("Read dummy atmos...\n"); - real2d p_lay("p_lay", ncol, nlay); - real2d t_lay("t_lay", ncol, nlay); - real2d p_lev("p_lev", ncol, nlay+1); - real2d t_lev("t_lev", ncol, nlay+1); - GasConcs gas_concs; - read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, ncol); + // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it + if (std::string(argv[i])=="--ekat-kokkos-device") { + expect_another_arg(i, argc); + ++i; + device = argv[i]; + } + } + + // Check to see that inputfiles exist + if (!rrtmgpTest::file_exists(inputfile.c_str())) { + logger->error("Inputfile " + inputfile + " does not exist.\n"); + return -1; + } + if (!rrtmgpTest::file_exists(baseline.c_str())) { + logger->error("Baseline " + baseline + " does not exist.\n"); + return -1; + } + + // Initialize yakl + logger->info("Initialize yakl...\n"); + scream::init_kls(); + + // Get reference fluxes from input file; do this here so we can get ncol dimension + logger->info("Read fluxes...\n"); + real2dk sw_flux_up_ref; + real2dk sw_flux_dn_ref; + real2dk sw_flux_dir_ref; + real2dk lw_flux_up_ref; + real2dk lw_flux_dn_ref; + rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + + // Get dimension sizes + int ncol = sw_flux_up_ref.extent(0); + int nlev = sw_flux_up_ref.extent(1); + int nlay = nlev - 1; + + // Read in dummy Garand atmosphere; if this were an actual model simulation, + // these would be passed as inputs to the driver + // NOTE: set ncol to size of col_flx dimension in the input file. This is so + // that we can compare to the reference data provided in that file. Note that + // this will copy the first column of the input data (the first profile) ncol + // times. We will then fill some fraction of these columns with clouds for + // the test problem. + logger->info("Read dummy atmos...\n"); + real2dk p_lay("p_lay", ncol, nlay); + real2dk t_lay("t_lay", ncol, nlay); + real2dk p_lev("p_lev", ncol, nlay+1); + real2dk t_lev("t_lev", ncol, nlay+1); + GasConcsK gas_concs; + read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, ncol); // Initialize absorption coefficients - logger->info("Initialize RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); - - // Setup our dummy atmosphere based on the input data we read in - logger->info("Setup dummy atmos...\n"); - real1d sfc_alb_dir_vis("sfc_alb_dir_vis", ncol); - real1d sfc_alb_dir_nir("sfc_alb_dir_nir", ncol); - real1d sfc_alb_dif_vis("sfc_alb_dif_vis", ncol); - real1d sfc_alb_dif_nir("sfc_alb_dif_nir", ncol); - real1d mu0("mu0", ncol); - real2d lwp("lwp", ncol, nlay); - real2d iwp("iwp", ncol, nlay); - real2d rel("rel", ncol, nlay); - real2d rei("rei", ncol, nlay); - real2d cld("cld", ncol, nlay); - rrtmgpTest::dummy_atmos( - inputfile, ncol, p_lay, t_lay, - sfc_alb_dir_vis, sfc_alb_dir_nir, - sfc_alb_dif_vis, sfc_alb_dif_nir, - mu0, - lwp, iwp, rel, rei, cld - ); - - // Setup flux outputs; In a real model run, the fluxes would be - // input/outputs into the driver (persisting between calls), and - // we would just have to setup the pointers to them in the - // FluxesBroadband object - logger->info("Setup fluxes...\n"); - const auto nswbands = scream::rrtmgp::k_dist_sw.get_nband(); - const auto nlwbands = scream::rrtmgp::k_dist_lw.get_nband(); - real2d sw_flux_up ("sw_flux_up" , ncol, nlay+1); - real2d sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); - real2d sw_flux_dir("sw_flux_dir", ncol, nlay+1); - real2d lw_flux_up ("lw_flux_up" , ncol, nlay+1); - real2d lw_flux_dn ("lw_flux_dn" , ncol, nlay+1); - real2d sw_clnclrsky_flux_up ("sw_clnclrsky_flux_up" , ncol, nlay+1); - real2d sw_clnclrsky_flux_dn ("sw_clnclrsky_flux_dn" , ncol, nlay+1); - real2d sw_clnclrsky_flux_dir("sw_clnclrsky_flux_dir", ncol, nlay+1); - real2d sw_clrsky_flux_up ("sw_clrsky_flux_up" , ncol, nlay+1); - real2d sw_clrsky_flux_dn ("sw_clrsky_flux_dn" , ncol, nlay+1); - real2d sw_clrsky_flux_dir("sw_clrsky_flux_dir", ncol, nlay+1); - real2d sw_clnsky_flux_up ("sw_clnsky_flux_up" , ncol, nlay+1); - real2d sw_clnsky_flux_dn ("sw_clnsky_flux_dn" , ncol, nlay+1); - real2d sw_clnsky_flux_dir("sw_clnsky_flux_dir", ncol, nlay+1); - real2d lw_clnclrsky_flux_up ("lw_clnclrsky_flux_up" , ncol, nlay+1); - real2d lw_clnclrsky_flux_dn ("lw_clnclrsky_flux_dn" , ncol, nlay+1); - real2d lw_clrsky_flux_up ("lw_clrsky_flux_up" , ncol, nlay+1); - real2d lw_clrsky_flux_dn ("lw_clrsky_flux_dn" , ncol, nlay+1); - real2d lw_clnsky_flux_up ("lw_clnsky_flux_up" , ncol, nlay+1); - real2d lw_clnsky_flux_dn ("lw_clnsky_flux_dn" , ncol, nlay+1); - real3d sw_bnd_flux_up ("sw_bnd_flux_up" , ncol, nlay+1, nswbands); - real3d sw_bnd_flux_dn ("sw_bnd_flux_dn" , ncol, nlay+1, nswbands); - real3d sw_bnd_flux_dir("sw_bnd_flux_dir", ncol, nlay+1, nswbands); - real3d lw_bnd_flux_up ("lw_bnd_flux_up" , ncol, nlay+1, nlwbands); - real3d lw_bnd_flux_dn ("lw_bnd_flux_dn" , ncol, nlay+1, nlwbands); - - // Compute band-by-band surface_albedos. - real2d sfc_alb_dir("sfc_alb_dir", ncol, nswbands); - real2d sfc_alb_dif("sfc_alb_dif", ncol, nswbands); - rrtmgp::compute_band_by_band_surface_albedos( - ncol, nswbands, - sfc_alb_dir_vis, sfc_alb_dir_nir, - sfc_alb_dif_vis, sfc_alb_dif_nir, - sfc_alb_dir, sfc_alb_dif); - - // Setup some dummy aerosol optical properties - auto aer_tau_sw = real3d("aer_tau_sw", ncol, nlay, nswbands); - auto aer_ssa_sw = real3d("aer_ssa_sw", ncol, nlay, nswbands); - auto aer_asm_sw = real3d("aer_asm_sw", ncol, nlay, nswbands); - auto aer_tau_lw = real3d("aer_tau_lw", ncol, nlay, nlwbands); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<3>(nswbands,nlay,ncol), YAKL_LAMBDA(int ibnd, int ilay, int icol) { - aer_tau_sw(icol,ilay,ibnd) = 0; - aer_ssa_sw(icol,ilay,ibnd) = 0; - aer_asm_sw(icol,ilay,ibnd) = 0; - }); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<3>(nlwbands,nlay,ncol), YAKL_LAMBDA(int ibnd, int ilay, int icol) { - aer_tau_lw(icol,ilay,ibnd) = 0; - }); - - // These are returned as outputs now from rrtmgp_main - // TODO: provide as inputs consistent with how aerosol is treated? - const auto nswgpts = scream::rrtmgp::k_dist_sw.get_ngpt(); - const auto nlwgpts = scream::rrtmgp::k_dist_lw.get_ngpt(); - auto cld_tau_sw_bnd = real3d("cld_tau_sw_bnd", ncol, nlay, nswbands); - auto cld_tau_lw_bnd = real3d("cld_tau_lw_bnd", ncol, nlay, nlwbands); - auto cld_tau_sw = real3d("cld_tau_sw", ncol, nlay, nswgpts); - auto cld_tau_lw = real3d("cld_tau_lw", ncol, nlay, nlwgpts); - - // Run RRTMGP code on dummy atmosphere - logger->info("Run RRTMGP...\n"); - const Real tsi_scaling = 1; - scream::rrtmgp::rrtmgp_main( - ncol, nlay, - p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, - lwp, iwp, rel, rei, cld, - aer_tau_sw, aer_ssa_sw, aer_asm_sw, aer_tau_lw, - cld_tau_sw_bnd, cld_tau_lw_bnd, // outputs - cld_tau_sw, cld_tau_lw, // outputs - sw_flux_up, sw_flux_dn, sw_flux_dir, - lw_flux_up, lw_flux_dn, - sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dir, - sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dir, - sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dir, - lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, - lw_clrsky_flux_up, lw_clrsky_flux_dn, - lw_clnsky_flux_up, lw_clnsky_flux_dn, - sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, - lw_bnd_flux_up, lw_bnd_flux_dn, tsi_scaling, logger, - true, true // extra_clnclrsky_diag, extra_clnsky_diag - // set them both to true because we are testing them below - ); - - // Check values against baseline - logger->info("Check values...\n"); - rrtmgpTest::read_fluxes( - baseline, - sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, - lw_flux_up_ref, lw_flux_dn_ref - ); - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; - - // Because the aerosol optical properties are all set to zero, these fluxes must be equal - if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - - - logger->info("Cleaning up...\n"); - // Clean up or else YAKL will throw errors - scream::rrtmgp::rrtmgp_finalize(); - sw_flux_up_ref.deallocate(); - sw_flux_dn_ref.deallocate(); - sw_flux_dir_ref.deallocate(); - lw_flux_up_ref.deallocate(); - lw_flux_dn_ref.deallocate(); - sw_flux_up.deallocate(); - sw_flux_dn.deallocate(); - sw_flux_dir.deallocate(); - lw_flux_up.deallocate(); - lw_flux_dn.deallocate(); - sw_clnclrsky_flux_up.deallocate(); - sw_clnclrsky_flux_dn.deallocate(); - sw_clnclrsky_flux_dir.deallocate(); - sw_clrsky_flux_up.deallocate(); - sw_clrsky_flux_dn.deallocate(); - sw_clrsky_flux_dir.deallocate(); - sw_clnsky_flux_up.deallocate(); - sw_clnsky_flux_dn.deallocate(); - sw_clnsky_flux_dir.deallocate(); - lw_clnclrsky_flux_up.deallocate(); - lw_clnclrsky_flux_dn.deallocate(); - lw_clrsky_flux_up.deallocate(); - lw_clrsky_flux_dn.deallocate(); - lw_clnsky_flux_up.deallocate(); - lw_clnsky_flux_dn.deallocate(); - sw_bnd_flux_up.deallocate(); - sw_bnd_flux_dn.deallocate(); - sw_bnd_flux_dir.deallocate(); - lw_bnd_flux_up.deallocate(); - lw_bnd_flux_dn.deallocate(); - p_lay.deallocate(); - t_lay.deallocate(); - p_lev.deallocate(); - t_lev.deallocate(); - gas_concs.reset(); - sfc_alb_dir_vis.deallocate(); - sfc_alb_dir_nir.deallocate(); - sfc_alb_dif_vis.deallocate(); - sfc_alb_dif_nir.deallocate(); - sfc_alb_dir.deallocate(); - sfc_alb_dif.deallocate(); - mu0.deallocate(); - lwp.deallocate(); - iwp.deallocate(); - rel.deallocate(); - rei.deallocate(); - cld.deallocate(); - aer_tau_sw.deallocate(); - aer_ssa_sw.deallocate(); - aer_asm_sw.deallocate(); - aer_tau_lw.deallocate(); - cld_tau_sw.deallocate(); - cld_tau_lw.deallocate(); - cld_tau_sw_bnd.deallocate(); - cld_tau_lw_bnd.deallocate(); - yakl::finalize(); - - return nerr != 0 ? 1 : 0; - + logger->info("Initialize RRTMGP...\n"); + scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + + // Setup our dummy atmosphere based on the input data we read in + logger->info("Setup dummy atmos...\n"); + real1dk sfc_alb_dir_vis("sfc_alb_dir_vis", ncol); + real1dk sfc_alb_dir_nir("sfc_alb_dir_nir", ncol); + real1dk sfc_alb_dif_vis("sfc_alb_dif_vis", ncol); + real1dk sfc_alb_dif_nir("sfc_alb_dif_nir", ncol); + real1dk mu0("mu0", ncol); + real2dk lwp("lwp", ncol, nlay); + real2dk iwp("iwp", ncol, nlay); + real2dk rel("rel", ncol, nlay); + real2dk rei("rei", ncol, nlay); + real2dk cld("cld", ncol, nlay); + rrtmgpTest::dummy_atmos( + inputfile, ncol, p_lay, t_lay, + sfc_alb_dir_vis, sfc_alb_dir_nir, + sfc_alb_dif_vis, sfc_alb_dif_nir, + mu0, + lwp, iwp, rel, rei, cld + ); + + // Setup flux outputs; In a real model run, the fluxes would be + // input/outputs into the driver (persisting between calls), and + // we would just have to setup the pointers to them in the + // FluxesBroadband object + logger->info("Setup fluxes...\n"); + const auto nswbands = scream::rrtmgp::k_dist_sw_k.get_nband(); + const auto nlwbands = scream::rrtmgp::k_dist_lw_k.get_nband(); + real2dk sw_flux_up ("sw_flux_up" , ncol, nlay+1); + real2dk sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); + real2dk sw_flux_dir("sw_flux_dir", ncol, nlay+1); + real2dk lw_flux_up ("lw_flux_up" , ncol, nlay+1); + real2dk lw_flux_dn ("lw_flux_dn" , ncol, nlay+1); + real2dk sw_clnclrsky_flux_up ("sw_clnclrsky_flux_up" , ncol, nlay+1); + real2dk sw_clnclrsky_flux_dn ("sw_clnclrsky_flux_dn" , ncol, nlay+1); + real2dk sw_clnclrsky_flux_dir("sw_clnclrsky_flux_dir", ncol, nlay+1); + real2dk sw_clrsky_flux_up ("sw_clrsky_flux_up" , ncol, nlay+1); + real2dk sw_clrsky_flux_dn ("sw_clrsky_flux_dn" , ncol, nlay+1); + real2dk sw_clrsky_flux_dir("sw_clrsky_flux_dir", ncol, nlay+1); + real2dk sw_clnsky_flux_up ("sw_clnsky_flux_up" , ncol, nlay+1); + real2dk sw_clnsky_flux_dn ("sw_clnsky_flux_dn" , ncol, nlay+1); + real2dk sw_clnsky_flux_dir("sw_clnsky_flux_dir", ncol, nlay+1); + real2dk lw_clnclrsky_flux_up ("lw_clnclrsky_flux_up" , ncol, nlay+1); + real2dk lw_clnclrsky_flux_dn ("lw_clnclrsky_flux_dn" , ncol, nlay+1); + real2dk lw_clrsky_flux_up ("lw_clrsky_flux_up" , ncol, nlay+1); + real2dk lw_clrsky_flux_dn ("lw_clrsky_flux_dn" , ncol, nlay+1); + real2dk lw_clnsky_flux_up ("lw_clnsky_flux_up" , ncol, nlay+1); + real2dk lw_clnsky_flux_dn ("lw_clnsky_flux_dn" , ncol, nlay+1); + real3dk sw_bnd_flux_up ("sw_bnd_flux_up" , ncol, nlay+1, nswbands); + real3dk sw_bnd_flux_dn ("sw_bnd_flux_dn" , ncol, nlay+1, nswbands); + real3dk sw_bnd_flux_dir("sw_bnd_flux_dir", ncol, nlay+1, nswbands); + real3dk lw_bnd_flux_up ("lw_bnd_flux_up" , ncol, nlay+1, nlwbands); + real3dk lw_bnd_flux_dn ("lw_bnd_flux_dn" , ncol, nlay+1, nlwbands); + + // Compute band-by-band surface_albedos. + real2dk sfc_alb_dir("sfc_alb_dir", ncol, nswbands); + real2dk sfc_alb_dif("sfc_alb_dif", ncol, nswbands); + rrtmgp::compute_band_by_band_surface_albedos( + ncol, nswbands, + sfc_alb_dir_vis, sfc_alb_dir_nir, + sfc_alb_dif_vis, sfc_alb_dif_nir, + sfc_alb_dir, sfc_alb_dif); + + // Setup some dummy aerosol optical properties + auto aer_tau_sw = real3dk("aer_tau_sw", ncol, nlay, nswbands); + auto aer_ssa_sw = real3dk("aer_ssa_sw", ncol, nlay, nswbands); + auto aer_asm_sw = real3dk("aer_asm_sw", ncol, nlay, nswbands); + auto aer_tau_lw = real3dk("aer_tau_lw", ncol, nlay, nlwbands); + Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + aer_tau_sw(icol,ilay,ibnd) = 0; + aer_ssa_sw(icol,ilay,ibnd) = 0; + aer_asm_sw(icol,ilay,ibnd) = 0; + }); + Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + aer_tau_lw(icol,ilay,ibnd) = 0; + }); + + // These are returned as outputs now from rrtmgp_main + // TODO: provide as inputs consistent with how aerosol is treated? + const auto nswgpts = scream::rrtmgp::k_dist_sw_k.get_ngpt(); + const auto nlwgpts = scream::rrtmgp::k_dist_lw_k.get_ngpt(); + auto cld_tau_sw_bnd = real3dk("cld_tau_sw_bnd", ncol, nlay, nswbands); + auto cld_tau_lw_bnd = real3dk("cld_tau_lw_bnd", ncol, nlay, nlwbands); + auto cld_tau_sw = real3dk("cld_tau_sw", ncol, nlay, nswgpts); + auto cld_tau_lw = real3dk("cld_tau_lw", ncol, nlay, nlwgpts); + + // Run RRTMGP code on dummy atmosphere + logger->info("Run RRTMGP...\n"); + const Real tsi_scaling = 1; + scream::rrtmgp::rrtmgp_main( + ncol, nlay, + p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, + lwp, iwp, rel, rei, cld, + aer_tau_sw, aer_ssa_sw, aer_asm_sw, aer_tau_lw, + cld_tau_sw_bnd, cld_tau_lw_bnd, // outputs + cld_tau_sw, cld_tau_lw, // outputs + sw_flux_up, sw_flux_dn, sw_flux_dir, + lw_flux_up, lw_flux_dn, + sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dir, + sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dir, + sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dir, + lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, + lw_clrsky_flux_up, lw_clrsky_flux_dn, + lw_clnsky_flux_up, lw_clnsky_flux_dn, + sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, + lw_bnd_flux_up, lw_bnd_flux_dn, tsi_scaling, logger, + true, true // extra_clnclrsky_diag, extra_clnsky_diag + // set them both to true because we are testing them below + ); + + // Check values against baseline + logger->info("Check values...\n"); + rrtmgpTest::read_fluxes( + baseline, + sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, + lw_flux_up_ref, lw_flux_dn_ref + ); + int nerr = 0; + if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + + // Because the aerosol optical properties are all set to zero, these fluxes must be equal + if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + + logger->info("Cleaning up...\n"); + // Clean up or else YAKL will throw errors + scream::rrtmgp::rrtmgp_finalize(); + scream::finalize_kls(); + + return nerr != 0 ? 1 : 0; } // end of main driver code +#endif + +} int main(int argc, char** argv) { MPI_Init(&argc,&argv); - int ret = run(argc,argv); + int ret = 0; +#ifdef RRTMGP_ENABLE_YAKL + ret += run_yakl(argc,argv); +#endif +#ifdef RRTMGP_ENABLE_KOKKOS + ret += run_kokkos(argc,argv); +#endif MPI_Finalize(); return ret; } - diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index f737c2e249c0..0a6d5bcde247 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -1,17 +1,29 @@ #include "catch2/catch.hpp" #include "physics/rrtmgp/rrtmgp_utils.hpp" #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" -#include "YAKL.h" #include "physics/share/physics_constants.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" #include "physics/rrtmgp/mo_load_coefficients.h" +#ifdef RRTMGP_ENABLE_YAKL +#include "YAKL.h" +#endif + +namespace { + +template +auto chc(const View& view) +{ + return Kokkos::create_mirror_view_and_copy(HostDevice(), view); +} + // Names of input files we will need. std::string coefficients_file_sw = SCREAM_DATA_DIR "/init/rrtmgp-data-sw-g112-210809.nc"; std::string coefficients_file_lw = SCREAM_DATA_DIR "/init/rrtmgp-data-lw-g128-210809.nc"; std::string cloud_optics_file_sw = SCREAM_DATA_DIR "/init/rrtmgp-cloud-optics-coeffs-sw.nc"; std::string cloud_optics_file_lw = SCREAM_DATA_DIR "/init/rrtmgp-cloud-optics-coeffs-lw.nc"; +#ifdef RRTMGP_ENABLE_YAKL TEST_CASE("rrtmgp_test_heating") { // Initialize YAKL if (!yakl::isInitialized()) { yakl::init(); } @@ -100,7 +112,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass") { cloud_mass_ref = 0.0; scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(cloud_mass.createHostCopy()(1,1) == cloud_mass_ref); - + // Test with empty clouds (cloud fraction but with no associated mixing ratio) // This case could happen if we use a total cloud fraction, but compute layer // cloud mass separately for liquid and ice. @@ -112,7 +124,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass") { cloud_mass_ref = 0.0; scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(cloud_mass.createHostCopy()(1,1) == cloud_mass_ref); - + // Test with cell half filled with cloud yakl::fortran::parallel_for(1, YAKL_LAMBDA(int /* dummy */) { dp(1,1) = 10.0; @@ -189,7 +201,7 @@ TEST_CASE("rrtmgp_test_zenith") { double lambm0; double mvelpp; // bool flag_print = false; - shr_orb_params_c2f(&orbital_year, &eccen, &obliq, &mvelp, + shr_orb_params_c2f(&orbital_year, &eccen, &obliq, &mvelp, &obliqr, &lambm0, &mvelpp); //, flag_print); // Note fortran code has optional arg REQUIRE(eccen == eccen_ref); REQUIRE(obliqr == obliqr_ref); @@ -334,7 +346,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux") { REQUIRE(std::abs(sfc_flux_dif_nir.createHostCopy()(1) - 9.0) < tol); REQUIRE(std::abs(sfc_flux_dif_vis.createHostCopy()(1) - 0.0) < tol); // --------------------------------- - + // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); @@ -842,3 +854,821 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { yakl::finalize(); } +#endif + +#ifdef RRTMGP_ENABLE_KOKKOS +TEST_CASE("rrtmgp_test_heating_k") { + // Initialize Kokkos + scream::init_kls(); + + // Test heating rate function by passing simple inputs + auto dp = real2dk("dp", 1, 1); + auto flux_up = real2dk("flux_up", 1, 2); + auto flux_dn = real2dk("flux_dn", 1, 2); + auto heating = real2dk("heating", 1, 1); + // Simple no-heating test + // NOTE: Kokkos::parallel_for because we need to do these in a kernel on the device + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + dp(0, 0) = 10; + flux_up(0, 0) = 1.0; + flux_up(0, 1) = 1.0; + flux_dn(0, 0) = 1.0; + flux_dn(0, 1) = 1.0; + }); + scream::rrtmgp::compute_heating_rate(flux_up, flux_dn, dp, heating); + REQUIRE(chc(heating)(0,0) == 0); + + // Simple net postive heating; net flux into layer should be 1.0 + // NOTE: Kokkos::parallel_for because we need to do these in a kernel on the device + Kokkoks::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + flux_up(0, 0) = 1.0; + flux_up(0, 1) = 1.0; + flux_dn(0, 0) = 1.5; + flux_dn(0, 1) = 0.5; + }); + using physconst = scream::physics::Constants; + auto g = physconst::gravit; //9.81; + auto cp_air = physconst::Cpair; //1005.0; + auto pdel = chc(dp)(0,0); + auto heating_ref = 1.0 * g / (cp_air * pdel); + scream::rrtmgp::compute_heating_rate(flux_up, flux_dn, dp, heating); + REQUIRE(chc(heating)(0,0) == heating_ref); + + // Simple net negative heating; net flux into layer should be -1.0 + // NOTE: Kokkos::parallel_for because we need to do these in a kernel on the device + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + flux_up(0,0) = 1.5; + flux_up(0,1) = 0.5; + flux_dn(0,0) = 1.0; + flux_dn(0,1) = 1.0; + }); + heating_ref = -1.0 * g / (cp_air * pdel); + scream::rrtmgp::compute_heating_rate(flux_up, flux_dn, dp, heating); + REQUIRE(chc(heating)(0,0) == heating_ref); + + // Clean up + dp.deallocate(); + flux_up.deallocate(); + flux_dn.deallocate(); + heating.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { + // Initialize YAKL + scream::init_kls(); + + using physconst = scream::physics::Constants; + + // Test mixing ratio to cloud mass function by passing simple inputs + auto dp = real2dk("dp", 1, 1); + auto mixing_ratio = real2dk("mixing_ratio", 1, 1); + auto cloud_fraction = real2dk("cloud_fration", 1, 1); + auto cloud_mass = real2dk("cloud_mass", 1, 1); + + // Test with cell completely filled with cloud + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + dp(0,0) = 10.0; + mixing_ratio(0,0) = 0.0001; + cloud_fraction(0,0) = 1.0; + }); + auto cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; + scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); + + // Test with no cloud + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + dp(0,0) = 10.0; + mixing_ratio(0,0) = 0.0; + cloud_fraction(0,0) = 0.0; + }); + cloud_mass_ref = 0.0; + scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); + + // Test with empty clouds (cloud fraction but with no associated mixing ratio) + // This case could happen if we use a total cloud fraction, but compute layer + // cloud mass separately for liquid and ice. + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + dp(0,0) = 10.0; + mixing_ratio(0,0) = 0.0; + cloud_fraction(0,0) = 0.1; + }); + cloud_mass_ref = 0.0; + scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); + + // Test with cell half filled with cloud + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + dp(0,0) = 10.0; + mixing_ratio(0,0) = 0.0001; + cloud_fraction(0,0) = 0.5; + }); + cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; + scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); + + // Clean up + dp.deallocate(); + mixing_ratio.deallocate(); + cloud_fraction.deallocate(); + cloud_mass.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_test_limit_to_bounds_k") { + // Initialize YAKL + scream::init_kls(); + + // Test limiter function + auto arr = real2dk("arr", 2, 2); + auto arr_limited = real2dk("arr_limited", 2, 2); + + // Setup dummy array + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + arr(0,0) = 1.0; + arr(0,1) = 2.0; + arr(1,0) = 3.0; + arr(1,1) = 4.0; + }); + + // Limit to bounds that contain the data; should be no change in values + scream::rrtmgp::limit_to_bounds(arr, 0.0, 5.0, arr_limited); + REQUIRE(chc(arr)(0,0) == chc(arr_limited)(0,0)); + REQUIRE(chc(arr)(0,1) == chc(arr_limited)(0,1)); + REQUIRE(chc(arr)(1,0) == chc(arr_limited)(1,0)); + REQUIRE(chc(arr)(1,1) == chc(arr_limited)(1,1)); + + // Limit to bounds that do not completely contain the data; should be a change in values! + scream::rrtmgp::limit_to_bounds(arr, 1.5, 3.5, arr_limited); + REQUIRE(chc(arr_limited)(0,0) == 1.5); + REQUIRE(chc(arr_limited)(0,1) == 2.0); + REQUIRE(chc(arr_limited)(1,0) == 3.0); + REQUIRE(chc(arr_limited)(1,1) == 3.5); + arr.deallocate(); + arr_limited.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_test_zenith_k") { + + // Create some dummy data + int orbital_year = 1990; + double calday = 1.0000000000000000; + double eccen_ref = 1.6707719799280658E-002; + double mvelpp_ref = 4.9344679089867318; + double lambm0_ref = -3.2503635878519378E-002; + double obliqr_ref = 0.40912382465788016; + double delta_ref = -0.40302893695478670; + double eccf_ref = 1.0342222039093694; + double lat = -7.7397590528644963E-002; + double lon = 2.2584340271163548; + double coszrs_ref = 0.61243613606766745; + + // Test shr_orb_params() + // Get orbital parameters based on calendar day + double eccen; + double obliq; // obliquity in degrees + double mvelp; // moving vernal equinox long of perihelion; degrees? + double obliqr; + double lambm0; + double mvelpp; + // bool flag_print = false; + shr_orb_params_c2f(&orbital_year, &eccen, &obliq, &mvelp, + &obliqr, &lambm0, &mvelpp); //, flag_print); // Note fortran code has optional arg + REQUIRE(eccen == eccen_ref); + REQUIRE(obliqr == obliqr_ref); + REQUIRE(mvelpp == mvelpp_ref); + REQUIRE(lambm0 == lambm0_ref); + REQUIRE(mvelpp == mvelpp_ref); + + // Test shr_orb_decl() + double delta; + double eccf; + shr_orb_decl_c2f(calday, eccen, mvelpp, lambm0, + obliqr, &delta, &eccf); + REQUIRE(delta == delta_ref); + REQUIRE(eccf == eccf_ref ); + + double dt_avg = 0.; //3600.0000000000000; + double coszrs = shr_orb_cosz_c2f(calday, lat, lon, delta, dt_avg); + REQUIRE(std::abs(coszrs-coszrs_ref)<1e-14); + + // Another case, this time WITH dt_avg flag: + calday = 1.0833333333333333; + eccen = 1.6707719799280658E-002; + mvelpp = 4.9344679089867318; + lambm0 = -3.2503635878519378E-002; + obliqr = 0.40912382465788016; + delta = -0.40292121709083456; + eccf = 1.0342248931660425; + lat = -1.0724153591027763; + lon = 4.5284876076962712; + dt_avg = 3600.0000000000000; + coszrs_ref = 0.14559973262047626; + coszrs = shr_orb_cosz_c2f(calday, lat, lon, delta, dt_avg); + REQUIRE(std::abs(coszrs-coszrs_ref)<1e-14); +} + +TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { + using namespace ekat::logger; + using logger_t = Logger; + + ekat::Comm comm(MPI_COMM_WORLD); + auto logger = std::make_shared("",LogLevel::info,comm); + + // Initialize YAKL + scream::init_kls(); + + // Create arrays + const int ncol = 1; + const int nlay = 1; + const int nbnd = 14; + const int kbot = nlay; + auto sfc_flux_dir_nir = real1dk("sfc_flux_dir_nir", ncol); + auto sfc_flux_dir_vis = real1dk("sfc_flux_dir_vis", ncol); + auto sfc_flux_dif_nir = real1dk("sfc_flux_dif_nir", ncol); + auto sfc_flux_dif_vis = real1dk("sfc_flux_dif_vis", ncol); + + // Need to initialize RRTMGP with dummy gases + logger->info("Init gases...\n"); + GasConcsK gas_concs; + int ngas = 8; + string1d gas_names(ngas) = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; + gas_concs.init(gas_names,ncol,nlay); + logger->info("Init RRTMGP...\n"); + scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + + // Create simple test cases; We expect, given the input data, that band 10 + // will straddle the NIR and VIS, bands 1-9 will be purely NIR, and bands 11-14 + // will be purely VIS. The implementation in EAMF90 was hard-coded with this + // band information, but our implementation of compute_broadband_surface_fluxes + // actually checks the wavenumber limits. These tests will mostly check to make + // sure our implementation of that is doing what we think it is. + + // --------------------------------- + // Test case: flux only in straddled band + auto sw_bnd_flux_dir = real3dk("sw_bnd_flux_dir", ncol, nlay+1, nbnd); + auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); + logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + if (ibnd < 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 0; + } else if (ibnd == 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 1; + sw_bnd_flux_dif(icol,ilay,ibnd) = 1; + } else { + sw_bnd_flux_dir(icol,ilay,ibnd) = 0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 0; + } + }); + // Compute surface fluxes + logger->info("Compute broadband surface fluxes...\n"); + scream::rrtmgp::compute_broadband_surface_fluxes( + ncol, kbot, nbnd, + sw_bnd_flux_dir, sw_bnd_flux_dif, + sfc_flux_dir_vis, sfc_flux_dir_nir, + sfc_flux_dif_vis, sfc_flux_dif_nir + ); + // Check computed surface fluxes + logger->info("Check computed fluxes...\n"); + const double tol = 1e-10; // tolerance on floating point inequality for assertions + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 0.5) < tol); + // --------------------------------- + + // --------------------------------- + // Test case, only flux in NIR bands + logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + if (ibnd < 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 1; + sw_bnd_flux_dif(icol,ilay,ibnd) = 1; + } else if (ibnd == 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 0; + } else { + sw_bnd_flux_dir(icol,ilay,ibnd) = 0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 0; + } + }); + // Compute surface fluxes + logger->info("Compute broadband surface fluxes...\n"); + scream::rrtmgp::compute_broadband_surface_fluxes( + ncol, kbot, nbnd, + sw_bnd_flux_dir, sw_bnd_flux_dif, + sfc_flux_dir_vis, sfc_flux_dir_nir, + sfc_flux_dif_vis, sfc_flux_dif_nir + ); + // Check computed surface fluxes + logger->info("Check computed fluxes...\n"); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 9.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 9.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 0.0) < tol); + // --------------------------------- + + // --------------------------------- + // Test case, only flux in VIS bands + logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + if (ibnd < 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 0; + } else if (ibnd == 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 0; + } else { + sw_bnd_flux_dir(icol,ilay,ibnd) = 1; + sw_bnd_flux_dif(icol,ilay,ibnd) = 1; + } + }); + // Compute surface fluxes + logger->info("Compute broadband surface fluxes...\n"); + scream::rrtmgp::compute_broadband_surface_fluxes( + ncol, kbot, nbnd, + sw_bnd_flux_dir, sw_bnd_flux_dif, + sfc_flux_dir_vis, sfc_flux_dir_nir, + sfc_flux_dif_vis, sfc_flux_dif_nir + ); + // Check computed surface fluxes + logger->info("Check computed fluxes...\n"); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 4.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 4.0) < tol); + // --------------------------------- + + // --------------------------------- + // Test case, only flux in all bands + logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); + Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + if (ibnd < 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; + } else if (ibnd == 10) { + sw_bnd_flux_dir(icol,ilay,ibnd) = 3.0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 4.0; + } else { + sw_bnd_flux_dir(icol,ilay,ibnd) = 5.0; + sw_bnd_flux_dif(icol,ilay,ibnd) = 6.0; + } + }); + // Compute surface fluxes + logger->info("Compute broadband surface fluxes...\n"); + scream::rrtmgp::compute_broadband_surface_fluxes( + ncol, kbot, nbnd, + sw_bnd_flux_dir, sw_bnd_flux_dif, + sfc_flux_dir_vis, sfc_flux_dir_nir, + sfc_flux_dif_vis, sfc_flux_dif_nir + ); + // Check computed surface fluxes + logger->info("Check computed fluxes...\n"); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 10.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 21.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 20.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 26.0) < tol); + // --------------------------------- + + // Finalize YAKL + logger->info("Free memory...\n"); + scream::rrtmgp::rrtmgp_finalize(); + gas_concs.reset(); + gas_names.deallocate(); + sw_bnd_flux_dir.deallocate(); + sw_bnd_flux_dif.deallocate(); + sfc_flux_dir_nir.deallocate(); + sfc_flux_dir_vis.deallocate(); + sfc_flux_dif_nir.deallocate(); + sfc_flux_dif_vis.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_test_radiation_do_k") { + // If we specify rad every step, radiation_do should always be true + REQUIRE(scream::rrtmgp::radiation_do(1, 0) == true); + REQUIRE(scream::rrtmgp::radiation_do(1, 1) == true); + REQUIRE(scream::rrtmgp::radiation_do(1, 2) == true); + + // Test cases where we want rad called every other step + REQUIRE(scream::rrtmgp::radiation_do(2, 0) == true); + REQUIRE(scream::rrtmgp::radiation_do(2, 1) == false); + REQUIRE(scream::rrtmgp::radiation_do(2, 2) == true); + REQUIRE(scream::rrtmgp::radiation_do(2, 3) == false); + + // Test cases where we want rad every third step + REQUIRE(scream::rrtmgp::radiation_do(3, 0) == true); + REQUIRE(scream::rrtmgp::radiation_do(3, 1) == false); + REQUIRE(scream::rrtmgp::radiation_do(3, 2) == false); + REQUIRE(scream::rrtmgp::radiation_do(3, 3) == true); + REQUIRE(scream::rrtmgp::radiation_do(3, 4) == false); + REQUIRE(scream::rrtmgp::radiation_do(3, 5) == false); + REQUIRE(scream::rrtmgp::radiation_do(3, 6) == true); +} + +TEST_CASE("rrtmgp_test_check_range_k") { + // Initialize YAKL + scream::init_kls(); + // Create some dummy data and test with both values inside valid range and outside + auto dummy = real2dk("dummy", 2, 1); + // All values within range + Kokkos::deep_copy(dummy, 0.1); + REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == true); + // At least one value below lower bound + Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 1) = -0.1;}); + REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); + // At least one value above upper bound + Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 1) = 1.1;}); + REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); + dummy.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_test_subcol_gen_k") { + // Initialize YAKL + scream::init_kls(); + // Create dummy data + const int ncol = 1; + const int nlay = 4; + const int ngpt = 10; + auto cldfrac = real2dk("cldfrac", ncol, nlay); + // Set cldfrac values + Kokkos::deep_copy(cldfrac, 0.0); + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac(0,0) = 1; + cldfrac(0,1) = 0.5; + cldfrac(0,2) = 0; + cldfrac(0,3) = 1; + }); + auto cldmask = int3dk("cldmask", ncol, nlay, ngpt); + auto cldfrac_from_mask = real2dk("cldfrac_from_mask", ncol, nlay); + // Run subcol gen, make sure we get what we expect; do this for some different seed values + for (unsigned seed = 0; seed < 10; seed++) { + auto seeds = int1dk("seeds", ncol); + Kokkos::deep_copy(seeds, seed); + cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + // Check answers by computing new cldfrac from mask + Kokkos::deep_copy(cldfrac_from_mask, 0.0); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + for (int igpt = 0; igpt < ngpt; ++igpt) { + real cldmask_real = cldmask(icol,ilay,igpt); + cldfrac_from_mask(icol,ilay) += cldmask_real; + } + }); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + cldfrac_from_mask(icol,ilay) = cldfrac_from_mask(icol,ilay) / ngpt; + }); + // For cldfrac 1 we should get 1, for cldfrac 0 we should get 0, but in between we cannot be sure + // deterministically, since the computed cloud mask depends on pseudo-random numbers + REQUIRE(chc(cldfrac_from_mask)(0,0) == 1); + REQUIRE(chc(cldfrac_from_mask)(0,1) <= 1); + REQUIRE(chc(cldfrac_from_mask)(0,2) == 0); + REQUIRE(chc(cldfrac_from_mask)(0,3) == 1); + } + + // For maximum-random overlap, vertically-contiguous layers maximimally overlap, + // thus if we have non-zero cloud fraction in two adjacent layers, then every subcolumn + // that has cloud in the layer above must also have cloud in the layer below; test + // this property by creating two layers with non-zero cloud fraction, creating subcolums, + // and verifying that every subcolumn with cloud in layer 1 has cloud in layer 2 + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac(0,0) = 0.5; + cldfrac(0,1) = 0.5; + cldfrac(0,2) = 0; + cldfrac(0,3) = 0; + }); + for (unsigned seed = 0; seed < 10; seed++) { + auto seeds = int1dk("seeds", ncol); + Kokkos::deep_copy(seeds, seed); + cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + auto cldmask_h = chc(cldmask); + for (int igpt = 0; igpt < ngpt; igpt++) { + if (cldmask_h(0,0,igpt) == 1) { + REQUIRE(cldmask_h(0,1,igpt) == 1); + } + } + } + // Clean up after test + cldfrac.deallocate(); + cldmask.deallocate(); + cldfrac_from_mask.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_cloud_area_k") { + // Initialize YAKL + scream::init_kls(); + // Create dummy data + const int ncol = 1; + const int nlay = 2; + const int ngpt = 3; + auto cldtau = real3dk("cldtau", ncol, nlay, ngpt); + auto cldtot = real1dk("cldtot", ncol); + auto pmid = real2dk("pmid", ncol, nlay); + + // Set up pressure levels for test problem + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + pmid(0,0) = 100; + pmid(0,1) = 200; + }); + + // Case: + // + // 0 0 0 + // 0 0 0 + // + // should give cldtot = 0.0 + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldtau(0,0,0) = 0; + cldtau(0,0,1) = 0; + cldtau(0,0,2) = 0; + cldtau(0,1,0) = 0; + cldtau(0,1,1) = 0; + cldtau(0,1,2) = 0; + }); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 0.0); + + // Case: + // + // 1 1 1 + // 1 1 1 + // + // should give cldtot = 1.0 + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldtau(0,0,0) = 1; + cldtau(0,0,1) = 1; + cldtau(0,0,2) = 1; + cldtau(0,1,0) = 1; + cldtau(0,1,1) = 1; + cldtau(0,1,2) = 1; + }); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 1.0); + + // Case: + // + // 1 1 0 100 + // 0 0 1 200 + // + // should give cldtot = 1.0 + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldtau(0,0,0) = 0.1; + cldtau(0,0,1) = 1.5; + cldtau(0,0,2) = 0; + cldtau(0,1,0) = 0; + cldtau(0,1,1) = 0; + cldtau(0,1,2) = 1.0; + }); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 1.0); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 1.0 / 3.0); + + // Case: + // + // 1 0 0 + // 1 0 1 + // + // should give cldtot = 2/3 + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldtau(0,0,0) = 1; + cldtau(0,0,1) = 0; + cldtau(0,0,2) = 0; + cldtau(0,1,0) = 1; + cldtau(0,1,1) = 0; + cldtau(0,1,2) = 1; + }); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 0.0); + scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); + REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); + pmid.deallocate(); + cldtau.deallocate(); + cldtot.deallocate(); + scream::finalize_kls(); +} + +TEST_CASE("rrtmgp_aerocom_cloudtop_k") { + // Initialize YAKL + scream::init_kls(); + + // Create dummy data + const int ncol = 1; + const int nlay = 9; + // Set up input fields + auto tmid = real2dk("tmid", ncol, nlay); + auto pmid = real2dk("pmid", ncol, nlay); + auto p_del = real2dk("p_del", ncol, nlay); + auto z_del = real2dk("z_del", ncol, nlay); + auto qc = real2dk("qc", ncol, nlay); + auto qi = real2dk("qi", ncol, nlay); + auto rel = real2dk("rel", ncol, nlay); + auto rei = real2dk("rei", ncol, nlay); + auto cldfrac_tot = real2dk("cldfrac_tot", ncol, nlay); + auto nc = real2dk("nc", ncol, nlay); + // Set up output fields + auto tmid_at_cldtop = real1dk("tmid_at_cldtop", ncol); + auto pmid_at_cldtop = real1dk("pmid_at_cldtop", ncol); + auto cldfrac_ice_at_cldtop = real1dk("cldfrac_ice_at_cldtop", ncol); + auto cldfrac_liq_at_cldtop = real1dk("cldfrac_liq_at_cldtop", ncol); + auto cldfrac_tot_at_cldtop = real1dk("cldfrac_tot_at_cldtop", ncol); + auto cdnc_at_cldtop = real1dk("cdnc_at_cldtop", ncol); + auto eff_radius_qc_at_cldtop = real1dk("eff_radius_qc_at_cldtop", ncol); + auto eff_radius_qi_at_cldtop = real1dk("eff_radius_qi_at_cldtop", ncol); + + // Case 1: if no clouds, everything goes to zero + Kokkos::deep_copy(tmid, 300.0); + Kokkos::deep_copy(pmid, 100.0); + Kokkos::deep_copy(p_del, 10.0); + Kokkos::deep_copy(z_del, 100.0); + Kokkos::deep_copy(qc, 1.0); + Kokkos::deep_copy(qi, 1.0); + Kokkos::deep_copy(cldfrac_tot, 0.0); + Kokkos::deep_copy(nc, 5.0); + Kokkos::deep_copy(rel, 10.0); + Kokkos::deep_copy(rei, 10.0); + // Call the function + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + // Check the results + REQUIRE(chc(tmid_at_cldtop)(0) == 0.0); + REQUIRE(chc(pmid_at_cldtop)(0) == 0.0); + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) == 0.0); + REQUIRE(chc(cldfrac_liq_at_cldtop)(0) == 0.0); + REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.0); + REQUIRE(chc(cdnc_at_cldtop)(0) == 0.0); + REQUIRE(chc(eff_radius_qc_at_cldtop)(0) == 0.0); + REQUIRE(chc(eff_radius_qi_at_cldtop)(0) == 0.0); + + // Case 2: if all clouds, everything goes to 1 * its value + Kokkos::deep_copy(cldfrac_tot, 1.0); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(chc(tmid_at_cldtop)(0) == 300.0); + REQUIRE(chc(pmid_at_cldtop)(0) == 100.0); + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) == 1.0); + REQUIRE(chc(cldfrac_liq_at_cldtop)(0) == 0.5); + REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.5); + REQUIRE(chc(cdnc_at_cldtop)(0) > 0.0); + REQUIRE(chc(eff_radius_qc_at_cldtop)(0) > 0.0); + REQUIRE(chc(eff_radius_qi_at_cldtop)(0) > 0.0); + + // Case 3: test max overlap (if contiguous cloudy layers, then max) + Kokkos::deep_copy(cldfrac_tot, 0.0); + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac_tot(0, 1) = 0.5; + cldfrac_tot(0, 2) = 0.7; + cldfrac_tot(0, 3) = 0.3; + cldfrac_tot(0, 4) = 0.2; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) == .7); + + // Case 3xtra: test max overlap + // This case produces >0.7 due to slight enhancement in the presence of a + // local minimum (0.1 is the local minimum between 0.2 and 0.4) + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac_tot(0, 4) = 0.1; + cldfrac_tot(0, 5) = 0.4; + cldfrac_tot(0, 6) = 0.2; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) > .7); + + // Case 4: test random overlap (if non-contiguous cloudy layers, then + // random) + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac_tot(0, 4) = 0.0; + cldfrac_tot(0, 5) = 0.1; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) > .7); // larger than the max + + // Case 5a: test independence of ice and liquid fractions + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac_tot(0, 1) = 1.0; + cldfrac_tot(0, 6) = 1.0; + cldfrac_tot(0, 7) = 0.2; + }); + Kokkos::deep_copy(qc, 1.0); + Kokkos::deep_copy(qi, 0.0); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) == 1.0); + REQUIRE(chc(cldfrac_liq_at_cldtop)(0) == 1.0); + REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.0); + + // Case 5b: test independence of ice and liquid fractions + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac_tot(0, 1) = 1.0; + cldfrac_tot(0, 6) = 1.0; + cldfrac_tot(0, 7) = 0.2; + }); + Kokkos::deep_copy(qc, 0.0); + Kokkos::deep_copy(qi, 1.0); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) == 1.0); + REQUIRE(chc(cldfrac_liq_at_cldtop)(0) == 0.0); + REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 1.0); + + // Case 6: test independence of ice and liquid fractions + // There is NOT complete independence... + // Essentially, higher ice clouds mask lower liquid clouds + // This can be problematic if the ice clouds are thin... + // We will revisit and validate this assumption later + Kokkos::deep_copy(cldfrac_tot, 0.0); + Kokkos::deep_copy(qc, 0.0); + Kokkos::deep_copy(qi, 0.0); + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + cldfrac_tot(0, 1) = 0.5; // ice + cldfrac_tot(0, 2) = 0.7; // ice ------> max + cldfrac_tot(0, 3) = 0.3; // ice + // note cldfrac_tot(1, 5) is 0 + cldfrac_tot(0, 5) = 0.2; // liq + cldfrac_tot(0, 6) = 0.5; // liq ------> not max + cldfrac_tot(0, 7) = 0.1; // liq + // note cldfrac_tot(1, 9) is 0 + qi(0, 1) = 100; + qi(0, 2) = 200; + qi(0, 3) = 50; + // note qc(1, 5) is 0 + // note qi(1, 5) is 0 + qc(0, 5) = 20; + qc(0, 6) = 50; + qc(0, 7) = 10; + }); + scream::rrtmgp::compute_aerocom_cloudtop( + ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, + tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); + REQUIRE(chc(cldfrac_tot_at_cldtop)(0) > 0.70); // unaffected + REQUIRE(chc(cldfrac_liq_at_cldtop)(0) < 0.50); // not max + REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.7); // max + + // cleanup + tmid.deallocate(); + pmid.deallocate(); + p_del.deallocate(); + z_del.deallocate(); + qc.deallocate(); + qi.deallocate(); + rel.deallocate(); + rei.deallocate(); + cldfrac_tot.deallocate(); + nc.deallocate(); + + tmid_at_cldtop.deallocate(); + pmid_at_cldtop.deallocate(); + cldfrac_ice_at_cldtop.deallocate(); + cldfrac_liq_at_cldtop.deallocate(); + cldfrac_tot_at_cldtop.deallocate(); + cdnc_at_cldtop.deallocate(); + eff_radius_qc_at_cldtop.deallocate(); + eff_radius_qi_at_cldtop.deallocate(); + + scream::finalize_kls(); +} +#endif + +} From 33f26110bec060a6475440fd089680e9452688ae Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 23 Apr 2024 13:29:30 -0600 Subject: [PATCH 045/476] builds --- components/eamxx/CMakeLists.txt | 17 +- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 11 +- .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 1 + .../rrtmgp/scream_rrtmgp_interface.cpp | 16 +- .../src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 6 +- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 51 +- .../rrtmgp/rrtmgp_standalone_unit.cpp | 1089 +++++++++++------ 7 files changed, 741 insertions(+), 450 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 453b2d4be8a3..cf8936c940ad 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -189,6 +189,9 @@ if (SCREAM_DEBUG) endif () endif() +set(DEFAULT_RRTMGP_ENABLE_YAKL TRUE) +set(DEFAULT_RRTMGP_ENABLE_KOKKOS FALSE) + ### Now that reasonable defaults have been computed, set CACHE vars set(SCREAM_MIMIC_GPU ${DEFAULT_MIMIC_GPU} CACHE BOOL "Mimic GPU to correctness-test inter-column parallelism on non-GPU platform") set(SCREAM_PACK_CHECK_BOUNDS FALSE CACHE BOOL "If defined, scream::pack objects check indices against bounds") @@ -203,13 +206,24 @@ if (NOT SCREAM_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() -# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS +# Add RRTMGP settings. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS # to turn on the RRTMGP internal checks here as well, via # option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG}) # and then adding to scream_config.h: # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" ${DEFAULT_RRTMGP_ENABLE_YAKL}) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" ${DEFAULT_RRTMGP_ENABLE_KOKKOS}) +if (SCREAM_RRTMGP_ENABLE_YAKL) + add_definitions("-DRRTMGP_ENABLE_YAKL") +endif() + +if (SCREAM_RRTMGP_ENABLE_KOKKOS) + add_definitions("-DRRTMGP_ENABLE_KOKKOS") +endif() + + set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") # For now, only used in share/grid/remap/refining_remapper_rma.*pp @@ -618,4 +632,3 @@ print_var(SCREAM_TEST_THREAD_INC) print_var(SCREAM_TEST_MAX_RANKS) message ("**************************************************") - diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 1c7736de76b4..446c1842192e 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -2,14 +2,6 @@ include(EkatUtils) include(EkatSetCompilerFlags) include(ScreamUtils) -if (SCREAM_RRTMGP_ENABLE_YAKL) - add_definitions("-DRRTMGP_ENABLE_YAKL") -endif() - -if (SCREAM_RRTMGP_ENABLE_KOKKOS) - add_definitions("-DRRTMGP_ENABLE_KOKKOS") -endif() - # Copied from EKAT, YAKL is an interface target so requires special # handling. Get rid of this once RRTMGP is using kokkos. macro (SetCudaFlagsYakl targetName) @@ -125,6 +117,8 @@ set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_rte_solver_kernels.cpp ${EAM_RRTMGP_DIR}/external/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_garand_atmos_io.cpp + ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_load_cloud_coefficients.cpp + ${EAM_RRTMGP_DIR}/external/cpp/examples/mo_load_coefficients.cpp ) add_library(rrtmgp ${EXTERNAL_SRC}) target_compile_definitions(rrtmgp PUBLIC EAMXX_HAS_RRTMGP) @@ -139,6 +133,7 @@ target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL ${EAM_RRTMGP_DIR}/external/cpp ${EAM_RRTMGP_DIR}/external/cpp/extensions/cloud_optics + ${EAM_RRTMGP_DIR}/external/cpp/examples ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky ${EAM_RRTMGP_DIR}/external/cpp/rte ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 947485705927..f72acd7528e2 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -3,6 +3,7 @@ #include "physics/share/physics_constants.hpp" #include "cpp/rrtmgp_const.h" +#include "cpp/rrtmgp_conversion.h" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL.h" diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 20d1c1564d79..e0f22c79eeed 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -52,12 +52,12 @@ using yakl::intrinsics::merge; * declare them here within the rrtmgp namespace. */ #ifdef RRTMGP_ENABLE_YAKL -extern GasOpticsRRTMGP k_dist_sw; -extern GasOpticsRRTMGP k_dist_lw; +GasOpticsRRTMGP k_dist_sw; +GasOpticsRRTMGP k_dist_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS -extern GasOpticsRRTMGPK k_dist_sw_k; -extern GasOpticsRRTMGPK k_dist_lw_k; +GasOpticsRRTMGPK k_dist_sw_k; +GasOpticsRRTMGPK k_dist_lw_k; #endif /* @@ -66,12 +66,12 @@ extern GasOpticsRRTMGPK k_dist_lw_k; * program, so declare here and read data in during rrtmgp_initialize(). */ #ifdef RRTMGP_ENABLE_YAKL -extern CloudOptics cloud_optics_sw; -extern CloudOptics cloud_optics_lw; +CloudOptics cloud_optics_sw; +CloudOptics cloud_optics_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS -extern CloudOpticsK cloud_optics_sw_k; -extern CloudOpticsK cloud_optics_lw_k; +CloudOpticsK cloud_optics_sw_k; +CloudOpticsK cloud_optics_lw_k; #endif bool initialized = false; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 2526a5401fda..26c7ff2b6305 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -107,8 +107,9 @@ int run_yakl(int argc, char** argv) { real2d t_lay("t_lay", ncol, nlay); real2d p_lev("p_lev", ncol, nlay+1); real2d t_lev("t_lev", ncol, nlay+1); + real2d col_dry; GasConcs gas_concs; - read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, ncol); + read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); @@ -391,8 +392,9 @@ int run_kokkos(int argc, char** argv) { real2dk t_lay("t_lay", ncol, nlay); real2dk p_lev("p_lev", ncol, nlay+1); real2dk t_lev("t_lev", ncol, nlay+1); + real2dk col_dry; GasConcsK gas_concs; - read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, ncol); + read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 0a6d5bcde247..03e2e8861ed7 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -3,7 +3,7 @@ #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/share/physics_constants.hpp" #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" -#include "physics/rrtmgp/mo_load_coefficients.h" +#include "mo_load_coefficients.h" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL.h" @@ -880,7 +880,7 @@ TEST_CASE("rrtmgp_test_heating_k") { // Simple net postive heating; net flux into layer should be 1.0 // NOTE: Kokkos::parallel_for because we need to do these in a kernel on the device - Kokkoks::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { + Kokkos::parallel_for(1, KOKKOS_LAMBDA(int /* dummy */) { flux_up(0, 0) = 1.0; flux_up(0, 1) = 1.0; flux_dn(0, 0) = 1.5; @@ -907,10 +907,6 @@ TEST_CASE("rrtmgp_test_heating_k") { REQUIRE(chc(heating)(0,0) == heating_ref); // Clean up - dp.deallocate(); - flux_up.deallocate(); - flux_dn.deallocate(); - heating.deallocate(); scream::finalize_kls(); } @@ -969,10 +965,6 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up - dp.deallocate(); - mixing_ratio.deallocate(); - cloud_fraction.deallocate(); - cloud_mass.deallocate(); scream::finalize_kls(); } @@ -1005,8 +997,6 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); REQUIRE(chc(arr_limited)(1,1) == 3.5); - arr.deallocate(); - arr_limited.deallocate(); scream::finalize_kls(); } @@ -1093,8 +1083,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); GasConcsK gas_concs; - int ngas = 8; - string1d gas_names(ngas) = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; + string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); @@ -1237,13 +1226,6 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { logger->info("Free memory...\n"); scream::rrtmgp::rrtmgp_finalize(); gas_concs.reset(); - gas_names.deallocate(); - sw_bnd_flux_dir.deallocate(); - sw_bnd_flux_dif.deallocate(); - sfc_flux_dir_nir.deallocate(); - sfc_flux_dir_vis.deallocate(); - sfc_flux_dif_nir.deallocate(); - sfc_flux_dif_vis.deallocate(); scream::finalize_kls(); } @@ -1283,7 +1265,6 @@ TEST_CASE("rrtmgp_test_check_range_k") { // At least one value above upper bound Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 1) = 1.1;}); REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); - dummy.deallocate(); scream::finalize_kls(); } @@ -1352,9 +1333,6 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { } } // Clean up after test - cldfrac.deallocate(); - cldmask.deallocate(); - cldfrac_from_mask.deallocate(); scream::finalize_kls(); } @@ -1450,9 +1428,6 @@ TEST_CASE("rrtmgp_cloud_area_k") { REQUIRE(chc(cldtot)(0) == 0.0); scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - pmid.deallocate(); - cldtau.deallocate(); - cldtot.deallocate(); scream::finalize_kls(); } @@ -1647,26 +1622,6 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.7); // max // cleanup - tmid.deallocate(); - pmid.deallocate(); - p_del.deallocate(); - z_del.deallocate(); - qc.deallocate(); - qi.deallocate(); - rel.deallocate(); - rei.deallocate(); - cldfrac_tot.deallocate(); - nc.deallocate(); - - tmid_at_cldtop.deallocate(); - pmid_at_cldtop.deallocate(); - cldfrac_ice_at_cldtop.deallocate(); - cldfrac_liq_at_cldtop.deallocate(); - cldfrac_tot_at_cldtop.deallocate(); - cdnc_at_cldtop.deallocate(); - eff_radius_qc_at_cldtop.deallocate(); - eff_radius_qi_at_cldtop.deallocate(); - scream::finalize_kls(); } #endif diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 9f1ca00aaff8..5197104787d0 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -22,7 +22,9 @@ // RRTMGP and YAKL #include #include +#ifdef RRTMGP_ENABLE_YAKL #include +#endif // System headers #include @@ -32,388 +34,711 @@ * Run standalone test problem for RRTMGP and compare with baseline */ -namespace scream { - - using KT = KokkosTypes; - using ExeSpace = KT::ExeSpace; - using MemberType = KT::MemberType; - - /* - * Run standalone test through SCREAM driver this time - */ - TEST_CASE("rrtmgp_scream_standalone", "") { - using namespace scream; - using namespace scream::control; - using PC = scream::physics::Constants; - - // Get baseline name (needs to be passed as an arg) - std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); - std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); - - // Check if files exists - REQUIRE(rrtmgpTest::file_exists(inputfile.c_str())); - REQUIRE(rrtmgpTest::file_exists(baseline.c_str())); - - // Initialize yakl - if(!yakl::isInitialized()) { yakl::init(); } - - // Read reference fluxes from baseline file - real2d sw_flux_up_ref; - real2d sw_flux_dn_ref; - real2d sw_flux_dn_dir_ref; - real2d lw_flux_up_ref; - real2d lw_flux_dn_ref; - rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); - - // Load ad parameter list - std::string fname = "input_unit.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - // Create a MPI communicator - ekat::Comm atm_comm (MPI_COMM_WORLD); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics (); - register_mesh_free_grids_manager(); - - // Create the driver - AtmosphereDriver ad; - - // Dummy timestamp - util::TimeStamp time ({2000,1,1},{0,0,0}); - - // Initialize the driver - ad.initialize(atm_comm, ad_params, time); - - /* - * Setup the dummy problem and overwrite default initial conditions - */ - - // Get dimension sizes from the field manager - const auto& grid = ad.get_grids_manager()->get_grid("Point Grid"); - const auto& field_mgr = *ad.get_field_mgr(grid->name()); - int ncol = grid->get_num_local_dofs(); - int nlay = grid->get_num_vertical_levels(); - - // In this test, we need to hack lat/lon. But the fields we get - // from the grid are read-only. Therefore, hack a bit, and cast - // away constness. It's bad, but it's only for this unit test - auto clat = grid->get_geometry_data("lat").get_view(); - auto clon = grid->get_geometry_data("lon").get_view(); - auto lat = const_cast(clat.data()); - auto lon = const_cast(clon.data()); - - // Get number of shortwave bands and number of gases from RRTMGP - int ngas = 8; // TODO: get this intelligently - - // Make sure we have the right dimension sizes - REQUIRE(nlay == static_cast(sw_flux_up_ref.dimension[1])-1); - - // Create yakl arrays to store the input data - auto p_lay = real2d("p_lay", ncol, nlay); - auto t_lay = real2d("t_lay", ncol, nlay); - auto p_del = real2d("p_del", ncol, nlay); - auto p_lev = real2d("p_lev", ncol, nlay+1); - auto t_lev = real2d("t_lev", ncol, nlay+1); - auto sfc_alb_dir_vis = real1d("sfc_alb_dir_vis", ncol); - auto sfc_alb_dir_nir = real1d("sfc_alb_dir_nir", ncol); - auto sfc_alb_dif_vis = real1d("sfc_alb_dif_vis", ncol); - auto sfc_alb_dif_nir = real1d("sfc_alb_dif_nir", ncol); - auto lwp = real2d("lwp", ncol, nlay); - auto iwp = real2d("iwp", ncol, nlay); - auto rel = real2d("rel", ncol, nlay); - auto rei = real2d("rei", ncol, nlay); - auto cld = real2d("cld", ncol, nlay); - auto mu0 = real1d("mu0", ncol); - auto gas_vmr = real3d("gas_vmr", ncol, nlay, ngas); - - // Setup dummy problem; need to use tmp arrays with ncol_all size - auto ncol_all = grid->get_num_global_dofs(); - auto p_lay_all = real2d("p_lay", ncol_all, nlay); - auto t_lay_all = real2d("t_lay", ncol_all, nlay); - auto p_lev_all = real2d("p_lev", ncol_all, nlay+1); - auto t_lev_all = real2d("t_lev", ncol_all, nlay+1); - auto sfc_alb_dir_vis_all = real1d("sfc_alb_dir_vis", ncol_all); - auto sfc_alb_dir_nir_all = real1d("sfc_alb_dir_nir", ncol_all); - auto sfc_alb_dif_vis_all = real1d("sfc_alb_dif_vis", ncol_all); - auto sfc_alb_dif_nir_all = real1d("sfc_alb_dif_nir", ncol_all); - auto lwp_all = real2d("lwp", ncol_all, nlay); - auto iwp_all = real2d("iwp", ncol_all, nlay); - auto rel_all = real2d("rel", ncol_all, nlay); - auto rei_all = real2d("rei", ncol_all, nlay); - auto cld_all = real2d("cld", ncol_all, nlay); - auto mu0_all = real1d("mu0", ncol_all); - // Read in dummy Garand atmosphere; if this were an actual model simulation, - // these would be passed as inputs to the driver - // NOTE: set ncol to size of col_flx dimension in the input file. This is so - // that we can compare to the reference data provided in that file. Note that - // this will copy the first column of the input data (the first profile) ncol - // times. We will then fill some fraction of these columns with clouds for - // the test problem. - GasConcs gas_concs; - read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, ncol_all); - // Setup dummy problem; need to use tmp arrays with ncol_all size - rrtmgpTest::dummy_atmos( - inputfile, ncol_all, p_lay_all, t_lay_all, - sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, - sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, - mu0_all, - lwp_all, iwp_all, rel_all, rei_all, cld_all - ); - // Populate our local input arrays with the proper columns, based on our rank - int irank = atm_comm.rank(); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - auto icol_all = ncol * irank + icol; - sfc_alb_dir_vis(icol) = sfc_alb_dir_vis_all(icol_all); - sfc_alb_dir_nir(icol) = sfc_alb_dir_nir_all(icol_all); - sfc_alb_dif_vis(icol) = sfc_alb_dif_vis_all(icol_all); - sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); - mu0(icol) = mu0_all(icol_all); - }); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { - auto icol_all = ncol * irank + icol; - p_lay(icol,ilay) = p_lay_all(icol_all,ilay); - t_lay(icol,ilay) = t_lay_all(icol_all,ilay); - lwp(icol,ilay) = lwp_all(icol_all,ilay); - iwp(icol,ilay) = iwp_all(icol_all,ilay); - rel(icol,ilay) = rel_all(icol_all,ilay); - rei(icol,ilay) = rei_all(icol_all,ilay); - cld(icol,ilay) = cld_all(icol_all,ilay); - }); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilay, int icol) { - auto icol_all = ncol * irank + icol; - p_lev(icol,ilay) = p_lev_all(icol_all,ilay); - t_lev(icol,ilay) = t_lev_all(icol_all,ilay); - }); - // Free temporary variables - p_lay_all.deallocate(); - t_lay_all.deallocate(); - p_lev_all.deallocate(); - t_lev_all.deallocate(); - sfc_alb_dir_vis_all.deallocate(); - sfc_alb_dir_nir_all.deallocate(); - sfc_alb_dif_vis_all.deallocate(); - sfc_alb_dif_nir_all.deallocate(); - lwp_all.deallocate(); - iwp_all.deallocate(); - rel_all.deallocate(); - rei_all.deallocate(); - cld_all.deallocate(); - mu0_all.deallocate(); - - // Need to calculate a dummy pseudo_density for our test problem - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { - p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); - }); - - // We do not want to pass lwp and iwp through the FM, so back out qc and qi for this test - // NOTE: test problem provides lwp/iwp in g/m2, not kg/m2! Factor of 1e3 here! - using physconst = scream::physics::Constants; - auto qc = real2d("qc", ncol, nlay); - auto nc = real2d("nc", ncol, nlay); - auto qi = real2d("qi", ncol, nlay); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay, ncol), YAKL_LAMBDA(int ilay, int icol) { - qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * physconst::gravit / p_del(icol,ilay); - qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * physconst::gravit / p_del(icol,ilay); - }); - - // Copy gases from gas_concs to gas_vmr array - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<3>(ncol,nlay,ngas), YAKL_LAMBDA(int icol, int ilay, int igas) { - auto icol_all = ncol * irank + icol; - gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); - }); - gas_concs.reset(); - - // Before running, make a copy of T_mid so we can see changes - auto T_mid0 = real2d("T_mid0", ncol, nlay); - t_lay.deep_copy_to(T_mid0); - - // Grab views from field manager and copy in values from yakl arrays. Making - // copies is necessary since the yakl::Array take in data arranged with ncol - // as the fastest index, but the field manager expects the 2nd dimension as - // the fastest index. - auto d_pmid = field_mgr.get_field("p_mid").get_view(); - auto d_tmid = field_mgr.get_field("T_mid").get_view(); - auto d_pint = field_mgr.get_field("p_int").get_view(); - auto d_pdel = field_mgr.get_field("pseudo_density").get_view(); - auto d_sfc_alb_dir_vis = field_mgr.get_field("sfc_alb_dir_vis").get_view(); - auto d_sfc_alb_dir_nir = field_mgr.get_field("sfc_alb_dir_nir").get_view(); - auto d_sfc_alb_dif_vis = field_mgr.get_field("sfc_alb_dif_vis").get_view(); - auto d_sfc_alb_dif_nir = field_mgr.get_field("sfc_alb_dif_nir").get_view(); - auto d_surf_lw_flux_up = field_mgr.get_field("surf_lw_flux_up").get_view(); - auto d_qc = field_mgr.get_field("qc").get_view(); - auto d_nc = field_mgr.get_field("nc").get_view(); - auto d_qi = field_mgr.get_field("qi").get_view(); - auto d_rel = field_mgr.get_field("eff_radius_qc").get_view(); - auto d_rei = field_mgr.get_field("eff_radius_qi").get_view(); - auto d_cld = field_mgr.get_field("cldfrac_tot").get_view(); - - auto d_qv = field_mgr.get_field("qv").get_view(); - auto d_co2 = field_mgr.get_field("co2_volume_mix_ratio").get_view(); - auto d_o3 = field_mgr.get_field("o3_volume_mix_ratio").get_view(); - auto d_n2o = field_mgr.get_field("n2o_volume_mix_ratio").get_view(); - auto d_co = field_mgr.get_field("co_volume_mix_ratio").get_view(); - auto d_ch4 = field_mgr.get_field("ch4_volume_mix_ratio").get_view(); - auto d_o2 = field_mgr.get_field("o2_volume_mix_ratio").get_view(); - auto d_n2 = field_mgr.get_field("n2_volume_mix_ratio").get_view(); - - // Gather molecular weights of all the active gases in the test for conversion - // to mass-mixing-ratio. - { - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlay); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - const int i = team.league_rank(); - - // Set lat and lon to single value for just this test: - // Note, these values will ensure that the cosine zenith - // angle will end up matching the constant value meant for - // the test, which is 0.86 - lat[i] = 5.224000000000; - lon[i] = 167.282000000000; - - d_sfc_alb_dir_vis(i) = sfc_alb_dir_vis(i+1); - d_sfc_alb_dir_nir(i) = sfc_alb_dir_nir(i+1); - d_sfc_alb_dif_vis(i) = sfc_alb_dif_vis(i+1); - d_sfc_alb_dif_nir(i) = sfc_alb_dif_nir(i+1); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { - d_pmid(i,k) = p_lay(i+1,k+1); - d_tmid(i,k) = t_lay(i+1,k+1); - d_pdel(i,k) = p_del(i+1,k+1); - d_qc(i,k) = qc(i+1,k+1); - d_nc(i,k) = nc(i+1,k+1); - d_qi(i,k) = qi(i+1,k+1); - d_rel(i,k) = rel(i+1,k+1); - d_rei(i,k) = rei(i+1,k+1); - d_cld(i,k) = cld(i+1,k+1); - d_pint(i,k) = p_lev(i+1,k+1); - // qv specified as a wet mixing ratio - Real qv_dry = gas_vmr(i+1,k+1,1)*PC::ep_2; - Real qv_wet = qv_dry/(1.0+qv_dry); - d_qv(i,k) = qv_wet; - // rest of active gases are specified as volume mixing ratios - d_co2(i,k) = gas_vmr(i+1,k+1,2); - d_o3(i,k) = gas_vmr(i+1,k+1,3); - d_n2o(i,k) = gas_vmr(i+1,k+1,4); - d_co(i,k) = gas_vmr(i+1,k+1,5); - d_ch4(i,k) = gas_vmr(i+1,k+1,6); - d_o2(i,k) = gas_vmr(i+1,k+1,7); - d_n2(i,k) = gas_vmr(i+1,k+1,8); - }); - - d_pint(i,nlay) = p_lev(i+1,nlay+1); - - // Compute surface flux from surface temperature - auto ibot = (p_lay(1,1) > p_lay(1,nlay)) ? 1 : nlay+1; - d_surf_lw_flux_up(i) = PC::stebol * pow(t_lev(i+1,ibot), 4.0); +namespace { + +using namespace scream; +using namespace scream::control; +using KT = KokkosTypes; +using ExeSpace = KT::ExeSpace; +using MemberType = KT::MemberType; +using PC = scream::physics::Constants; + +/* + * Run standalone test through SCREAM driver this time + */ +#ifdef RRTMGP_ENABLE_YAKL +TEST_CASE("rrtmgp_scream_standalone", "") { + // Get baseline name (needs to be passed as an arg) + std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); + std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); + + // Check if files exists + REQUIRE(rrtmgpTest::file_exists(inputfile.c_str())); + REQUIRE(rrtmgpTest::file_exists(baseline.c_str())); + + // Initialize yakl + if(!yakl::isInitialized()) { yakl::init(); } + + // Read reference fluxes from baseline file + real2d sw_flux_up_ref; + real2d sw_flux_dn_ref; + real2d sw_flux_dn_dir_ref; + real2d lw_flux_up_ref; + real2d lw_flux_dn_ref; + rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + + // Load ad parameter list + std::string fname = "input_unit.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + // Create a MPI communicator + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics (); + register_mesh_free_grids_manager(); + + // Create the driver + AtmosphereDriver ad; + + // Dummy timestamp + util::TimeStamp time ({2000,1,1},{0,0,0}); + + // Initialize the driver + ad.initialize(atm_comm, ad_params, time); + + /* + * Setup the dummy problem and overwrite default initial conditions + */ + + // Get dimension sizes from the field manager + const auto& grid = ad.get_grids_manager()->get_grid("Point Grid"); + const auto& field_mgr = *ad.get_field_mgr(grid->name()); + int ncol = grid->get_num_local_dofs(); + int nlay = grid->get_num_vertical_levels(); + + // In this test, we need to hack lat/lon. But the fields we get + // from the grid are read-only. Therefore, hack a bit, and cast + // away constness. It's bad, but it's only for this unit test + auto clat = grid->get_geometry_data("lat").get_view(); + auto clon = grid->get_geometry_data("lon").get_view(); + auto lat = const_cast(clat.data()); + auto lon = const_cast(clon.data()); + + // Get number of shortwave bands and number of gases from RRTMGP + int ngas = 8; // TODO: get this intelligently + + // Make sure we have the right dimension sizes + REQUIRE(nlay == static_cast(sw_flux_up_ref.dimension[1])-1); + + // Create yakl arrays to store the input data + auto p_lay = real2d("p_lay", ncol, nlay); + auto t_lay = real2d("t_lay", ncol, nlay); + auto p_del = real2d("p_del", ncol, nlay); + auto p_lev = real2d("p_lev", ncol, nlay+1); + auto t_lev = real2d("t_lev", ncol, nlay+1); + auto sfc_alb_dir_vis = real1d("sfc_alb_dir_vis", ncol); + auto sfc_alb_dir_nir = real1d("sfc_alb_dir_nir", ncol); + auto sfc_alb_dif_vis = real1d("sfc_alb_dif_vis", ncol); + auto sfc_alb_dif_nir = real1d("sfc_alb_dif_nir", ncol); + auto lwp = real2d("lwp", ncol, nlay); + auto iwp = real2d("iwp", ncol, nlay); + auto rel = real2d("rel", ncol, nlay); + auto rei = real2d("rei", ncol, nlay); + auto cld = real2d("cld", ncol, nlay); + auto mu0 = real1d("mu0", ncol); + auto gas_vmr = real3d("gas_vmr", ncol, nlay, ngas); + + // Setup dummy problem; need to use tmp arrays with ncol_all size + auto ncol_all = grid->get_num_global_dofs(); + auto p_lay_all = real2d("p_lay", ncol_all, nlay); + auto t_lay_all = real2d("t_lay", ncol_all, nlay); + auto p_lev_all = real2d("p_lev", ncol_all, nlay+1); + auto t_lev_all = real2d("t_lev", ncol_all, nlay+1); + auto sfc_alb_dir_vis_all = real1d("sfc_alb_dir_vis", ncol_all); + auto sfc_alb_dir_nir_all = real1d("sfc_alb_dir_nir", ncol_all); + auto sfc_alb_dif_vis_all = real1d("sfc_alb_dif_vis", ncol_all); + auto sfc_alb_dif_nir_all = real1d("sfc_alb_dif_nir", ncol_all); + auto lwp_all = real2d("lwp", ncol_all, nlay); + auto iwp_all = real2d("iwp", ncol_all, nlay); + auto rel_all = real2d("rel", ncol_all, nlay); + auto rei_all = real2d("rei", ncol_all, nlay); + auto cld_all = real2d("cld", ncol_all, nlay); + auto mu0_all = real1d("mu0", ncol_all); + // Read in dummy Garand atmosphere; if this were an actual model simulation, + // these would be passed as inputs to the driver + // NOTE: set ncol to size of col_flx dimension in the input file. This is so + // that we can compare to the reference data provided in that file. Note that + // this will copy the first column of the input data (the first profile) ncol + // times. We will then fill some fraction of these columns with clouds for + // the test problem. + GasConcs gas_concs; + read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, ncol_all); + // Setup dummy problem; need to use tmp arrays with ncol_all size + rrtmgpTest::dummy_atmos( + inputfile, ncol_all, p_lay_all, t_lay_all, + sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, + sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, + mu0_all, + lwp_all, iwp_all, rel_all, rei_all, cld_all + ); + // Populate our local input arrays with the proper columns, based on our rank + int irank = atm_comm.rank(); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { + auto icol_all = ncol * irank + icol; + sfc_alb_dir_vis(icol) = sfc_alb_dir_vis_all(icol_all); + sfc_alb_dir_nir(icol) = sfc_alb_dir_nir_all(icol_all); + sfc_alb_dif_vis(icol) = sfc_alb_dif_vis_all(icol_all); + sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); + mu0(icol) = mu0_all(icol_all); + }); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { + auto icol_all = ncol * irank + icol; + p_lay(icol,ilay) = p_lay_all(icol_all,ilay); + t_lay(icol,ilay) = t_lay_all(icol_all,ilay); + lwp(icol,ilay) = lwp_all(icol_all,ilay); + iwp(icol,ilay) = iwp_all(icol_all,ilay); + rel(icol,ilay) = rel_all(icol_all,ilay); + rei(icol,ilay) = rei_all(icol_all,ilay); + cld(icol,ilay) = cld_all(icol_all,ilay); + }); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilay, int icol) { + auto icol_all = ncol * irank + icol; + p_lev(icol,ilay) = p_lev_all(icol_all,ilay); + t_lev(icol,ilay) = t_lev_all(icol_all,ilay); + }); + // Free temporary variables + p_lay_all.deallocate(); + t_lay_all.deallocate(); + p_lev_all.deallocate(); + t_lev_all.deallocate(); + sfc_alb_dir_vis_all.deallocate(); + sfc_alb_dir_nir_all.deallocate(); + sfc_alb_dif_vis_all.deallocate(); + sfc_alb_dif_nir_all.deallocate(); + lwp_all.deallocate(); + iwp_all.deallocate(); + rel_all.deallocate(); + rei_all.deallocate(); + cld_all.deallocate(); + mu0_all.deallocate(); + + // Need to calculate a dummy pseudo_density for our test problem + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { + p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); + }); + + // We do not want to pass lwp and iwp through the FM, so back out qc and qi for this test + // NOTE: test problem provides lwp/iwp in g/m2, not kg/m2! Factor of 1e3 here! + auto qc = real2d("qc", ncol, nlay); + auto nc = real2d("nc", ncol, nlay); + auto qi = real2d("qi", ncol, nlay); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay, ncol), YAKL_LAMBDA(int ilay, int icol) { + qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); + qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); + }); + + // Copy gases from gas_concs to gas_vmr array + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<3>(ncol,nlay,ngas), YAKL_LAMBDA(int icol, int ilay, int igas) { + auto icol_all = ncol * irank + icol; + gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); + }); + gas_concs.reset(); + + // Before running, make a copy of T_mid so we can see changes + auto T_mid0 = real2d("T_mid0", ncol, nlay); + t_lay.deep_copy_to(T_mid0); + + // Grab views from field manager and copy in values from yakl arrays. Making + // copies is necessary since the yakl::Array take in data arranged with ncol + // as the fastest index, but the field manager expects the 2nd dimension as + // the fastest index. + auto d_pmid = field_mgr.get_field("p_mid").get_view(); + auto d_tmid = field_mgr.get_field("T_mid").get_view(); + auto d_pint = field_mgr.get_field("p_int").get_view(); + auto d_pdel = field_mgr.get_field("pseudo_density").get_view(); + auto d_sfc_alb_dir_vis = field_mgr.get_field("sfc_alb_dir_vis").get_view(); + auto d_sfc_alb_dir_nir = field_mgr.get_field("sfc_alb_dir_nir").get_view(); + auto d_sfc_alb_dif_vis = field_mgr.get_field("sfc_alb_dif_vis").get_view(); + auto d_sfc_alb_dif_nir = field_mgr.get_field("sfc_alb_dif_nir").get_view(); + auto d_surf_lw_flux_up = field_mgr.get_field("surf_lw_flux_up").get_view(); + auto d_qc = field_mgr.get_field("qc").get_view(); + auto d_nc = field_mgr.get_field("nc").get_view(); + auto d_qi = field_mgr.get_field("qi").get_view(); + auto d_rel = field_mgr.get_field("eff_radius_qc").get_view(); + auto d_rei = field_mgr.get_field("eff_radius_qi").get_view(); + auto d_cld = field_mgr.get_field("cldfrac_tot").get_view(); + + auto d_qv = field_mgr.get_field("qv").get_view(); + auto d_co2 = field_mgr.get_field("co2_volume_mix_ratio").get_view(); + auto d_o3 = field_mgr.get_field("o3_volume_mix_ratio").get_view(); + auto d_n2o = field_mgr.get_field("n2o_volume_mix_ratio").get_view(); + auto d_co = field_mgr.get_field("co_volume_mix_ratio").get_view(); + auto d_ch4 = field_mgr.get_field("ch4_volume_mix_ratio").get_view(); + auto d_o2 = field_mgr.get_field("o2_volume_mix_ratio").get_view(); + auto d_n2 = field_mgr.get_field("n2_volume_mix_ratio").get_view(); + + // Gather molecular weights of all the active gases in the test for conversion + // to mass-mixing-ratio. + { + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + + // Set lat and lon to single value for just this test: + // Note, these values will ensure that the cosine zenith + // angle will end up matching the constant value meant for + // the test, which is 0.86 + lat[i] = 5.224000000000; + lon[i] = 167.282000000000; + + d_sfc_alb_dir_vis(i) = sfc_alb_dir_vis(i+1); + d_sfc_alb_dir_nir(i) = sfc_alb_dir_nir(i+1); + d_sfc_alb_dif_vis(i) = sfc_alb_dif_vis(i+1); + d_sfc_alb_dif_nir(i) = sfc_alb_dif_nir(i+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + d_pmid(i,k) = p_lay(i+1,k+1); + d_tmid(i,k) = t_lay(i+1,k+1); + d_pdel(i,k) = p_del(i+1,k+1); + d_qc(i,k) = qc(i+1,k+1); + d_nc(i,k) = nc(i+1,k+1); + d_qi(i,k) = qi(i+1,k+1); + d_rel(i,k) = rel(i+1,k+1); + d_rei(i,k) = rei(i+1,k+1); + d_cld(i,k) = cld(i+1,k+1); + d_pint(i,k) = p_lev(i+1,k+1); + // qv specified as a wet mixing ratio + Real qv_dry = gas_vmr(i+1,k+1,1)*PC::ep_2; + Real qv_wet = qv_dry/(1.0+qv_dry); + d_qv(i,k) = qv_wet; + // rest of active gases are specified as volume mixing ratios + d_co2(i,k) = gas_vmr(i+1,k+1,2); + d_o3(i,k) = gas_vmr(i+1,k+1,3); + d_n2o(i,k) = gas_vmr(i+1,k+1,4); + d_co(i,k) = gas_vmr(i+1,k+1,5); + d_ch4(i,k) = gas_vmr(i+1,k+1,6); + d_o2(i,k) = gas_vmr(i+1,k+1,7); + d_n2(i,k) = gas_vmr(i+1,k+1,8); }); - } - Kokkos::fence(); - - // Run driver - ad.run(300); - - // Check values; The correct values have been stored in the field manager, we need to - // copy back to YAKL::Array. - auto d_sw_flux_up = field_mgr.get_field("sw_flux_up").get_view(); - auto d_sw_flux_dn = field_mgr.get_field("sw_flux_dn").get_view(); - auto d_sw_flux_dn_dir = field_mgr.get_field("sw_flux_dn_dir").get_view(); - auto d_lw_flux_up = field_mgr.get_field("lw_flux_up").get_view(); - auto d_lw_flux_dn = field_mgr.get_field("lw_flux_dn").get_view(); - auto sw_flux_up_test = real2d("sw_flux_up_test", ncol, nlay+1); - auto sw_flux_dn_test = real2d("sw_flux_dn_test", ncol, nlay+1); - auto sw_flux_dn_dir_test = real2d("sw_flux_dn_dir_test", ncol, nlay+1); - auto lw_flux_up_test = real2d("lw_flux_up_test", ncol, nlay+1); - auto lw_flux_dn_test = real2d("lw_flux_dn_test", ncol, nlay+1); - { - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlay); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - const int i = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); - - sw_flux_up_test(i+1,k+1) = d_sw_flux_up(i,k); - sw_flux_dn_test(i+1,k+1) = d_sw_flux_dn(i,k); - sw_flux_dn_dir_test(i+1,k+1) = d_sw_flux_dn_dir(i,k); - lw_flux_up_test(i+1,k+1) = d_lw_flux_up(i,k); - lw_flux_dn_test(i+1,k+1) = d_lw_flux_dn(i,k); - }); + + d_pint(i,nlay) = p_lev(i+1,nlay+1); + + // Compute surface flux from surface temperature + auto ibot = (p_lay(1,1) > p_lay(1,nlay)) ? 1 : nlay+1; + d_surf_lw_flux_up(i) = PC::stebol * pow(t_lev(i+1,ibot), 4.0); + }); + } + Kokkos::fence(); + + // Run driver + ad.run(300); + + // Check values; The correct values have been stored in the field manager, we need to + // copy back to YAKL::Array. + auto d_sw_flux_up = field_mgr.get_field("sw_flux_up").get_view(); + auto d_sw_flux_dn = field_mgr.get_field("sw_flux_dn").get_view(); + auto d_sw_flux_dn_dir = field_mgr.get_field("sw_flux_dn_dir").get_view(); + auto d_lw_flux_up = field_mgr.get_field("lw_flux_up").get_view(); + auto d_lw_flux_dn = field_mgr.get_field("lw_flux_dn").get_view(); + auto sw_flux_up_test = real2d("sw_flux_up_test", ncol, nlay+1); + auto sw_flux_dn_test = real2d("sw_flux_dn_test", ncol, nlay+1); + auto sw_flux_dn_dir_test = real2d("sw_flux_dn_dir_test", ncol, nlay+1); + auto lw_flux_up_test = real2d("lw_flux_up_test", ncol, nlay+1); + auto lw_flux_dn_test = real2d("lw_flux_dn_test", ncol, nlay+1); + { + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { + if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + + sw_flux_up_test(i+1,k+1) = d_sw_flux_up(i,k); + sw_flux_dn_test(i+1,k+1) = d_sw_flux_dn(i,k); + sw_flux_dn_dir_test(i+1,k+1) = d_sw_flux_dn_dir(i,k); + lw_flux_up_test(i+1,k+1) = d_lw_flux_up(i,k); + lw_flux_dn_test(i+1,k+1) = d_lw_flux_dn(i,k); }); - } - Kokkos::fence(); - - // Dumb check to verify that we did indeed update temperature - REQUIRE(t_lay.createHostCopy()(1,1) != T_mid0.createHostCopy()(1,1)); - T_mid0.deallocate(); - - // Make sure fluxes from field manager that were calculated in AD call of RRTMGP match reference fluxes from input file - // We use all_close here instead of all_equals because we are only able - // to approximate the solar zenith angle used in the RRTMGP clear-sky - // test problem with our trial and error lat/lon values, so fluxes will - // be slightly off. We just verify that they are all "close" here, within - // some tolerance. Computation of level temperatures from midpoints is - // also unable to exactly reproduce the values in the test problem, so - // computed fluxes will be further off from the reference calculation. - // - // We need to create local copies with only the columns specific to our rank in case we have split columns over multiple ranks - auto sw_flux_up_loc = real2d("sw_flux_up_loc" , ncol, nlay+1); - auto sw_flux_dn_loc = real2d("sw_flux_dn_loc" , ncol, nlay+1); - auto sw_flux_dn_dir_loc = real2d("sw_flux_dn_dir_loc", ncol, nlay+1); - auto lw_flux_up_loc = real2d("lw_flux_up_loc" , ncol, nlay+1); - auto lw_flux_dn_loc = real2d("lw_flux_dn_loc" , ncol, nlay+1); - yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilay, int icol) { - auto icol_all = ncol * irank + icol; - sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); - sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); - sw_flux_dn_dir_loc(icol,ilay) = sw_flux_dn_dir_ref(icol_all,ilay); - lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); - lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); - }); - REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); - - // Deallocate YAKL arrays - p_lay.deallocate(); - t_lay.deallocate(); - p_del.deallocate(); - p_lev.deallocate(); - t_lev.deallocate(); - sfc_alb_dir_vis.deallocate(); - sfc_alb_dir_nir.deallocate(); - sfc_alb_dif_vis.deallocate(); - sfc_alb_dif_nir.deallocate(); - lwp.deallocate(); - iwp.deallocate(); - rel.deallocate(); - rei.deallocate(); - cld.deallocate(); - qc.deallocate(); - nc.deallocate(); - qi.deallocate(); - mu0.deallocate(); - gas_vmr.deallocate(); - sw_flux_up_test.deallocate(); - sw_flux_dn_test.deallocate(); - sw_flux_dn_dir_test.deallocate(); - lw_flux_up_test.deallocate(); - lw_flux_dn_test.deallocate(); - sw_flux_up_ref.deallocate(); - sw_flux_dn_ref.deallocate(); - sw_flux_dn_dir_ref.deallocate(); - lw_flux_up_ref.deallocate(); - lw_flux_dn_ref.deallocate(); - sw_flux_up_loc.deallocate(); - sw_flux_dn_loc.deallocate(); - sw_flux_dn_dir_loc.deallocate(); - lw_flux_up_loc.deallocate(); - lw_flux_dn_loc.deallocate(); - - // Finalize the driver. YAKL will be finalized inside - // RRTMGPRadiation::finalize_impl after RRTMGP has had the - // opportunity to deallocate all it's arrays. - ad.finalize(); - } + }); + } + Kokkos::fence(); + + // Dumb check to verify that we did indeed update temperature + REQUIRE(t_lay.createHostCopy()(1,1) != T_mid0.createHostCopy()(1,1)); + T_mid0.deallocate(); + + // Make sure fluxes from field manager that were calculated in AD call of RRTMGP match reference fluxes from input file + // We use all_close here instead of all_equals because we are only able + // to approximate the solar zenith angle used in the RRTMGP clear-sky + // test problem with our trial and error lat/lon values, so fluxes will + // be slightly off. We just verify that they are all "close" here, within + // some tolerance. Computation of level temperatures from midpoints is + // also unable to exactly reproduce the values in the test problem, so + // computed fluxes will be further off from the reference calculation. + // + // We need to create local copies with only the columns specific to our rank in case we have split columns over multiple ranks + auto sw_flux_up_loc = real2d("sw_flux_up_loc" , ncol, nlay+1); + auto sw_flux_dn_loc = real2d("sw_flux_dn_loc" , ncol, nlay+1); + auto sw_flux_dn_dir_loc = real2d("sw_flux_dn_dir_loc", ncol, nlay+1); + auto lw_flux_up_loc = real2d("lw_flux_up_loc" , ncol, nlay+1); + auto lw_flux_dn_loc = real2d("lw_flux_dn_loc" , ncol, nlay+1); + yakl::fortran::parallel_for(yakl::fortran::SimpleBounds<2>(nlay+1,ncol), YAKL_LAMBDA(int ilay, int icol) { + auto icol_all = ncol * irank + icol; + sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); + sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); + sw_flux_dn_dir_loc(icol,ilay) = sw_flux_dn_dir_ref(icol_all,ilay); + lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); + lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); + }); + REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + + // Deallocate YAKL arrays + p_lay.deallocate(); + t_lay.deallocate(); + p_del.deallocate(); + p_lev.deallocate(); + t_lev.deallocate(); + sfc_alb_dir_vis.deallocate(); + sfc_alb_dir_nir.deallocate(); + sfc_alb_dif_vis.deallocate(); + sfc_alb_dif_nir.deallocate(); + lwp.deallocate(); + iwp.deallocate(); + rel.deallocate(); + rei.deallocate(); + cld.deallocate(); + qc.deallocate(); + nc.deallocate(); + qi.deallocate(); + mu0.deallocate(); + gas_vmr.deallocate(); + sw_flux_up_test.deallocate(); + sw_flux_dn_test.deallocate(); + sw_flux_dn_dir_test.deallocate(); + lw_flux_up_test.deallocate(); + lw_flux_dn_test.deallocate(); + sw_flux_up_ref.deallocate(); + sw_flux_dn_ref.deallocate(); + sw_flux_dn_dir_ref.deallocate(); + lw_flux_up_ref.deallocate(); + lw_flux_dn_ref.deallocate(); + sw_flux_up_loc.deallocate(); + sw_flux_dn_loc.deallocate(); + sw_flux_dn_dir_loc.deallocate(); + lw_flux_up_loc.deallocate(); + lw_flux_dn_loc.deallocate(); + + // Finalize the driver. YAKL will be finalized inside + // RRTMGPRadiation::finalize_impl after RRTMGP has had the + // opportunity to deallocate all it's arrays. + ad.finalize(); +} +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +TEST_CASE("rrtmgp_scream_standalone_k", "") { + // Get baseline name (needs to be passed as an arg) + std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); + std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); + + // Check if files exists + REQUIRE(rrtmgpTest::file_exists(inputfile.c_str())); + REQUIRE(rrtmgpTest::file_exists(baseline.c_str())); + + // Initialize yakl + scream::init_kls(); + + // Read reference fluxes from baseline file + real2dk sw_flux_up_ref; + real2dk sw_flux_dn_ref; + real2dk sw_flux_dn_dir_ref; + real2dk lw_flux_up_ref; + real2dk lw_flux_dn_ref; + rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + + // Load ad parameter list + std::string fname = "input_unit.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + // Create a MPI communicator + ekat::Comm atm_comm (MPI_COMM_WORLD); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics (); + register_mesh_free_grids_manager(); + + // Create the driver + AtmosphereDriver ad; + + // Dummy timestamp + util::TimeStamp time ({2000,1,1},{0,0,0}); + + // Initialize the driver + ad.initialize(atm_comm, ad_params, time); + + /* + * Setup the dummy problem and overwrite default initial conditions + */ + + // Get dimension sizes from the field manager + const auto& grid = ad.get_grids_manager()->get_grid("Point Grid"); + const auto& field_mgr = *ad.get_field_mgr(grid->name()); + int ncol = grid->get_num_local_dofs(); + int nlay = grid->get_num_vertical_levels(); + + // In this test, we need to hack lat/lon. But the fields we get + // from the grid are read-only. Therefore, hack a bit, and cast + // away constness. It's bad, but it's only for this unit test + auto clat = grid->get_geometry_data("lat").get_view(); + auto clon = grid->get_geometry_data("lon").get_view(); + auto lat = const_cast(clat.data()); + auto lon = const_cast(clon.data()); + + // Get number of shortwave bands and number of gases from RRTMGP + int ngas = 8; // TODO: get this intelligently + + // Make sure we have the right dimension sizes + REQUIRE(nlay == static_cast(sw_flux_up_ref.extent(1))-1); + + // Create yakl arrays to store the input data + auto p_lay = real2dk("p_lay", ncol, nlay); + auto t_lay = real2dk("t_lay", ncol, nlay); + auto p_del = real2dk("p_del", ncol, nlay); + auto p_lev = real2dk("p_lev", ncol, nlay+1); + auto t_lev = real2dk("t_lev", ncol, nlay+1); + auto sfc_alb_dir_vis = real1dk("sfc_alb_dir_vis", ncol); + auto sfc_alb_dir_nir = real1dk("sfc_alb_dir_nir", ncol); + auto sfc_alb_dif_vis = real1dk("sfc_alb_dif_vis", ncol); + auto sfc_alb_dif_nir = real1dk("sfc_alb_dif_nir", ncol); + auto lwp = real2dk("lwp", ncol, nlay); + auto iwp = real2dk("iwp", ncol, nlay); + auto rel = real2dk("rel", ncol, nlay); + auto rei = real2dk("rei", ncol, nlay); + auto cld = real2dk("cld", ncol, nlay); + auto mu0 = real1dk("mu0", ncol); + auto gas_vmr = real3dk("gas_vmr", ncol, nlay, ngas); + + // Setup dummy problem; need to use tmp arrays with ncol_all size + auto ncol_all = grid->get_num_global_dofs(); + auto p_lay_all = real2dk("p_lay", ncol_all, nlay); + auto t_lay_all = real2dk("t_lay", ncol_all, nlay); + auto p_lev_all = real2dk("p_lev", ncol_all, nlay+1); + auto t_lev_all = real2dk("t_lev", ncol_all, nlay+1); + auto sfc_alb_dir_vis_all = real1dk("sfc_alb_dir_vis", ncol_all); + auto sfc_alb_dir_nir_all = real1dk("sfc_alb_dir_nir", ncol_all); + auto sfc_alb_dif_vis_all = real1dk("sfc_alb_dif_vis", ncol_all); + auto sfc_alb_dif_nir_all = real1dk("sfc_alb_dif_nir", ncol_all); + auto lwp_all = real2dk("lwp", ncol_all, nlay); + auto iwp_all = real2dk("iwp", ncol_all, nlay); + auto rel_all = real2dk("rel", ncol_all, nlay); + auto rei_all = real2dk("rei", ncol_all, nlay); + auto cld_all = real2dk("cld", ncol_all, nlay); + auto mu0_all = real1dk("mu0", ncol_all); + // Read in dummy Garand atmosphere; if this were an actual model simulation, + // these would be passed as inputs to the driver + // NOTE: set ncol to size of col_flx dimension in the input file. This is so + // that we can compare to the reference data provided in that file. Note that + // this will copy the first column of the input data (the first profile) ncol + // times. We will then fill some fraction of these columns with clouds for + // the test problem. + GasConcsK gas_concs; + real2dk col_dry; + read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); + // Setup dummy problem; need to use tmp arrays with ncol_all size + rrtmgpTest::dummy_atmos( + inputfile, ncol_all, p_lay_all, t_lay_all, + sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, + sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, + mu0_all, + lwp_all, iwp_all, rel_all, rei_all, cld_all + ); + // Populate our local input arrays with the proper columns, based on our rank + int irank = atm_comm.rank(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + auto icol_all = ncol * irank + icol; + sfc_alb_dir_vis(icol) = sfc_alb_dir_vis_all(icol_all); + sfc_alb_dir_nir(icol) = sfc_alb_dir_nir_all(icol_all); + sfc_alb_dif_vis(icol) = sfc_alb_dif_vis_all(icol_all); + sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); + mu0(icol) = mu0_all(icol_all); + }); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + auto icol_all = ncol * irank + icol; + p_lay(icol,ilay) = p_lay_all(icol_all,ilay); + t_lay(icol,ilay) = t_lay_all(icol_all,ilay); + lwp(icol,ilay) = lwp_all(icol_all,ilay); + iwp(icol,ilay) = iwp_all(icol_all,ilay); + rel(icol,ilay) = rel_all(icol_all,ilay); + rei(icol,ilay) = rei_all(icol_all,ilay); + cld(icol,ilay) = cld_all(icol_all,ilay); + }); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + auto icol_all = ncol * irank + icol; + p_lev(icol,ilay) = p_lev_all(icol_all,ilay); + t_lev(icol,ilay) = t_lev_all(icol_all,ilay); + }); + + // Need to calculate a dummy pseudo_density for our test problem + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); + }); + + // We do not want to pass lwp and iwp through the FM, so back out qc and qi for this test + // NOTE: test problem provides lwp/iwp in g/m2, not kg/m2! Factor of 1e3 here! + auto qc = real2dk("qc", ncol, nlay); + auto nc = real2dk("nc", ncol, nlay); + auto qi = real2dk("qi", ncol, nlay); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); + qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); + }); + + // Copy gases from gas_concs to gas_vmr array + Kokkos::parallel_for(conv::get_mdrp<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { + auto icol_all = ncol * irank + icol; + gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); + }); + gas_concs.reset(); + + // Before running, make a copy of T_mid so we can see changes + auto T_mid0 = real2dk("T_mid0", ncol, nlay); + Kokkos::deep_copy(T_mid0, t_lay); + + // Grab views from field manager and copy in values from yakl arrays. Making + // copies is necessary since the yakl::Array take in data arranged with ncol + // as the fastest index, but the field manager expects the 2nd dimension as + // the fastest index. + auto d_pmid = field_mgr.get_field("p_mid").get_view(); + auto d_tmid = field_mgr.get_field("T_mid").get_view(); + auto d_pint = field_mgr.get_field("p_int").get_view(); + auto d_pdel = field_mgr.get_field("pseudo_density").get_view(); + auto d_sfc_alb_dir_vis = field_mgr.get_field("sfc_alb_dir_vis").get_view(); + auto d_sfc_alb_dir_nir = field_mgr.get_field("sfc_alb_dir_nir").get_view(); + auto d_sfc_alb_dif_vis = field_mgr.get_field("sfc_alb_dif_vis").get_view(); + auto d_sfc_alb_dif_nir = field_mgr.get_field("sfc_alb_dif_nir").get_view(); + auto d_surf_lw_flux_up = field_mgr.get_field("surf_lw_flux_up").get_view(); + auto d_qc = field_mgr.get_field("qc").get_view(); + auto d_nc = field_mgr.get_field("nc").get_view(); + auto d_qi = field_mgr.get_field("qi").get_view(); + auto d_rel = field_mgr.get_field("eff_radius_qc").get_view(); + auto d_rei = field_mgr.get_field("eff_radius_qi").get_view(); + auto d_cld = field_mgr.get_field("cldfrac_tot").get_view(); + + auto d_qv = field_mgr.get_field("qv").get_view(); + auto d_co2 = field_mgr.get_field("co2_volume_mix_ratio").get_view(); + auto d_o3 = field_mgr.get_field("o3_volume_mix_ratio").get_view(); + auto d_n2o = field_mgr.get_field("n2o_volume_mix_ratio").get_view(); + auto d_co = field_mgr.get_field("co_volume_mix_ratio").get_view(); + auto d_ch4 = field_mgr.get_field("ch4_volume_mix_ratio").get_view(); + auto d_o2 = field_mgr.get_field("o2_volume_mix_ratio").get_view(); + auto d_n2 = field_mgr.get_field("n2_volume_mix_ratio").get_view(); + + // Gather molecular weights of all the active gases in the test for conversion + // to mass-mixing-ratio. + { + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + + // Set lat and lon to single value for just this test: + // Note, these values will ensure that the cosine zenith + // angle will end up matching the constant value meant for + // the test, which is 0.86 + lat[i] = 5.224000000000; + lon[i] = 167.282000000000; + + d_sfc_alb_dir_vis(i) = sfc_alb_dir_vis(i); + d_sfc_alb_dir_nir(i) = sfc_alb_dir_nir(i); + d_sfc_alb_dif_vis(i) = sfc_alb_dif_vis(i); + d_sfc_alb_dif_nir(i) = sfc_alb_dif_nir(i); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + d_pmid(i,k) = p_lay(i,k); + d_tmid(i,k) = t_lay(i,k); + d_pdel(i,k) = p_del(i,k); + d_qc(i,k) = qc(i,k); + d_nc(i,k) = nc(i,k); + d_qi(i,k) = qi(i,k); + d_rel(i,k) = rel(i,k); + d_rei(i,k) = rei(i,k); + d_cld(i,k) = cld(i,k); + d_pint(i,k) = p_lev(i,k); + // qv specified as a wet mixing ratio + Real qv_dry = gas_vmr(i,k,0)*PC::ep_2; + Real qv_wet = qv_dry/(1.0+qv_dry); + d_qv(i,k) = qv_wet; + // rest of active gases are specified as volume mixing ratios + d_co2(i,k) = gas_vmr(i,k,1); + d_o3(i,k) = gas_vmr(i,k,2); + d_n2o(i,k) = gas_vmr(i,k,3); + d_co(i,k) = gas_vmr(i,k,4); + d_ch4(i,k) = gas_vmr(i,k,5); + d_o2(i,k) = gas_vmr(i,k,6); + d_n2(i,k) = gas_vmr(i,k,7); + }); + + d_pint(i,nlay) = p_lev(i,nlay); + + // Compute surface flux from surface temperature + auto ibot = (p_lay(0,0) > p_lay(0,nlay-1)) ? 0 : nlay; + d_surf_lw_flux_up(i) = PC::stebol * pow(t_lev(i,ibot), 4.0); + }); + } + Kokkos::fence(); + + // Run driver + ad.run(300); + + // Check values; The correct values have been stored in the field manager, we need to + // copy back to YAKL::Array. + auto d_sw_flux_up = field_mgr.get_field("sw_flux_up").get_view(); + auto d_sw_flux_dn = field_mgr.get_field("sw_flux_dn").get_view(); + auto d_sw_flux_dn_dir = field_mgr.get_field("sw_flux_dn_dir").get_view(); + auto d_lw_flux_up = field_mgr.get_field("lw_flux_up").get_view(); + auto d_lw_flux_dn = field_mgr.get_field("lw_flux_dn").get_view(); + auto sw_flux_up_test = real2dk("sw_flux_up_test", ncol, nlay+1); + auto sw_flux_dn_test = real2dk("sw_flux_dn_test", ncol, nlay+1); + auto sw_flux_dn_dir_test = real2dk("sw_flux_dn_dir_test", ncol, nlay+1); + auto lw_flux_up_test = real2dk("lw_flux_up_test", ncol, nlay+1); + auto lw_flux_dn_test = real2dk("lw_flux_dn_test", ncol, nlay+1); + { + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, nlay); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const int i = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { + if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + + sw_flux_up_test(i,k) = d_sw_flux_up(i,k); + sw_flux_dn_test(i,k) = d_sw_flux_dn(i,k); + sw_flux_dn_dir_test(i,k) = d_sw_flux_dn_dir(i,k); + lw_flux_up_test(i,k) = d_lw_flux_up(i,k); + lw_flux_dn_test(i,k) = d_lw_flux_dn(i,k); + }); + }); + } + Kokkos::fence(); + + // Dumb check to verify that we did indeed update temperature + REQUIRE(Kokkos::create_mirror_view_and_copy(HostDevice(), t_lay)(0,0) != + Kokkos::create_mirror_view_and_copy(HostDevice(), T_mid0)(0,0)); + + // Make sure fluxes from field manager that were calculated in AD call of RRTMGP match reference fluxes from input file + // We use all_close here instead of all_equals because we are only able + // to approximate the solar zenith angle used in the RRTMGP clear-sky + // test problem with our trial and error lat/lon values, so fluxes will + // be slightly off. We just verify that they are all "close" here, within + // some tolerance. Computation of level temperatures from midpoints is + // also unable to exactly reproduce the values in the test problem, so + // computed fluxes will be further off from the reference calculation. + // + // We need to create local copies with only the columns specific to our rank in case we have split columns over multiple ranks + auto sw_flux_up_loc = real2dk("sw_flux_up_loc" , ncol, nlay+1); + auto sw_flux_dn_loc = real2dk("sw_flux_dn_loc" , ncol, nlay+1); + auto sw_flux_dn_dir_loc = real2dk("sw_flux_dn_dir_loc", ncol, nlay+1); + auto lw_flux_up_loc = real2dk("lw_flux_up_loc" , ncol, nlay+1); + auto lw_flux_dn_loc = real2dk("lw_flux_dn_loc" , ncol, nlay+1); + Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + auto icol_all = ncol * irank + icol; + sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); + sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); + sw_flux_dn_dir_loc(icol,ilay) = sw_flux_dn_dir_ref(icol_all,ilay); + lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); + lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); + }); + REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + + // Finalize the driver. YAKL will be finalized inside + // RRTMGPRadiation::finalize_impl after RRTMGP has had the + // opportunity to deallocate all it's arrays. + ad.finalize(); +} +#endif + } From cf413c4366683ee41ded56eb51e65e3d7634d1af Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 23 Apr 2024 16:24:21 -0700 Subject: [PATCH 046/476] move constants to shared utils --- components/eamxx/src/diagnostics/aodvis.cpp | 11 +++++------ components/eamxx/src/diagnostics/aodvis.hpp | 5 ++++- .../eamxx/src/diagnostics/tests/aodvis_test.cpp | 8 ++++---- components/eamxx/src/share/util/scream_utils.hpp | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 66b1a7532abb..1b321f4bb4fb 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -21,12 +21,10 @@ void AODVis::set_grids( m_ncols = grid->get_num_local_dofs(); m_nlevs = grid->get_num_vertical_levels(); - // TODO: don't hardcode this! - m_swbnd = 14; // Define layouts we need (both inputs and outputs) FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, - {m_ncols, m_swbnd, m_nlevs}}; + {m_ncols, m_swbands, m_nlevs}}; FieldLayout scalar1d_layout{{COL}, {m_ncols}}; // The fields required for this diagnostic to be computed @@ -48,8 +46,9 @@ void AODVis::compute_diagnostic_impl() { const auto aod = m_diagnostic_output.get_view(); // TODO: don't hardcode swbnd 10 // Get slice of tau at swbnd 10 - const auto tau_vis = - get_field_in("aero_tau_sw").subfield(1, 10).get_view(); + const auto tau_vis = get_field_in("aero_tau_sw") + .subfield(1, n_vis_bnd) + .get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); @@ -57,7 +56,7 @@ void AODVis::compute_diagnostic_impl() { "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); }); } diff --git a/components/eamxx/src/diagnostics/aodvis.hpp b/components/eamxx/src/diagnostics/aodvis.hpp index f03d9a9e3405..5814c0b0b517 100644 --- a/components/eamxx/src/diagnostics/aodvis.hpp +++ b/components/eamxx/src/diagnostics/aodvis.hpp @@ -2,6 +2,7 @@ #define EAMXX_AODVIS_DIAG #include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_utils.hpp" namespace scream { @@ -29,7 +30,9 @@ class AODVis : public AtmosphereDiagnostic { int m_ncols; int m_nlevs; - int m_swbnd; + + int m_swbands = eamxx_swbands(); + int n_vis_bnd = eamxx_vis_swband_idx(); }; } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 7846400287cc..78add2eda24a 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -3,6 +3,7 @@ #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_utils.hpp" namespace scream { @@ -41,8 +42,8 @@ TEST_CASE("aodvis") { constexpr int nlevs = 33; const int ngcols = 2 * comm.size(); - // TODO: Don't hardcode this! - constexpr int nbnds = 14; + int nbnds = eamxx_swbands(); + int swvis = eamxx_vis_swband_idx(); auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); @@ -93,8 +94,7 @@ TEST_CASE("aodvis") { for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { for(int ilev = 0; ilev < nlevs; ++ilev) { - // TODO: Don't hardcode the 10! - aod_t(icol) += tau_h(icol, 10, ilev); + aod_t(icol) += tau_h(icol, swvis, ilev); } } aod_hf.sync_to_dev(); diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index 6e4f08ffd22e..28a9b3af9d95 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -353,6 +353,20 @@ std::vector filename_glob(const std::vector& patterns) // Use globloc for each filename pattern std::vector globloc(const std::string& pattern); +constexpr int eamxx_swbands() { + // This function returns the total number of SW bands in RRTMGP, + return 14; +} + +constexpr int eamxx_vis_swband_idx() { + // This function returns the index of the visible SW band in RRTMGP, + // which currently (as of 2024-04-23) is supposed to be 10. + // This index is used in the AODVis diagnostic, and should ideally + // be shared across interested processes for further diagnostics. + // This index (10) corresponds to the band that has wavelength 550 nm. + return 10; +} + } // namespace scream #endif // SCREAM_UTILS_HPP From face4630b8a4e1260d7abd060877e7c01b793bb7 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 23 Apr 2024 16:36:53 -0700 Subject: [PATCH 047/476] remove the no-longer-needed comment and auto-format --- components/eamxx/src/diagnostics/aodvis.cpp | 4 +--- components/eamxx/src/diagnostics/tests/aodvis_test.cpp | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 1b321f4bb4fb..2b981c007bdb 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -43,9 +43,7 @@ void AODVis::compute_diagnostic_impl() { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; - const auto aod = m_diagnostic_output.get_view(); - // TODO: don't hardcode swbnd 10 - // Get slice of tau at swbnd 10 + const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") .subfield(1, n_vis_bnd) .get_view(); diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 78add2eda24a..6a2f2f3efd16 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -100,7 +100,7 @@ TEST_CASE("aodvis") { aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); // Workaround for non-bfb behavior of view_reduction() in release builds - if (SCREAM_BFB_TESTING) { + if(SCREAM_BFB_TESTING) { REQUIRE(views_are_equal(aod_hf, aod_tf)); } } From 19d8d30eb2c38f6d8d71b49e1187f5be7f2455e3 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 23 Apr 2024 16:46:47 -0700 Subject: [PATCH 048/476] remove no-longer-needed comment and rename member --- components/eamxx/src/diagnostics/aodvis.cpp | 3 +-- components/eamxx/src/diagnostics/aodvis.hpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 2b981c007bdb..5eea4fcd6a17 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -31,7 +31,6 @@ void AODVis::set_grids( add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); // Construct and allocate the aodvis field - // We are going to assume we have nondim units here for ease FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, grid_name); m_diagnostic_output = Field(fid); @@ -45,7 +44,7 @@ void AODVis::compute_diagnostic_impl() { const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") - .subfield(1, n_vis_bnd) + .subfield(1, m_vis_bnd) .get_view(); const auto num_levs = m_nlevs; diff --git a/components/eamxx/src/diagnostics/aodvis.hpp b/components/eamxx/src/diagnostics/aodvis.hpp index 5814c0b0b517..4d8289131d0e 100644 --- a/components/eamxx/src/diagnostics/aodvis.hpp +++ b/components/eamxx/src/diagnostics/aodvis.hpp @@ -32,7 +32,7 @@ class AODVis : public AtmosphereDiagnostic { int m_nlevs; int m_swbands = eamxx_swbands(); - int n_vis_bnd = eamxx_vis_swband_idx(); + int m_vis_bnd = eamxx_vis_swband_idx(); }; } // namespace scream From b25a0e18836e1807c78cf9753b0a715bdfbd12b0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 24 Apr 2024 10:47:39 -0500 Subject: [PATCH 049/476] move rad calls from tech to user, update nudging and aerocom docs --- .../eamxx/docs/technical/aerocom_cldtop.md | 9 +- .../eamxx/docs/technical/clean_clear_sky.md | 10 --- components/eamxx/docs/user/clean_clear_sky.md | 83 +++++++++++++++++++ components/eamxx/docs/user/coarse_nudging.md | 26 +++--- components/eamxx/mkdocs.yml | 4 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 6 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 2 +- .../rrtmgp/scream_rrtmgp_interface.hpp | 2 +- .../homme_shoc_cld_p3_rrtmgp_pg2/output.yaml | 2 +- .../tests/single-process/rrtmgp/output.yaml | 2 +- 10 files changed, 112 insertions(+), 34 deletions(-) delete mode 100644 components/eamxx/docs/technical/clean_clear_sky.md create mode 100644 components/eamxx/docs/user/clean_clear_sky.md diff --git a/components/eamxx/docs/technical/aerocom_cldtop.md b/components/eamxx/docs/technical/aerocom_cldtop.md index 18fea456cf6c..cda0c46c3b8b 100644 --- a/components/eamxx/docs/technical/aerocom_cldtop.md +++ b/components/eamxx/docs/technical/aerocom_cldtop.md @@ -1,6 +1,6 @@ -# The AeroCOM algorithm +# The AeroCom algorithm -The goal of the AeroCOM algorithm is to calculate properties at cloud top based on the AeroCOM recommendation. There are two main parts of the algorithm: probabilistically determining "cloud top" and then "calculating properties" at said cloud top. +The goal of the AeroCom algorithm is to calculate properties at cloud top based on the AeroCom recommendation. There are two main parts of the algorithm: probabilistically determining "cloud top" and then "calculating properties" at said cloud top. We treat model columns independently, so we loop over all columns in parallel. We then loop over all layers in serial (due to needing an accumulative product), starting at 2 (second highest) layer because the highest is assumed to have no clouds. Let's take a photonic approach from above the model top. Let's say that $p_{k}$ is the probability of a photon passing through the layer $k$. We follow the maximum-random overlap assumption. In all cases, we assume the cloudiness (or cloudy fraction) is completely opaque. @@ -24,4 +24,7 @@ $$c\Phi_{i,k} = \frac{cQ_{i,k}}{iQ_{i,k} + cQ_{i,k}}$$ The thermodynamic phase is used only for cloud properties (e.g., cloud-top cloud droplet number concentration) or cloud content (e.g., cloud liquid content). Further, $X_{i,k}$ is the three-dimensional cloud property of interest which is needed if we are converting a property from three-dimensional ($X$) to its two-dimensional counterpart ($x$). "Other" properties here include temperature and pressure which are not dependent on the thermodynamic phase. -A helpful references: Räisänen, P., Barker, H. W., Khairoutdinov, M. F., Li, J., & Randall, D. A. (2004). Stochastic generation of subgrid‐scale cloudy columns for large‐scale models. Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography, 130(601), 2047-2067. +Helpful pointers: + +- Räisänen, P., Barker, H. W., Khairoutdinov, M. F., Li, J., & Randall, D. A. (2004). Stochastic generation of subgrid‐scale cloudy columns for large‐scale models. Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography, 130(601), 2047-2067. +- References in this ECMWF report: Tiedtke, M., Geleyn, J.-F., Hollingsworth, A., and Louis, J.-F.: ECMWF model parameterisation of sub-grid scale processes, Technical Report, ECMWF, Shinfield Park, Reading, 10, 1979. diff --git a/components/eamxx/docs/technical/clean_clear_sky.md b/components/eamxx/docs/technical/clean_clear_sky.md deleted file mode 100644 index 82380f903156..000000000000 --- a/components/eamxx/docs/technical/clean_clear_sky.md +++ /dev/null @@ -1,10 +0,0 @@ -# Clean- and clean-clear-sky diagnostics - -In order to decompose the aerosol effective radiative forcing, additional diagnostic radiation calls are needed. -These extra diagnostics are optionally added to the main radiation call. The extra diagnostics are: - -- Clean-clear-sky fluxes: the fluxes that would be present if there were neither aerosols nor clouds, and are calculated by adding an additional radiation call at the very beginning of the logic before the optics class is endowed with aerosol and cloud properties. -- Clean-sky fluxes: the fluxes that would be present if there were no aerosols, and are calculated by adding an additional radiation call after substantiating an additional optics class, but not endowing it with aerosol properties. - -It was necessary to add an additional optics class because the original optics class is endowed with aerosols before clouds (in order to calculate the clear-sky fluxes). -The extra calls are controlled by runtime flags `extra_clnclrsky_diag` and `extra_clnsky_diag` (they take either `true` or `false` as their values). diff --git a/components/eamxx/docs/user/clean_clear_sky.md b/components/eamxx/docs/user/clean_clear_sky.md new file mode 100644 index 000000000000..56ae29bf9337 --- /dev/null +++ b/components/eamxx/docs/user/clean_clear_sky.md @@ -0,0 +1,83 @@ +# Clean- and clean-clear-sky diagnostics + +In order to decompose the aerosol effective radiative forcing, additional diagnostic radiation calls are needed. +These extra diagnostics are optionally added to the main radiation call. The extra diagnostics are: + +- Clean-clear-sky fluxes: the fluxes that would be present if there were neither aerosols nor clouds, and are calculated by adding an additional radiation call at the very beginning of the logic before the optics class is endowed with aerosol and cloud properties. +- Clean-sky fluxes: the fluxes that would be present if there were no aerosols, and are calculated by adding an additional radiation call after substantiating an additional optics class, but not endowing it with aerosol properties. + +It was necessary to add an additional optics class because the original optics class is endowed with aerosols before clouds (in order to calculate the clear-sky fluxes). + +## Example setup (current as of April 2024) + +The extra calls are controlled by runtime flags `extra_clnclrsky_diag` and `extra_clnsky_diag` (they take either `true` or `false` as their values). + +```shell + ./atmchange extra_clnclrsky_diag=true + ./atmchange extra_clnsky_diag=true +``` + +An example output file with the additional radiation diagnostics requested. + +```yaml +%YAML 1.1 +--- +filename_prefix: monthly.outputs +Averaging Type: Average +Max Snapshots Per File: 1 +Fields: + Physics PG2: + Field Names: + #2D vars + - SW_flux_up_at_model_top + - SW_flux_dn_at_model_top + - LW_flux_up_at_model_top + - SW_clnclrsky_flux_up_at_model_top + - LW_clnclrsky_flux_up_at_model_top + - SW_clrsky_flux_up_at_model_top + - LW_clrsky_flux_up_at_model_top + - SW_clnsky_flux_up_at_model_top + - LW_clnsky_flux_up_at_model_top + - SW_flux_dn_at_model_bot + - SW_clnclrsky_flux_dn_at_model_bot + - SW_clrsky_flux_dn_at_model_bot + - SW_clnsky_flux_dn_at_model_bot + - SW_flux_up_at_model_bot + - SW_clnclrsky_flux_up_at_model_bot + - SW_clrsky_flux_up_at_model_bot + - SW_clnsky_flux_up_at_model_bot + - LW_flux_dn_at_model_bot + - LW_clnclrsky_flux_dn_at_model_bot + - LW_clrsky_flux_dn_at_model_bot + - LW_clnsky_flux_dn_at_model_bot + - LW_flux_up_at_model_bot + - LongwaveCloudForcing + - ShortwaveCloudForcing + - ps + - SeaLevelPressure + - T_2m + - qv_2m + - surf_radiative_T + - VapWaterPath + - IceWaterPath + - LiqWaterPath + - RainWaterPath + - ZonalVapFlux + - MeridionalVapFlux + - surf_evap + - surf_sens_flux + - surface_upward_latent_heat_flux + - precip_liq_surf_mass_flux + - precip_ice_surf_mass_flux + - landfrac + - ocnfrac + - PotentialTemperature_at_700hPa + - PotentialTemperature_at_1000hPa + - omega_at_500hPa + - RelativeHumidity_at_700hPa +output_control: + Frequency: 1 + frequency_units: nmonths + MPI Ranks in Filename: false + +``` diff --git a/components/eamxx/docs/user/coarse_nudging.md b/components/eamxx/docs/user/coarse_nudging.md index 3d7309c42aaa..6dbf4d7eeb46 100644 --- a/components/eamxx/docs/user/coarse_nudging.md +++ b/components/eamxx/docs/user/coarse_nudging.md @@ -5,21 +5,23 @@ Instead, in EAMxx, it is possible to nudge from coarse data. This is done by remapping the coarse data provided by the user to the runtime physics grid of EAMxx. In order to enable nudging from coarse data, the user must provide nudging data at the coarse resolution desired and an appropriate ncremap-compatible mapping file. -## Example setup +## Example setup (current as of April 2024) A user can produce coarse nudging data from running EAMxx or EAM at a ne30pg2 or any other applicable resolution. -Additionally, several users in the E3SM projects have produced nudging data at the ne30pg2 resolution from the MERRA2 and ERA5 datasets. -A limitation for now is that the nudging data must be provided explicitly, either as one file or as a list of files. -This can be problematic for long list of files, but we are working on a solution to this problem. +Additionally, several users in the E3SM projects have produced nudging data at the ne30pg2 resolution from the MERRA2 and ERA5 datasets. Then, to enable nudging as a process, one must declare it in the `atm_procs_list` runtime parameter. -Let's say that the nudging data is provided as one file in the following path: `/path/to/nudging_data_ne4pg2_L72.nc`. -Then, a mapping file is provided as `/another/path/to/mapping_file_ne4pg2_to_ne120pg2.nc`. -Then if the physics grid is ne120pg2, the user must enable the nudging process, specify the nudging files, and provide the specifies the nudging data and a remap file. -In other words, the following options are needed: +```shell + ./atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp,nudging" +``` + +The following options are needed to specify the nudging. ```shell -./atmchange atm_procs_list=(sc_import,nudging,homme,physics,sc_export) -./atmchange nudging_fields=U,V -./atmchange nudging_filenames_patterns=/path/to/nudging_data_ne4pg2_L72.nc -./atmchange nudging_refine_remap_mapfile=/another/path/to/mapping_file_ne4pg2_to_ne120pg2.nc + ./atmchange nudging::nudging_filenames_patterns="${PSCRATCH}/mo_ne45pg2_nudg_trim_merra/*.nc" + ./atmchange nudging::source_pressure_type=TIME_DEPENDENT_3D_PROFILE + ./atmchange nudging::nudging_fields=U,V + ./atmchange nudging::nudging_timescale=21600 # 6-hr + ./atmchange nudging::nudging_refine_remap_mapfile="${PSCRATCH}/map_ne45pg2_to_ne256pg2.trbilin.20240410.nc" ``` + +To gain a deeper understanding of these parameters and options, please refer to the overarching nudging documentation. diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index dde0970a4ed2..6cc8fdeae0a8 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -9,6 +9,7 @@ nav: - 'Model input': 'user/model_input.md' - 'Runtime parameters': 'common/eamxx_params.md' - 'Coarse nudging': 'user/coarse_nudging.md' + - 'Extra radiation calls': 'user/clean_clear_sky.md' - 'Developer Guide': - 'Overview': 'developer/index.md' - 'Installation': 'common/installation.md' @@ -26,8 +27,7 @@ nav: - 'Full model': 'developer/cime_testing.md' - 'CI and Nightly Testing': 'developer/ci_nightly.md' - 'Technical Guide': - - 'AeroCOM cloud top': 'technical/aerocom_cldtop.md' - - 'Extra radiation calls': 'technical/clean_clear_sky.md' + - 'AeroCom cloud top': 'technical/aerocom_cldtop.md' edit_uri: "" diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 16811885a3b9..a228267f12f4 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -160,7 +160,7 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ add_field("dtau105" , scalar3d_layout_mid, nondim, grid_name); add_field("sunlit" , scalar2d_layout , nondim, grid_name); add_field("cldfrac_rad" , scalar3d_layout_mid, nondim, grid_name); - // Cloud-top diagnostics following AeroCOM recommendation + // Cloud-top diagnostics following AeroCom recommendation add_field("T_mid_at_cldtop", scalar2d_layout, K, grid_name); add_field("p_mid_at_cldtop", scalar2d_layout, Pa, grid_name); add_field("cldfrac_ice_at_cldtop", scalar2d_layout, nondim, grid_name); @@ -521,7 +521,7 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::deep_copy(d_dtau067,0.0); Kokkos::deep_copy(d_dtau105,0.0); - // Outputs for AeroCOM cloud-top diagnostics + // Outputs for AeroCom cloud-top diagnostics auto d_T_mid_at_cldtop = get_field_out("T_mid_at_cldtop").get_view(); auto d_p_mid_at_cldtop = get_field_out("p_mid_at_cldtop").get_view(); auto d_cldfrac_ice_at_cldtop = @@ -998,7 +998,7 @@ void RRTMGPRadiation::run_impl (const double dt) { // Get IR 10.5 micron band for COSP auto idx_105 = rrtmgp::get_wavelength_index_lw(10.5e-6); - // Compute cloud-top diagnostics following AeroCOM recommendation + // Compute cloud-top diagnostics following AeroCom recommendation real1d T_mid_at_cldtop ("T_mid_at_cldtop", d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1d p_mid_at_cldtop ("p_mid_at_cldtop", d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1d cldfrac_ice_at_cldtop ("cldfrac_ice_at_cldtop", d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index ec3460293951..04aef4fd003e 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1047,7 +1047,7 @@ namespace scream { real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop) { /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCOM recommendation. See reference for routine + * based on the AeroCom recommendation. See reference for routine * get_subcolumn_mask above, where equation 14 is used for the * maximum-random overlap assumption for subcolumn generation. We use * equation 13, the column counterpart. diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 65e165422865..71904aef8863 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -122,7 +122,7 @@ namespace scream { int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); /* - * Return select cloud-top diagnostics following AeroCOM recommendation + * Return select cloud-top diagnostics following AeroCom recommendation */ void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml index d07cdf5a4d0b..c492f864ff98 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml @@ -64,7 +64,7 @@ Fields: - rad_heating_pdel - sfc_flux_lw_dn - sfc_flux_sw_net - # AeroCOM in RRTMGP + # AeroCom in RRTMGP - cdnc_at_cldtop - cldfrac_tot_at_cldtop - cldfrac_liq_at_cldtop diff --git a/components/eamxx/tests/single-process/rrtmgp/output.yaml b/components/eamxx/tests/single-process/rrtmgp/output.yaml index b067d1412bc6..94a7dc94ebad 100644 --- a/components/eamxx/tests/single-process/rrtmgp/output.yaml +++ b/components/eamxx/tests/single-process/rrtmgp/output.yaml @@ -17,7 +17,7 @@ Field Names: - sfc_flux_lw_dn - sfc_flux_sw_net - rad_heating_pdel - # AeroCOM in RRTMGP + # AeroCom in RRTMGP - cdnc_at_cldtop - cldfrac_tot_at_cldtop - cldfrac_liq_at_cldtop From da2d2f5aa30767fb7a7b7a5efa6b767cbf698683 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 Nov 2023 14:40:44 -0700 Subject: [PATCH 050/476] EAMxx: change signature of a couple of AbstractGrid methods The get_Nd_vector_layout methods accepted a FieldTag for the vector component. However, all places that used this method passed CMP as tag. Hence, remove tag from input args, and hard-code CMP as vector component. --- .../diagnostics/tests/wind_speed_tests.cpp | 2 +- .../eamxx/src/diagnostics/vapor_flux.cpp | 2 +- .../eamxx/src/diagnostics/wind_speed.cpp | 2 +- .../nudging/tests/nudging_tests_helpers.hpp | 2 +- .../eamxx/src/share/field/field_manager.cpp | 8 ++-- .../eamxx/src/share/grid/abstract_grid.hpp | 8 ++-- .../eamxx/src/share/grid/point_grid.cpp | 8 ++-- .../eamxx/src/share/grid/point_grid.hpp | 8 ++-- .../share/grid/remap/do_nothing_remapper.hpp | 8 ++-- .../grid/remap/horiz_interp_remapper_base.cpp | 38 +++++++++---------- .../share/grid/remap/vertical_remapper.cpp | 9 ++--- components/eamxx/src/share/grid/se_grid.cpp | 8 ++-- components/eamxx/src/share/grid/se_grid.hpp | 8 ++-- .../share/tests/coarsening_remapper_tests.cpp | 9 ++--- .../tests/refining_remapper_p2p_tests.cpp | 5 +-- .../tests/refining_remapper_rma_tests.cpp | 5 +-- .../share/tests/vertical_remapper_tests.cpp | 6 +-- 17 files changed, 63 insertions(+), 73 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp index 7e3affedacb2..5e6be61ba4f0 100644 --- a/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp @@ -45,7 +45,7 @@ TEST_CASE("wind_speed") auto grid = gm->get_grid("Physics"); // Input (randomized) velocity - auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + auto vector3d = grid->get_3d_vector_layout(true,2); FieldIdentifier uv_fid ("horiz_winds",vector3d,m/s,grid->name()); Field uv(uv_fid); uv.allocate_view(); diff --git a/components/eamxx/src/diagnostics/vapor_flux.cpp b/components/eamxx/src/diagnostics/vapor_flux.cpp index 576dd41a1fb5..66abf10d5bb1 100644 --- a/components/eamxx/src/diagnostics/vapor_flux.cpp +++ b/components/eamxx/src/diagnostics/vapor_flux.cpp @@ -49,7 +49,7 @@ void VaporFluxDiagnostic::set_grids(const std::shared_ptr gr auto scalar2d = grid->get_2d_scalar_layout(); auto scalar3d = grid->get_3d_scalar_layout(true); - auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + auto vector3d = grid->get_3d_vector_layout(true,2); // The fields required for this diagnostic to be computed add_field("pseudo_density", scalar3d, Pa, grid_name); diff --git a/components/eamxx/src/diagnostics/wind_speed.cpp b/components/eamxx/src/diagnostics/wind_speed.cpp index 406346af9152..7c7fc462f500 100644 --- a/components/eamxx/src/diagnostics/wind_speed.cpp +++ b/components/eamxx/src/diagnostics/wind_speed.cpp @@ -25,7 +25,7 @@ set_grids(const std::shared_ptr grids_manager) m_nlevs = grid->get_num_vertical_levels(); auto scalar3d = grid->get_3d_scalar_layout(true); - auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + auto vector3d = grid->get_3d_vector_layout(true,2); // The fields required for this diagnostic to be computed add_field("horiz_winds", vector3d, Pa, grid_name); diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp index 52e2b8ca0e30..ea3b1b2f238b 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp @@ -49,7 +49,7 @@ create_fm (const std::shared_ptr& grid) const std::string& gn = grid->name(); auto scalar3d = grid->get_3d_scalar_layout(true); - auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + auto vector3d = grid->get_3d_vector_layout(true,2); FieldIdentifier fid1("p_mid",scalar3d,Pa,gn); FieldIdentifier fid2("horiz_winds",vector3d,m/s,gn); diff --git a/components/eamxx/src/share/field/field_manager.cpp b/components/eamxx/src/share/field/field_manager.cpp index 5867f30ba01e..53f0c2b10a97 100644 --- a/components/eamxx/src/share/field/field_manager.cpp +++ b/components/eamxx/src/share/field/field_manager.cpp @@ -540,9 +540,9 @@ void FieldManager::registration_ends () FieldLayout c_layout = FieldLayout::invalid(); if (lt==LayoutType::Scalar2D) { - c_layout = m_grid->get_2d_vector_layout(CMP,cluster_ordered_fields.size()); + c_layout = m_grid->get_2d_vector_layout(cluster_ordered_fields.size()); } else { - c_layout = m_grid->get_3d_vector_layout(f_layout.tags().back()==LEV,CMP,cluster_ordered_fields.size()); + c_layout = m_grid->get_3d_vector_layout(f_layout.tags().back()==LEV,cluster_ordered_fields.size()); } // The units for the bundled field are nondimensional, cause checking whether @@ -654,10 +654,10 @@ void FieldManager::registration_ends () auto lt = get_layout_type(f1_layout.tags()); FieldLayout g_layout = FieldLayout::invalid(); if (lt==LayoutType::Scalar2D) { - g_layout = m_grid->get_2d_vector_layout(CMP,size); + g_layout = m_grid->get_2d_vector_layout(size); } else { bool mid = f1_layout.tags().back()==LEV; - g_layout = m_grid->get_3d_vector_layout(mid,CMP,size); + g_layout = m_grid->get_3d_vector_layout(mid,size); } FieldIdentifier g_fid(gname,g_layout,nondim,m_grid->name()); diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 10b1b2676540..9a5421a25c66 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -73,13 +73,11 @@ class AbstractGrid : public ekat::enable_shared_from_this // for a vector 3d field on a Point grid it will be (ncols,vector_dim,nlevs) FieldLayout get_vertical_layout (const bool midpoints) const; virtual FieldLayout get_2d_scalar_layout () const = 0; - virtual FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const = 0; - virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, - const std::vector& cmp_dims) const = 0; + virtual FieldLayout get_2d_vector_layout (const int vector_dim) const = 0; + virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const = 0; virtual FieldLayout get_3d_scalar_layout (const bool midpoints) const = 0; - virtual FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const = 0; + virtual FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const = 0; virtual FieldLayout get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_tags, const std::vector& cmp_dims) const = 0; int get_num_vertical_levels () const { return m_num_vert_levs; } diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index c1db5c795e34..ceb7680992c6 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -48,11 +48,11 @@ PointGrid::get_2d_scalar_layout () const } FieldLayout -PointGrid::get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const +PointGrid::get_2d_vector_layout (const int vector_dim) const { using namespace ShortFieldTagsNames; - return FieldLayout({COL,vector_tag},{get_num_local_dofs(),vector_dim}); + return FieldLayout({COL,CMP},{get_num_local_dofs(),vector_dim}); } FieldLayout @@ -81,14 +81,14 @@ PointGrid::get_3d_scalar_layout (const bool midpoints) const } FieldLayout -PointGrid::get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const +PointGrid::get_3d_vector_layout (const bool midpoints, const int vector_dim) const { using namespace ShortFieldTagsNames; int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - return FieldLayout({COL,vector_tag,VL},{get_num_local_dofs(),vector_dim,nvl}); + return FieldLayout({COL,CMP,VL},{get_num_local_dofs(),vector_dim,nvl}); } FieldLayout diff --git a/components/eamxx/src/share/grid/point_grid.hpp b/components/eamxx/src/share/grid/point_grid.hpp index a66bf5d7d764..89e72b5be9f0 100644 --- a/components/eamxx/src/share/grid/point_grid.hpp +++ b/components/eamxx/src/share/grid/point_grid.hpp @@ -44,13 +44,11 @@ class PointGrid : public AbstractGrid // Native layout of a dof. This is the natural way to index a dof in the grid. // E.g., for a 2d structured grid, this could be a set of 2 indices. FieldLayout get_2d_scalar_layout () const override; - FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const override; - FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, - const std::vector& cmp_dims) const override; + FieldLayout get_2d_vector_layout (const int vector_dim) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; - FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const override; FieldLayout get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_tags, const std::vector& cmp_dims) const override; FieldTag get_partitioned_dim_tag () const override { diff --git a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp index 3bb593923795..210556241b78 100644 --- a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp @@ -48,13 +48,13 @@ class DoNothingRemapper : public AbstractRemapper src = this->m_src_grid->get_2d_scalar_layout(); break; case LayoutType::Vector2D: - src = this->m_src_grid->get_2d_vector_layout(CMP,tgt.dim(CMP)); + src = this->m_src_grid->get_2d_vector_layout(tgt.dim(CMP)); break; case LayoutType::Scalar3D: src = this->m_src_grid->get_3d_scalar_layout(tgt.has_tag(LEV)); break; case LayoutType::Vector3D: - src = this->m_src_grid->get_3d_vector_layout(tgt.has_tag(LEV),CMP,tgt.dim(CMP)); + src = this->m_src_grid->get_3d_vector_layout(tgt.has_tag(LEV),tgt.dim(CMP)); break; default: EKAT_ERROR_MSG ("Error! Unsupported field layout.\n"); @@ -76,13 +76,13 @@ class DoNothingRemapper : public AbstractRemapper tgt = this->m_tgt_grid->get_2d_scalar_layout(); break; case LayoutType::Vector2D: - tgt = this->m_tgt_grid->get_2d_vector_layout(CMP,tgt.dim(CMP)); + tgt = this->m_tgt_grid->get_2d_vector_layout(tgt.dim(CMP)); break; case LayoutType::Scalar3D: tgt = this->m_tgt_grid->get_3d_scalar_layout(tgt.has_tag(LEV)); break; case LayoutType::Vector3D: - tgt = this->m_tgt_grid->get_3d_vector_layout(tgt.has_tag(LEV),CMP,tgt.dim(CMP)); + tgt = this->m_tgt_grid->get_3d_vector_layout(tgt.has_tag(LEV),tgt.dim(CMP)); break; default: EKAT_ERROR_MSG ("Error! Unsupported field layout.\n"); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 5fa1e46ba646..0493f6d50825 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -124,37 +124,37 @@ create_layout (const FieldLayout& fl_in, const bool midpoints = fl_in.has_tag(LEV); const bool is3d = fl_in.has_tag(LEV) or fl_in.has_tag(ILEV); switch (type) { - case LayoutType::Scalar2D: [[ fallthrough ]]; - case LayoutType::Scalar3D: - fl_out = is3d - ? grid->get_3d_scalar_layout(midpoints) - : grid->get_2d_scalar_layout(); + case LayoutType::Scalar2D: + fl_out = m_tgt_grid->get_2d_scalar_layout(); break; - case LayoutType::Vector2D: [[ fallthrough ]]; - case LayoutType::Vector3D: + case LayoutType::Vector2D: + fl_out = m_tgt_grid->get_2d_vector_layout(fl_in.dim(CMP)); + break; + case LayoutType::Tensor2D: { - auto vtag = fl_in.get_vector_tag(); - auto vdim = fl_in.dim(vtag); - fl_out = is3d - ? grid->get_3d_vector_layout(midpoints,vtag,vdim) - : grid->get_2d_vector_layout(vtag,vdim); + std::vector tdims; + for (auto idx : fl_in.get_tensor_dims()) { + tdims.push_back(fl_in.dim(idx)); + } + + fl_out = m_tgt_grid->get_2d_tensor_layout(fl_in.dim(CMP)); break; } - - case LayoutType::Tensor2D: [[ fallthrough ]]; + case LayoutType::Scalar3D: + fl_out = grid->get_3d_scalar_layout(midpoints); + break; + case LayoutType::Vector3D: + fl_out = grid->get_3d_vector_layout(midpoints,fl_in.dim(CMP)); + break; case LayoutType::Tensor3D: { - auto ttags = fl_in.get_tensor_tags(); std::vector tdims; for (auto idx : fl_in.get_tensor_dims()) { tdims.push_back(fl_in.dim(idx)); } - fl_out = is3d - ? grid->get_3d_tensor_layout(midpoints,ttags,tdims) - : grid->get_2d_tensor_layout(ttags,tdims); + fl_out = grid->get_3d_tensor_layout(midpoints,tdims); break; } - default: EKAT_ERROR_MSG ("Layout not supported by HorizInterpRemapperBase:\n" " - layout: " + to_string(fl_in) + "\n"); diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 9abefed9eed7..71a961efe3f7 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -121,16 +121,15 @@ create_layout (const FieldLayout& fl_in, case LayoutType::Scalar1D: fl_out = grid_out->get_vertical_layout(true); break; + case LayoutType::Vector2D: + tgt = m_tgt_grid->get_2d_vector_layout(fl_in.dim(CMP)); + break; case LayoutType::Scalar3D: fl_out = grid_out->get_3d_scalar_layout(true); break; case LayoutType::Vector3D: - { - const auto vec_tag = fl_in.get_vector_tag(); - const auto vec_dim = fl_in.dim(vec_tag); - fl_out = grid_out->get_3d_vector_layout(true,vec_tag,vec_dim); + fl_out = grid_out->get_3d_vector_layout(true,vec_dim,fl_in.dim(CMP)); break; - } default: // NOTE: this also include Tensor3D. We don't really have any atm proc // that needs to handle a tensor3d quantity, so no need to add it diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index 1b941125e1cb..6181c8e7c497 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -40,11 +40,11 @@ SEGrid::get_2d_scalar_layout () const } FieldLayout -SEGrid::get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const +SEGrid::get_2d_vector_layout (const int vector_dim) const { using namespace ShortFieldTagsNames; - return FieldLayout({EL,vector_tag,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); + return FieldLayout({EL,CMP,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); } FieldLayout @@ -77,14 +77,14 @@ SEGrid::get_3d_scalar_layout (const bool midpoints) const } FieldLayout -SEGrid::get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const +SEGrid::get_3d_vector_layout (const bool midpoints, const int vector_dim) const { using namespace ShortFieldTagsNames; int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - return FieldLayout({EL,vector_tag,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); + return FieldLayout({EL,CMP,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); } FieldLayout diff --git a/components/eamxx/src/share/grid/se_grid.hpp b/components/eamxx/src/share/grid/se_grid.hpp index 90af8de066da..b9c1127b13b4 100644 --- a/components/eamxx/src/share/grid/se_grid.hpp +++ b/components/eamxx/src/share/grid/se_grid.hpp @@ -21,13 +21,11 @@ class SEGrid : public AbstractGrid // Native layout of a dof. This is the natural way to index a dof in the grid. FieldLayout get_2d_scalar_layout () const override; - FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const override; - FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, - const std::vector& cmp_dims) const override; + FieldLayout get_2d_vector_layout (const int vector_dim) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; - FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const override; FieldLayout get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_tags, const std::vector& cmp_dims) const override; FieldTag get_partitioned_dim_tag () const override { diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index 13832c490441..a82ca03b1d38 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -110,26 +110,25 @@ constexpr int tens_dim2 = 4; Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, const bool midpoints) { const auto u = ekat::units::Units::nondimensional(); - const auto CMP = ShortFieldTagsNames::CMP; const auto& gn = grid.name(); Field f; switch (lt) { case LayoutType::Scalar2D: f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; case LayoutType::Vector2D: - f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,vec_dim),u,gn)); break; + f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(vec_dim),u,gn)); break; case LayoutType::Tensor2D: - f = Field(FieldIdentifier(name,grid.get_2d_tensor_layout({CMP,CMP},{tens_dim1,tens_dim2}),u,gn)); break; + f = Field(FieldIdentifier(name,grid.get_2d_tensor_layout({tens_dim1,tens_dim2}),u,gn)); break; case LayoutType::Scalar3D: f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(midpoints),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); break; case LayoutType::Vector3D: - f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(midpoints,CMP,vec_dim),u,gn)); + f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(midpoints,vec_dim),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); break; case LayoutType::Tensor3D: - f = Field(FieldIdentifier(name,grid.get_3d_tensor_layout(midpoints,{CMP,CMP},{tens_dim1,tens_dim2}),u,gn)); + f = Field(FieldIdentifier(name,grid.get_3d_tensor_layout(midpoints,{tens_dim1,tens_dim2}),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); break; default: diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp index 00ee58ef503c..2b600b52c416 100644 --- a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -21,7 +21,6 @@ class RefiningRemapperP2PTester : public RefiningRemapperP2P { Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid) { const auto u = ekat::units::Units::nondimensional(); - const auto CMP = ShortFieldTagsNames::CMP; const auto& gn = grid.name(); const auto ndims = 2; Field f; @@ -29,12 +28,12 @@ Field create_field (const std::string& name, const LayoutType lt, const Abstract case LayoutType::Scalar2D: f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; case LayoutType::Vector2D: - f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,ndims),u,gn)); break; + f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(ndims),u,gn)); break; case LayoutType::Scalar3D: f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(true),u,gn)); break; f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); case LayoutType::Vector3D: - f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(false,CMP,ndims),u,gn)); break; + f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(false,ndims),u,gn)); break; f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); default: EKAT_ERROR_MSG ("Invalid layout type for this unit test.\n"); diff --git a/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp index d7cef33b07ce..45f8d2751911 100644 --- a/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp @@ -76,7 +76,6 @@ class RefiningRemapperRMATester : public RefiningRemapperRMA { Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid) { const auto u = ekat::units::Units::nondimensional(); - const auto CMP = ShortFieldTagsNames::CMP; const auto& gn = grid.name(); const auto ndims = 2; Field f; @@ -84,12 +83,12 @@ Field create_field (const std::string& name, const LayoutType lt, const Abstract case LayoutType::Scalar2D: f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; case LayoutType::Vector2D: - f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,ndims),u,gn)); break; + f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(ndims),u,gn)); break; case LayoutType::Scalar3D: f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(true),u,gn)); break; f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); case LayoutType::Vector3D: - f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(false,CMP,ndims),u,gn)); break; + f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(false,ndims),u,gn)); break; f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); default: EKAT_ERROR_MSG ("Invalid layout type for this unit test.\n"); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 6ec4a40c1236..8c720a5464b9 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -67,13 +67,13 @@ build_src_grid(const ekat::Comm& comm, const int nldofs_src, const int nlevs_src Field create_field(const std::string& name, const std::shared_ptr& grid, const bool twod, const bool vec, const bool mid = false, const int ps = 1) { + using namespace ShortFieldTagsNames; constexpr int vec_dim = 3; - constexpr auto CMP = FieldTag::Component; constexpr auto units = ekat::units::Units::nondimensional(); auto fl = twod - ? (vec ? grid->get_2d_vector_layout (CMP,vec_dim) + ? (vec ? grid->get_2d_vector_layout (vec_dim) : grid->get_2d_scalar_layout ()) - : (vec ? grid->get_3d_vector_layout (mid,CMP,vec_dim) + : (vec ? grid->get_3d_vector_layout (mid,vec_dim) : grid->get_3d_scalar_layout (mid)); FieldIdentifier fid(name,fl,units,grid->name()); Field f(fid); From 4948164ea522375183ecffcf19a9a90a53af306b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 Nov 2023 15:25:53 -0700 Subject: [PATCH 051/476] EAMxx: turn get_layout_type into a method of FieldLayout --- .../src/control/surface_coupling_utils.cpp | 4 +-- .../homme/eamxx_homme_process_interface.cpp | 2 +- .../homme/physics_dynamics_remapper.cpp | 8 ++--- .../homme/physics_dynamics_remapper.hpp | 2 +- components/eamxx/src/share/field/field.cpp | 2 +- .../eamxx/src/share/field/field_layout.cpp | 31 +++++++------------ .../eamxx/src/share/field/field_layout.hpp | 3 +- .../eamxx/src/share/field/field_manager.cpp | 8 ++--- .../eamxx/src/share/grid/abstract_grid.cpp | 2 +- .../share/grid/remap/do_nothing_remapper.hpp | 26 ++++++---------- .../grid/remap/horiz_interp_remapper_base.cpp | 3 +- .../share/grid/remap/vertical_remapper.cpp | 7 ++--- .../share/grid/remap/vertical_remapper.hpp | 2 +- .../share/tests/coarsening_remapper_tests.cpp | 2 +- .../eamxx/src/share/tests/field_tests.cpp | 12 +++---- .../share/tests/vertical_remapper_tests.cpp | 4 +-- 16 files changed, 52 insertions(+), 66 deletions(-) diff --git a/components/eamxx/src/control/surface_coupling_utils.cpp b/components/eamxx/src/control/surface_coupling_utils.cpp index c0674e5520e1..519d25922215 100644 --- a/components/eamxx/src/control/surface_coupling_utils.cpp +++ b/components/eamxx/src/control/surface_coupling_utils.cpp @@ -11,7 +11,7 @@ void get_col_info_for_surface_values(const std::shared_ptr& f const auto& layout = fh->get_identifier().get_layout(); const auto& dims = layout.dims(); - auto lt = get_layout_type(layout.tags()); + auto lt = layout.type(); const bool scalar = lt==LayoutType::Scalar2D || lt==LayoutType::Scalar3D; const bool vector = lt==LayoutType::Vector2D || lt==LayoutType::Vector3D; const bool layout3d = lt==LayoutType::Scalar3D || lt==LayoutType::Vector3D; @@ -50,7 +50,7 @@ void get_col_info_for_surface_values(const std::shared_ptr& f EKAT_REQUIRE_MSG(parent->get_parent().lock() == nullptr, "Error! Currently support isn't added for fields with grandparents.\n"); - const auto parent_lt = get_layout_type(parent->get_identifier().get_layout().tags()); + const auto parent_lt = parent->get_identifier().get_layout().type(); EKAT_REQUIRE_MSG(parent_lt==LayoutType::Vector3D, "Error! SurfaceCoupling expects all subfields to have parents " diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index f33c453c117c..6bf400f2bce7 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -769,7 +769,7 @@ create_helper_field (const std::string& name, using namespace ekat::units; FieldIdentifier id(name,FieldLayout{tags,dims},Units::nondimensional(),grid); - const auto lt = get_layout_type(id.get_layout().tags()); + const auto lt = id.get_layout().type(); // Only request packed field for 3d quantities int pack_size = 1; diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp index 5ac1bb2eca6a..9ff24886cf46 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp @@ -112,7 +112,7 @@ create_tgt_layout (const FieldLayout& src_layout) const { dims[0] = this->m_tgt_grid->get_num_local_dofs() / (HOMMEXX_NP*HOMMEXX_NP); // For position of GP and NP, it's easier to switch between 2d and 3d - auto lt = get_layout_type(src_layout.tags()); + auto lt = src_layout.type(); switch (lt) { case LayoutType::Scalar2D: case LayoutType::Vector2D: @@ -260,7 +260,7 @@ initialize_device_variables() const auto& pl = ph.get_identifier().get_layout(); - auto lt = get_layout_type(pl.tags()); + auto lt = pl.type(); h_layout(i) = etoi(lt); const bool is_field_3d = lt==LayoutType::Scalar3D || lt==LayoutType::Vector3D; @@ -478,7 +478,7 @@ setup_boundary_exchange () { int num_3d_int = 0; for (int i=0; im_num_fields; ++i) { const auto& layout = m_dyn_fields[i].get_header().get_identifier().get_layout(); - const auto lt = get_layout_type(layout.tags()); + const auto lt = layout.type(); switch (lt) { case LayoutType::Scalar2D: ++num_2d; @@ -528,7 +528,7 @@ setup_boundary_exchange () { for (int i=0; im_num_fields; ++i) { const auto& layout = m_dyn_fields[i].get_header().get_identifier().get_layout(); const auto& dims = layout.dims(); - const auto lt = get_layout_type(layout.tags()); + const auto lt = layout.type(); switch (lt) { case LayoutType::Scalar2D: m_be->register_field(getHommeView(m_dyn_fields[i])); diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp index 2ac247696b4b..0a346ad2c1b8 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp @@ -46,7 +46,7 @@ class PhysicsDynamicsRemapper : public AbstractRemapper bool compatible_layouts (const layout_type& src, const layout_type& tgt) const override { - return get_layout_type(src.tags())==get_layout_type(tgt.tags()); + return src.type()==tgt.type(); } protected: diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index f754adc80f0c..7f872e037224 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -121,7 +121,7 @@ get_component (const int i, const bool dynamic) { const auto& fname = get_header().get_identifier().name(); EKAT_REQUIRE_MSG (layout.is_vector_layout(), "Error! 'get_component' available only for vector fields.\n" - " Layout of '" + fname + "': " + e2str(get_layout_type(layout.tags())) + "\n"); + " Layout of '" + fname + "': " + e2str(layout.type()) + "\n"); const int idim = layout.get_vector_component_idx(); EKAT_REQUIRE_MSG (i>=0 && i& tags, } bool FieldLayout::is_vector_layout () const { - const auto lt = get_layout_type (m_tags); - return lt==LayoutType::Vector2D || lt==LayoutType::Vector3D; + using namespace ShortFieldTagsNames; + return ekat::count(m_tags,CMP)==1; } bool FieldLayout::is_tensor_layout () const { - const auto lt = get_layout_type (m_tags); + const auto lt = type(); return lt==LayoutType::Tensor2D || lt==LayoutType::Tensor3D; } @@ -33,17 +33,10 @@ bool FieldLayout::is_tensor_layout () const { int FieldLayout::get_vector_component_idx () const { EKAT_REQUIRE_MSG (is_vector_layout(), "Error! 'get_vector_dim' available only for vector layouts.\n" - " Current layout: " + e2str(get_layout_type(m_tags)) + "\n"); + " Current layout: " + e2str(type()) + "\n"); using namespace ShortFieldTagsNames; - std::vector vec_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; - auto it = std::find_first_of (m_tags.cbegin(),m_tags.cend(),vec_tags.cbegin(),vec_tags.cend()); - - EKAT_REQUIRE_MSG (it!=m_tags.cend(), - "Error! Could not find a vector tag in the layout.\n" - " - layout: " + to_string(*this) + "\n"); - - return std::distance(m_tags.cbegin(),it); + return std::distance(m_tags.begin(),ekat::find(m_tags,CMP)); } // get the extent of the CMP (Components) tag in the FieldLayout @@ -63,15 +56,14 @@ std::vector FieldLayout::get_tensor_dims () const { EKAT_REQUIRE_MSG (is_tensor_layout(), "Error! 'get_tensor_dims' available only for tensor layouts.\n" " Current layout: " + to_string(*this) + "\n" - " Layout type : " + e2str(get_layout_type(m_tags)) + "\n"); + " Layout type : " + e2str(type()) + "\n"); using namespace ShortFieldTagsNames; - std::vector cmp_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; std::vector idx; auto it = m_tags.begin(); do { - it = std::find_first_of (it,m_tags.cend(),cmp_tags.cbegin(),cmp_tags.cend()); + it = std::find(it,m_tags.cend(),CMP); if (it!=m_tags.end()) { idx.push_back(std::distance(m_tags.begin(),it)); ++it; @@ -136,12 +128,13 @@ void FieldLayout::set_dimension (const int idim, const int dimension) { Kokkos::deep_copy(m_extents,extents_h); } -LayoutType get_layout_type (const std::vector& field_tags) { +LayoutType FieldLayout::type () const { + using namespace ShortFieldTagsNames; + using ekat::erase; using ekat::count; - using namespace ShortFieldTagsNames; - auto tags = field_tags; + auto tags = this->tags(); const int n_element = count(tags,EL); const int n_column = count(tags,COL); @@ -177,7 +170,7 @@ LayoutType get_layout_type (const std::vector& field_tags) { return LayoutType::Vector1D; } else { // Not a supported layout. - return result; + return LayoutType::Invalid; } // Get the size of what's left diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index d72f8649e9ef..55569d1f53bf 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -74,6 +74,8 @@ class FieldLayout { // ----- Getters ----- // + LayoutType type () const; + // Name and layout informations const std::vector& tags () const { return m_tags; } FieldTag tag (const int idim) const; @@ -128,7 +130,6 @@ class FieldLayout { }; bool operator== (const FieldLayout& fl1, const FieldLayout& fl2); -LayoutType get_layout_type (const std::vector& field_tags); std::string to_string (const FieldLayout& l); // ========================== IMPLEMENTATION ======================= // diff --git a/components/eamxx/src/share/field/field_manager.cpp b/components/eamxx/src/share/field/field_manager.cpp index 53f0c2b10a97..42c4d9f73e0c 100644 --- a/components/eamxx/src/share/field/field_manager.cpp +++ b/components/eamxx/src/share/field/field_manager.cpp @@ -525,13 +525,13 @@ void FieldManager::registration_ends () const auto& id = f->get_header().get_identifier(); if (lt==LayoutType::Invalid) { f_layout = id.get_layout(); - lt = get_layout_type(f_layout.tags()); + lt = f_layout.type(); } else { - EKAT_REQUIRE_MSG (lt==get_layout_type(id.get_layout().tags()), + EKAT_REQUIRE_MSG (lt==id.get_layout().type(), "Error! Found a group to bundle containing fields with different layouts.\n" " Group name: " + cluster_name + "\n" " Layout 1: " + e2str(lt) + "\n" - " Layout 2: " + e2str(get_layout_type(id.get_layout().tags())) + "\n"); + " Layout 2: " + e2str(id.get_layout().type()) + "\n"); } } @@ -651,7 +651,7 @@ void FieldManager::registration_ends () // whether they are 2d or 3d. auto f1 = m_fields.at(info.m_fields_names.front()); auto f1_layout = f1->get_header().get_identifier().get_layout(); - auto lt = get_layout_type(f1_layout.tags()); + auto lt = f1_layout.type(); FieldLayout g_layout = FieldLayout::invalid(); if (lt==LayoutType::Scalar2D) { g_layout = m_grid->get_2d_vector_layout(size); diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index ae304f18afd8..7f77c8e8ee3e 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -146,7 +146,7 @@ is_valid_layout (const FieldLayout& layout) const { using namespace ShortFieldTagsNames; - const auto lt = get_layout_type(layout.tags()); + const auto lt = layout.type(); if (lt==LayoutType::Scalar0D or lt==LayoutType::Vector0D) { // 0d layouts are compatible with any grid // Let's return true early to avoid segfautls below diff --git a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp index 210556241b78..2bce51a14a95 100644 --- a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp @@ -37,13 +37,8 @@ class DoNothingRemapper : public AbstractRemapper FieldLayout create_src_layout (const FieldLayout& tgt) const override { using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt), - "[DoNothingRemapper] Error! Input target layout is not valid for this remapper.\n" - " - input layout: " + to_string(tgt)); - - auto type = get_layout_type(tgt.tags()); - auto src = FieldLayout::invalid(); - switch (type) { + FieldLayout src = {{},{}}; + switch (tgt.type()) { case LayoutType::Scalar2D: src = this->m_src_grid->get_2d_scalar_layout(); break; @@ -57,7 +52,9 @@ class DoNothingRemapper : public AbstractRemapper src = this->m_src_grid->get_3d_vector_layout(tgt.has_tag(LEV),tgt.dim(CMP)); break; default: - EKAT_ERROR_MSG ("Error! Unsupported field layout.\n"); + EKAT_ERROR_MSG ( + "[DoNothingRemapper] Error! Input target layout is not valid for this remapper.\n" + " - input layout: " + to_string(tgt)); } return src; } @@ -65,13 +62,8 @@ class DoNothingRemapper : public AbstractRemapper FieldLayout create_tgt_layout (const FieldLayout& src) const override { using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG (is_valid_src_layout(src), - "[DoNothingRemapper] Error! Input source layout is not valid for this remapper.\n" - " - input layout: " + to_string(src)); - - auto type = get_layout_type(src.tags()); - auto tgt = FieldLayout::invalid(); - switch (type) { + FieldLayout tgt = {{},{}}; + switch (src.type()) { case LayoutType::Scalar2D: tgt = this->m_tgt_grid->get_2d_scalar_layout(); break; @@ -85,7 +77,9 @@ class DoNothingRemapper : public AbstractRemapper tgt = this->m_tgt_grid->get_3d_vector_layout(tgt.has_tag(LEV),tgt.dim(CMP)); break; default: - EKAT_ERROR_MSG ("Error! Unsupported field layout.\n"); + EKAT_ERROR_MSG ( + "[DoNothingRemapper] Error! Input source layout is not valid for this remapper.\n" + " - input layout: " + to_string(src)); } return tgt; } diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 0493f6d50825..676b315ee6f7 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -119,11 +119,10 @@ create_layout (const FieldLayout& fl_in, const grid_ptr_type& grid) const { using namespace ShortFieldTagsNames; - const auto type = get_layout_type(fl_in.tags()); auto fl_out = FieldLayout::invalid(); const bool midpoints = fl_in.has_tag(LEV); const bool is3d = fl_in.has_tag(LEV) or fl_in.has_tag(ILEV); - switch (type) { + switch (fl_in.type()) { case LayoutType::Scalar2D: fl_out = m_tgt_grid->get_2d_scalar_layout(); break; diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 71a961efe3f7..8cc4d146f38a 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -107,9 +107,8 @@ create_layout (const FieldLayout& fl_in, // between midpoints and interfaces: we're simply asking for a quantity // at a given set of pressure levels. So we choose to have fl_out // to *always* have LEV as vertical tag. - const auto lt = get_layout_type(fl_in.tags()); auto fl_out = FieldLayout::invalid(); - switch (lt) { + switch (fl_in.type()) { case LayoutType::Scalar0D: [[ fallthrough ]]; case LayoutType::Vector0D: [[ fallthrough ]]; case LayoutType::Scalar2D: [[ fallthrough ]]; @@ -134,8 +133,8 @@ create_layout (const FieldLayout& fl_in, // NOTE: this also include Tensor3D. We don't really have any atm proc // that needs to handle a tensor3d quantity, so no need to add it EKAT_ERROR_MSG ( - "Layout not supported by VerticalRemapper.\n" - " - input layout: " + to_string(fl_in) + "\n"); + "[VerticalRemapper] Error! Layout not supported by VerticalRemapper.\n" + " - input layout: " + to_string(fl_in) + "\n"); } return fl_out; } diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp index f04baa933cc8..e3e5c0781047 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp @@ -64,7 +64,7 @@ class VerticalRemapper : public AbstractRemapper for (int i=0; i(); diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 9897bee08aee..69fa64880e82 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -31,12 +31,12 @@ TEST_CASE("field_layout", "") { FieldLayout fl5 ({COL,CMP,LEV},{1,1,1}); FieldLayout fl6 ({COL,ISCCPTAU,ISCCPPRS,ILEV},{1,1,1,1}); - REQUIRE (get_layout_type(fl1.tags())==LayoutType::Scalar2D); - REQUIRE (get_layout_type(fl2.tags())==LayoutType::Vector2D); - REQUIRE (get_layout_type(fl3.tags())==LayoutType::Tensor2D); - REQUIRE (get_layout_type(fl4.tags())==LayoutType::Scalar3D); - REQUIRE (get_layout_type(fl5.tags())==LayoutType::Vector3D); - REQUIRE (get_layout_type(fl6.tags())==LayoutType::Tensor3D); + REQUIRE (fl1.type()==LayoutType::Scalar2D); + REQUIRE (fl2.type()==LayoutType::Vector2D); + REQUIRE (fl3.type()==LayoutType::Tensor2D); + REQUIRE (fl4.type()==LayoutType::Scalar3D); + REQUIRE (fl5.type()==LayoutType::Vector3D); + REQUIRE (fl6.type()==LayoutType::Tensor3D); REQUIRE (not fl1.is_vector_layout()); REQUIRE ( fl2.is_vector_layout()); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 8c720a5464b9..191b157bddb6 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -263,7 +263,7 @@ TEST_CASE ("vertical_remap") { auto src_gids = remap->get_src_grid()->get_dofs_gids().get_view(); for (const auto& f : src_f) { const auto& l = f.get_header().get_identifier().get_layout(); - switch (get_layout_type(l.tags())) { + switch (l.type()) { case LayoutType::Scalar2D: { const auto v_src = f.get_view(); @@ -334,7 +334,7 @@ TEST_CASE ("vertical_remap") { f.sync_to_host(); - switch (get_layout_type(lsrc.tags())) { + switch (lsrc.type()) { case LayoutType::Scalar2D: { // This is a flat array w/ no LEV tag so the interpolated value for source and target should match. From e6e926af090ce015f712d73da5460835609c83b6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 Nov 2023 15:29:48 -0700 Subject: [PATCH 052/476] EAMxx: precompute layout type and store inside FieldLayout --- .../eamxx/src/share/field/field_layout.cpp | 70 +++++++++---------- .../eamxx/src/share/field/field_layout.hpp | 5 +- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 7601b7b7af42..d64d556ff0a3 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -15,6 +15,8 @@ FieldLayout::FieldLayout (const std::vector& tags, for (int idim=0; idim FieldLayout::get_tensor_dims () const { EKAT_REQUIRE_MSG (is_tensor_layout(), "Error! 'get_tensor_dims' available only for tensor layouts.\n" " Current layout: " + to_string(*this) + "\n" - " Layout type : " + e2str(type()) + "\n"); + " Layout type : " + e2str(m_type) + "\n"); using namespace ShortFieldTagsNames; @@ -128,7 +129,12 @@ void FieldLayout::set_dimension (const int idim, const int dimension) { Kokkos::deep_copy(m_extents,extents_h); } -LayoutType FieldLayout::type () const { +void FieldLayout::compute_type () { + if (m_rank==0) { + m_type = LayoutType::Scalar0D; + return; + } + using namespace ShortFieldTagsNames; using ekat::erase; @@ -142,9 +148,6 @@ LayoutType FieldLayout::type () const { const int nvlevs = count(tags,LEV) + count(tags,ILEV); const int ncomps = count(tags,CMP); - // Start from undefined/invalid - LayoutType result = LayoutType::Invalid; - // We don't care about TimeLevel erase (tags,TL); @@ -161,61 +164,56 @@ LayoutType FieldLayout::type () const { // Remove the column tag erase(tags,COL); } else if (tags.size()==0) { - return LayoutType::Scalar0D; + m_type = LayoutType::Scalar0D; return; } else if (tags.size()==1 and tags[0]==CMP) { - return LayoutType::Vector0D; + m_type = LayoutType::Vector0D; return; } else if (tags.size()==1 and nvlevs==1) { - return LayoutType::Scalar1D; + m_type = LayoutType::Scalar1D; return; } else if (tags.size()==2 and ncomps==1 and nvlevs==1) { - return LayoutType::Vector1D; + m_type = LayoutType::Vector1D; return; } else { // Not a supported layout. - return LayoutType::Invalid; + m_type = LayoutType::Invalid; return; } // Get the size of what's left const auto size = tags.size(); auto is_lev_tag = [](const FieldTag t) { - std::vector lev_tags = {LEV,ILEV}; - return ekat::contains(lev_tags,t); - }; - auto is_cmp_tag = [](const FieldTag t) { - std::vector cmp_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; - return ekat::contains(cmp_tags,t); + return t==LEV or t==ILEV; }; switch (size) { case 0: - result = LayoutType::Scalar2D; + m_type = LayoutType::Scalar2D; break; case 1: - // The only tag left should be a cmp tag or a lev tag - if (is_cmp_tag(tags[0])) { - result = LayoutType::Vector2D; + // The only tag left should be 'CMP', 'TL', or 'LEV'/'ILEV' + if (tags[0]==CMP || tags[0]==TL) { + m_type = LayoutType::Vector2D; } else if (is_lev_tag(tags[0])) { - result = LayoutType::Scalar3D; + m_type = LayoutType::Scalar3D; } break; case 2: // Possible supported scenarios: - // 1) - // 3) - // where CMP,CMP1,CMP2 are any tag in cmp_tags - if ( is_cmp_tag(tags[0]) and is_lev_tag(tags[1]) ) { - result = LayoutType::Vector3D; - } else if (is_cmp_tag(tags[0]) and is_cmp_tag(tags[1])) { - result = LayoutType::Tensor2D; + // 1) + // 2) + if ( is_lev_tag(tags[1]) && (tags[0]==CMP || tags[0]==TL)) { + m_type = LayoutType::Vector3D; + } else if (tags[0]==TL && tags[1]==CMP ) { + m_type = LayoutType::Tensor2D; } break; case 3: // The only supported scenario is: - // 1) - // where CMP1,CMP2 are any tag in cmp_tags - if (is_cmp_tag(tags[0]) and is_cmp_tag(tags[1]) and is_lev_tag(tags[2])) { - result = LayoutType::Tensor3D; + // 1) + if ( tags[0]==TL && tags[1]==CMP && is_lev_tag(tags[2])) { + m_type = LayoutType::Tensor3D; } + break; + default: + // If nothing worked, this type is not recognized + m_type = LayoutType::Invalid; } - - return result; } std::string to_string (const FieldLayout& layout) diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 55569d1f53bf..f2cb7eab1a2c 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -74,7 +74,7 @@ class FieldLayout { // ----- Getters ----- // - LayoutType type () const; + LayoutType type () const { return m_type; } // Name and layout informations const std::vector& tags () const { return m_tags; } @@ -118,6 +118,7 @@ class FieldLayout { FieldLayout clone_with_different_extent (const int idim, const int extent) const; protected: + void compute_type (); // Only this class is allowed to change a layout. Customers can request // a *slightly* different layout (via strip_dim or clone_with_different_extent) @@ -127,6 +128,8 @@ class FieldLayout { std::vector m_tags; std::vector m_dims; extents_type m_extents; + + LayoutType m_type; }; bool operator== (const FieldLayout& fl1, const FieldLayout& fl2); From 3bc30edc6dacc60b8a92ae7ffce4ea5f4d84591e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 Nov 2023 15:42:27 -0700 Subject: [PATCH 053/476] EAMxx: allow to set names for each dimension inside a field layout If not provided, we default to what e2str(FieldTag) returns --- .../eamxx/src/share/field/field_layout.cpp | 40 ++++++++++++++++--- .../eamxx/src/share/field/field_layout.hpp | 40 ++++++++++++------- .../eamxx/src/share/field/field_tag.hpp | 9 +++++ 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index d64d556ff0a3..38da2ab7663f 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -7,9 +7,28 @@ namespace scream FieldLayout::FieldLayout (const std::vector& tags, const std::vector& dims) - : m_rank(tags.size()) - , m_tags(tags) + : FieldLayout (tags,dims,tags2str(tags)) { + // Nothing to do here +} + +FieldLayout::FieldLayout (const std::vector& tags, + const std::vector& dims, + const std::vector& names) + : m_rank (tags.size()) + , m_tags (tags) + , m_names(names) +{ + EKAT_REQUIRE_MSG (dims.size()==tags.size(), + "Error! Tags and dims vectors dimensions mismatch.\n" + " tags size: " + std::to_string(tags.size()) + "\n" + " dims size: " + std::to_string(dims.size()) + "\n"); + + EKAT_REQUIRE_MSG (names.size()==tags.size(), + "Error! Tags and names vectors dimensions mismatch.\n" + " tags size : " + std::to_string(tags.size()) + "\n" + " names size: " + std::to_string(names.size()) + "\n"); + m_dims.resize(m_rank,-1); m_extents = decltype(m_extents)("",m_rank); for (int idim=0; idim t = tags(); - std::vector d = dims(); + std::vector t = tags(); + std::vector d = dims(); + std::vector n = names(); t.erase(t.begin()+idim); d.erase(d.begin()+idim); - return FieldLayout (t,d); + n.erase(n.begin()+idim); + return FieldLayout (t,n,d); } FieldLayout FieldLayout::clone_with_different_extent (const int idim, const int extent) const @@ -129,6 +150,15 @@ void FieldLayout::set_dimension (const int idim, const int dimension) { Kokkos::deep_copy(m_extents,extents_h); } +void FieldLayout::rename_dim (const int idim, const std::string& n) { + EKAT_REQUIRE_MSG(idim>=0 && idim& tags, const std::vector& dims); + FieldLayout (const std::vector& tags, + const std::vector& dims, + const std::vector& names); // Assignment (defaulted) FieldLayout& operator= (const FieldLayout&) = default; @@ -77,8 +80,10 @@ class FieldLayout { LayoutType type () const { return m_type; } // Name and layout informations + const std::vector& names () const { return m_names; } const std::vector& tags () const { return m_tags; } FieldTag tag (const int idim) const; + const std::string& name (const int idim) const; bool has_tag (const FieldTag t) const { return ekat::contains(m_tags,t); } bool has_tags (const std::vector& tags) const; @@ -117,19 +122,20 @@ class FieldLayout { FieldLayout strip_dim (const int idim) const; FieldLayout clone_with_different_extent (const int idim, const int extent) const; + // Change the name of a dimension + void rename_dim (const int idim, const std::string& n); + void rename_dim (const FieldTag tag, const std::string& n); + protected: void compute_type (); - // Only this class is allowed to change a layout. Customers can request - // a *slightly* different layout (via strip_dim or clone_with_different_extent) - void set_dimension (const int idim, const int dimension); - - int m_rank; - std::vector m_tags; - std::vector m_dims; - extents_type m_extents; + int m_rank; + std::vector m_tags; + std::vector m_names; + std::vector m_dims; + extents_type m_extents; - LayoutType m_type; + LayoutType m_type; }; bool operator== (const FieldLayout& fl1, const FieldLayout& fl2); @@ -146,14 +152,14 @@ inline int FieldLayout::dim (const FieldTag t) const { // Check only one tag (no ambiguity) EKAT_REQUIRE_MSG(ekat::count(m_tags,t)==1, - "Error! Tag '" + e2str(t) + "' appears multiple times.\n" - " You must inspect tags() and dims() manually.\n"); + "Error! Tag '" + e2str(t) + "' appears multiple times.\n" + " You must inspect tags() and dims() manually.\n"); return m_dims[std::distance(m_tags.begin(),it)]; } inline int FieldLayout::dim (const int idim) const { - ekat::error::runtime_check(idim>=0 && idim=0 && idim=0 && idim=0 && idim=0 && idim& tags) const { bool b = true; for (auto t : tags) { diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index e685399a8bdc..0b07e265e666 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -151,6 +151,15 @@ inline std::string e2str (const FieldTag ft) { return name; } +inline std::vector tags2str (const std::vector& tags) { + std::vector names; + names.reserve(tags.size()); + for (auto t : tags) { + names.push_back(e2str(t)); + } + return names; +} + // Allow to stream FieldTag values as strings. inline std::ostream& operator<< (std::ostream& out, const FieldTag t) { out << e2str(t); From dcabd1e761e812e61ee186639288eaf7d6e45370 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 25 Mar 2024 15:38:32 -0600 Subject: [PATCH 054/476] EAMxx: distinguish between equality and congruence in FieldLayout Congruence requires only equality of tags and extents. Equality requires being congruent AND have same dim names. --- .../eamxx/src/share/field/field_layout.hpp | 14 ++++-- .../eamxx/src/share/grid/abstract_grid.cpp | 50 ++++++------------- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 27084dbf10d8..a739f8b015a4 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -122,6 +122,10 @@ class FieldLayout { FieldLayout strip_dim (const int idim) const; FieldLayout clone_with_different_extent (const int idim, const int extent) const; + // NOTE: congruent does not check the tags names. It only checks + // rank, m_tags, and m_dims. Use operator== if names are important + bool congruent (const FieldLayout& rhs) const; + // Change the name of a dimension void rename_dim (const int idim, const std::string& n); void rename_dim (const FieldTag tag, const std::string& n); @@ -201,10 +205,14 @@ inline bool FieldLayout::are_dimensions_set () const { return true; } +inline bool FieldLayout::congruent (const FieldLayout& rhs) const { + return rank()==rhs.rank() && + tags()==rhs.tags() && + dims()==rhs.dims(); +} + inline bool operator== (const FieldLayout& fl1, const FieldLayout& fl2) { - return fl1.rank()==fl2.rank() && - fl1.tags()==fl2.tags() && - fl1.dims()==fl2.dims(); + return fl1.congruent(fl2) and fl1.names()==fl2.names(); } } // namespace scream diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 7f77c8e8ee3e..4508762f658f 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -146,47 +146,25 @@ is_valid_layout (const FieldLayout& layout) const { using namespace ShortFieldTagsNames; - const auto lt = layout.type(); - if (lt==LayoutType::Scalar0D or lt==LayoutType::Vector0D) { - // 0d layouts are compatible with any grid - // Let's return true early to avoid segfautls below - return true; - } - - const bool midpoints = layout.tags().back()==LEV; - const bool is3d = layout.tags().back()==LEV or layout.tags().back()==ILEV; - - switch (lt) { + const bool midpoints = layout.has_tag(LEV); + switch (layout.type()) { + case LayoutType::Scalar0D: [[fallthrough]]; + case LayoutType::Vector0D: + // 0d quantities are always ok + return true; case LayoutType::Scalar1D: [[fallthrough]]; case LayoutType::Vector1D: - // 1d layouts need the right number of levels - return layout.dims().back() == m_num_vert_levs or - layout.dims().back() == (m_num_vert_levs+1); - case LayoutType::Scalar2D: [[fallthrough]]; + return layout.congruent(get_vertical_layout(midpoints); + case LayoutType::Scalar2D: + return layout.congruent(get_2d_scalar_layout()); case LayoutType::Scalar3D: - return is3d ? layout==get_3d_scalar_layout(midpoints) - : layout==get_2d_scalar_layout(); - case LayoutType::Vector2D: [[fallthrough]]; + return layout.congruent(get_3d_scalar_layout(midpoints); + case LayoutType::Vector2D: + return layout.congruent(get_2d_vector_layout(layout.get_vector_dim())); case LayoutType::Vector3D: - { - const auto vec_dim = layout.dims()[layout.get_vector_component_idx()]; - const auto vec_tag = layout.get_vector_tag(); - return is3d ? layout==get_3d_vector_layout(midpoints,vec_tag,vec_dim) - : layout==get_2d_vector_layout(vec_tag,vec_dim); - } - case LayoutType::Tensor2D: [[fallthrough]]; - case LayoutType::Tensor3D: - { - const auto ttags = layout.get_tensor_tags(); - std::vector tdims; - for (auto idx : layout.get_tensor_dims()) { - tdims.push_back(layout.dim(idx)); - } - return is3d ? layout==get_3d_tensor_layout(midpoints,ttags,tdims) - : layout==get_2d_tensor_layout(ttags,tdims); - } + return layout.congruent(get_3d_vector_layout(midpoints,layout.get_vector_dim())); default: - // Anything else is probably no + // Anything else is probably not ok return false; } } From 1f17b42564378ab84f5f96c43c01baeefeaa0ac0 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 Nov 2023 15:46:02 -0700 Subject: [PATCH 055/476] EAMxx: cannot call set_dimension in constructor, since dims may be invalid In fact, the set_dimension method is no longer needed --- .../eamxx/src/share/field/field_layout.cpp | 34 ++++++++----------- .../eamxx/src/share/field/field_layout.hpp | 1 + 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 38da2ab7663f..5b28790ac79e 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -13,11 +13,13 @@ FieldLayout::FieldLayout (const std::vector& tags, } FieldLayout::FieldLayout (const std::vector& tags, - const std::vector& dims, + const std::vector& dims, const std::vector& names) : m_rank (tags.size()) , m_tags (tags) , m_names(names) + , m_dims (dims) + , m_extents ("",tags.size()) { EKAT_REQUIRE_MSG (dims.size()==tags.size(), "Error! Tags and dims vectors dimensions mismatch.\n" @@ -29,12 +31,7 @@ FieldLayout::FieldLayout (const std::vector& tags, " tags size : " + std::to_string(tags.size()) + "\n" " names size: " + std::to_string(names.size()) + "\n"); - m_dims.resize(m_rank,-1); - m_extents = decltype(m_extents)("",m_rank); - for (int idim=0; idim=0 && idim=0, "Error! Dimensions must be non-negative."); - m_dims[idim] = dimension; + EKAT_REQUIRE_MSG(extent>=0, "Error! Dimensions must be non-negative."); - // Recompute device extents - auto extents_h = Kokkos::create_mirror_view(m_extents); - std::copy_n(m_dims.begin(),m_rank,extents_h.data()); - Kokkos::deep_copy(m_extents,extents_h); + auto dims = m_dims; + dims[idim] = extent; + return FieldLayout (m_tags,m_names,dims); } void FieldLayout::rename_dim (const int idim, const std::string& n) { @@ -159,6 +147,12 @@ void FieldLayout::rename_dim (const FieldTag tag, const std::string& n) { rename_dim(dim(tag),n); } +void FieldLayout::set_extents () { + auto extents_h = Kokkos::create_mirror_view(m_extents); + std::copy_n(m_dims.begin(),m_rank,extents_h.data()); + Kokkos::deep_copy(m_extents,extents_h); +} + void FieldLayout::compute_type () { if (m_rank==0) { m_type = LayoutType::Scalar0D; diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index a739f8b015a4..54d1ee0cb9d1 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -132,6 +132,7 @@ class FieldLayout { protected: void compute_type (); + void set_extents (); int m_rank; std::vector m_tags; From 582ff93cef3666e3733176d0b59fd95321a0f84e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 28 Mar 2024 18:26:04 -0600 Subject: [PATCH 056/476] EAMxx: change interfaces of FieldLayout and add new ones * The rename/strip method are now changing the current object, and return *this, so they can be chained * Add append_dim to allow easier creation of layout by means of continuous appending of dims * As done for vector layouts, distinguish getting dim_idx vs dim_extent for tensor components * Allow to query dim extent by dim name --- .../control/intensive_observation_period.cpp | 2 +- .../eamxx/src/diagnostics/field_at_height.cpp | 2 +- .../eamxx/src/diagnostics/field_at_level.cpp | 2 +- .../diagnostics/field_at_pressure_level.cpp | 2 +- .../physics/nudging/tests/nudging_tests.cpp | 2 +- components/eamxx/src/share/field/field.cpp | 2 +- .../src/share/field/field_alloc_prop.cpp | 2 +- .../eamxx/src/share/field/field_layout.cpp | 82 ++++++++++++++----- .../eamxx/src/share/field/field_layout.hpp | 35 ++++++-- .../src/share/field/field_utils_impl.hpp | 4 +- .../share/grid/remap/coarsening_remapper.cpp | 2 +- .../grid/remap/horiz_interp_remapper_base.hpp | 2 +- .../grid/remap/refining_remapper_p2p.cpp | 2 +- .../grid/remap/refining_remapper_rma.cpp | 2 +- .../share/grid/remap/vertical_remapper.cpp | 4 +- .../share/tests/coarsening_remapper_tests.cpp | 2 +- .../tests/refining_remapper_p2p_tests.cpp | 2 +- .../tests/refining_remapper_rma_tests.cpp | 4 +- 18 files changed, 111 insertions(+), 44 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 5d617fcd4ced..9f9a3474caf5 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -522,7 +522,7 @@ read_fields_from_file_for_iop (const std::string& file_name, // Create a temporary field to store the data from the // single column of the closest lat/lon pair const auto io_fid = io_field.get_header().get_identifier(); - FieldLayout col_data_fl = io_fid.get_layout().strip_dim(0); + FieldLayout col_data_fl = io_fid.get_layout().clone().strip_dim(0); FieldIdentifier col_data_fid("col_data", col_data_fl, dummy_units, ""); Field col_data(col_data_fid); col_data.allocate_view(); diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index 38ae5e3e5503..39be5b1442e0 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -100,7 +100,7 @@ initialize_impl (const RunType /*run_type*/) m_z_suffix = tag==LEV ? "_mid" : "_int"; // All good, create the diag output - FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); + FieldIdentifier d_fid (m_diag_name,layout.clone().strip_dim(tag),fid.get_units(),fid.get_grid_name()); m_diagnostic_output = Field(d_fid); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/field_at_level.cpp b/components/eamxx/src/diagnostics/field_at_level.cpp index 3634cac2729c..7c36a508a8e9 100644 --- a/components/eamxx/src/diagnostics/field_at_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_level.cpp @@ -65,7 +65,7 @@ initialize_impl (const RunType /*run_type*/) } // All good, create the diag output - FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); + FieldIdentifier d_fid (m_diag_name,layout.clone().strip_dim(tag),fid.get_units(),fid.get_grid_name()); m_diagnostic_output = Field(d_fid); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index ea47b8c8f54e..27e97cc90c0a 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -74,7 +74,7 @@ initialize_impl (const RunType /*run_type*/) " - field layout: " + to_string(layout) + "\n"); // All good, create the diag output - FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); + FieldIdentifier d_fid (m_diag_name,layout.clone().strip_dim(tag),fid.get_units(),fid.get_grid_name()); m_diagnostic_output = Field(d_fid); m_diagnostic_output.allocate_view(); diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 0f0742875fd1..f54112056279 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -295,7 +295,7 @@ TEST_CASE("nudging_tests") { int ncols = fid.get_layout().dim(0); comm.all_reduce(&ncols,1,MPI_SUM); - FieldLayout glb_layout = fid.get_layout().clone_with_different_extent(0,ncols); + FieldLayout glb_layout = fid.get_layout().clone().reset_dim(0,ncols); FieldIdentifier glb_fid(fid.name(),glb_layout,fid.get_units(),fid.get_grid_name()); Field glb(glb_fid); glb.allocate_view(); diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 7f872e037224..e022278d5030 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -92,7 +92,7 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, "Error! Subview dimension index must be either 0 or 1.\n"); // Create identifier for subfield - FieldIdentifier sf_id(sf_name,lt.strip_dim(idim),sf_units,id.get_grid_name()); + FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index 7c589678defb..fcffc821bbaa 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -47,7 +47,7 @@ subview (const int idim, const int k, const bool dynamic) const { FieldAllocProp props(m_scalar_type_size); props.m_committed = true; props.m_scalar_type_size = m_scalar_type_size; - props.m_layout = m_layout.strip_dim(idim); + props.m_layout = m_layout.clone().strip_dim(idim); // Output is contioguous if either // - this->m_contiguous=true AND idim==0 diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 5b28790ac79e..14cc20551257 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -5,6 +5,13 @@ namespace scream { +FieldLayout:: +FieldLayout () + : FieldLayout({},{}) +{ + +} + FieldLayout::FieldLayout (const std::vector& tags, const std::vector& dims) : FieldLayout (tags,dims,tags2str(tags)) @@ -69,7 +76,7 @@ FieldTag FieldLayout::get_vector_tag () const { return m_tags[get_vector_component_idx()]; } -std::vector FieldLayout::get_tensor_dims () const { +std::vector FieldLayout::get_tensor_components_ids () const { EKAT_REQUIRE_MSG (is_tensor_layout(), "Error! 'get_tensor_dims' available only for tensor layouts.\n" " Current layout: " + to_string(*this) + "\n" @@ -95,12 +102,20 @@ std::vector FieldLayout::get_tensor_dims () const { return idx; } +std::vector FieldLayout::get_tensor_dims () const { + auto idx = get_tensor_components_ids(); + for (auto& i : idx) { + i = m_dims[i]; + } + return idx; +} + std::vector FieldLayout::get_tensor_tags () const { auto idx = get_tensor_dims(); return {m_tags[idx[0]], m_tags[idx[1]]}; } -FieldLayout FieldLayout::strip_dim (const FieldTag tag) const { +FieldLayout& FieldLayout::strip_dim (const FieldTag tag) { auto it = ekat::find(m_tags,tag); // Check if found @@ -111,40 +126,69 @@ FieldLayout FieldLayout::strip_dim (const FieldTag tag) const { "Error! Tag '" + e2str(tag) + "' appears multiple times.\n" " You must inspect tags() and dims() manually.\n"); - return strip_dim (std::distance(m_tags.begin(),it)); + auto pos = std::distance(m_tags.begin(),it); + return strip_dim(pos); } -FieldLayout FieldLayout::strip_dim (const int idim) const { +FieldLayout& FieldLayout::strip_dim (const int idim) { EKAT_REQUIRE_MSG (idim>=0 and idim t = tags(); - std::vector d = dims(); - std::vector n = names(); - t.erase(t.begin()+idim); - d.erase(d.begin()+idim); - n.erase(n.begin()+idim); - return FieldLayout (t,n,d); + + m_tags.erase(m_tags.begin()+idim); + m_names.erase(m_names.begin()+idim); + m_dims.erase(m_dims.begin()+idim); + --m_rank; + + set_extents (); + compute_type (); + return *this; } -FieldLayout FieldLayout::clone_with_different_extent (const int idim, const int extent) const +FieldLayout& +FieldLayout::append_dim (const FieldTag t, const int extent) { - EKAT_REQUIRE_MSG(idim>=0 && idim=0, "Error! Dimensions must be non-negative."); + return append_dim(t,extent,e2str(t)); +} - auto dims = m_dims; - dims[idim] = extent; - return FieldLayout (m_tags,m_names,dims); +FieldLayout& +FieldLayout::append_dim (const FieldTag t, const int extent, const std::string& name) +{ + m_tags.push_back(t); + m_names.push_back(name); + m_dims.push_back(extent); + + ++m_rank; + set_extents(); + compute_type(); + return *this; +} + +FieldLayout FieldLayout::clone() const +{ + return *this; } -void FieldLayout::rename_dim (const int idim, const std::string& n) { +FieldLayout& FieldLayout::rename_dim (const int idim, const std::string& n) +{ EKAT_REQUIRE_MSG(idim>=0 && idim=0 && idim +#include #include #include @@ -61,7 +62,7 @@ class FieldLayout { using extents_type = typename KokkosTypes::view_1d; // Constructor(s) - FieldLayout () = delete; + FieldLayout (); FieldLayout (const FieldLayout&) = default; FieldLayout (const std::vector& tags, const std::vector& dims); @@ -90,6 +91,7 @@ class FieldLayout { // The rank is the number of tags associated to this field. int rank () const { return m_rank; } + int dim (const std::string& name) const; int dim (const FieldTag tag) const; int dim (const int idim) const; const std::vector& dims () const { return m_dims; } @@ -114,21 +116,26 @@ class FieldLayout { // If this is the layout of a tensor field, get the idx of the tensor dimensions // Note: throws if is_tensor_layout()==false. + std::vector get_tensor_components_ids () const; + // Get the dimension (extent) of the tensor components. Calls get_tensor_components_ids std::vector get_tensor_dims () const; std::vector get_tensor_tags () const; // Returns a copy of this layout with a given dimension stripped - FieldLayout strip_dim (const FieldTag tag) const; - FieldLayout strip_dim (const int idim) const; - FieldLayout clone_with_different_extent (const int idim, const int extent) const; + FieldLayout& strip_dim (const FieldTag tag); + FieldLayout& strip_dim (const int idim); + FieldLayout& append_dim (const FieldTag t, const int extent); + FieldLayout& append_dim (const FieldTag t, const int extent, const std::string& name); + FieldLayout clone() const; // NOTE: congruent does not check the tags names. It only checks // rank, m_tags, and m_dims. Use operator== if names are important bool congruent (const FieldLayout& rhs) const; // Change the name of a dimension - void rename_dim (const int idim, const std::string& n); - void rename_dim (const FieldTag tag, const std::string& n); + FieldLayout& rename_dim (const int idim, const std::string& n); + FieldLayout& rename_dim (const FieldTag tag, const std::string& n); + FieldLayout& reset_dim (const int idim, const int extent); protected: void compute_type (); @@ -163,6 +170,22 @@ inline int FieldLayout::dim (const FieldTag t) const { return m_dims[std::distance(m_tags.begin(),it)]; } +inline int FieldLayout::dim (const std::string& name) const { + auto it = ekat::find(m_names,name); + + // Check if found + EKAT_REQUIRE_MSG(it!=m_names.end(), + "Error! Dim name '" + name + "' not found in this layout.\n" + " - layout dims: " + ekat::join(m_names,",") + "\n"); + + // Check only one tag (no ambiguity) + EKAT_REQUIRE_MSG(ekat::count(m_names,name)==1, + "Error! Dimension name '" + name + "' appears multiple times.\n" + " - layout dims: " + ekat::join(m_names,",") + "\n"); + + return m_dims[std::distance(m_names.begin(),it)]; +} + inline int FieldLayout::dim (const int idim) const { EKAT_REQUIRE_MSG (idim>=0 && idim Date: Thu, 30 Nov 2023 17:03:12 -0700 Subject: [PATCH 057/476] EAMxx: add some shortcuts to layout creation methods in AbstractGrid --- .../eamxx/src/share/field/field_manager.cpp | 2 +- .../eamxx/src/share/grid/abstract_grid.cpp | 33 ++++++++- .../eamxx/src/share/grid/abstract_grid.hpp | 20 ++++- .../eamxx/src/share/grid/point_grid.cpp | 58 ++++++++++----- .../eamxx/src/share/grid/point_grid.hpp | 12 ++- .../grid/remap/horiz_interp_remapper_base.cpp | 29 ++------ .../share/grid/remap/vertical_remapper.cpp | 9 +-- components/eamxx/src/share/grid/se_grid.cpp | 74 +++++++++++-------- components/eamxx/src/share/grid/se_grid.hpp | 12 ++- 9 files changed, 156 insertions(+), 93 deletions(-) diff --git a/components/eamxx/src/share/field/field_manager.cpp b/components/eamxx/src/share/field/field_manager.cpp index 42c4d9f73e0c..1ea13c8e8dd0 100644 --- a/components/eamxx/src/share/field/field_manager.cpp +++ b/components/eamxx/src/share/field/field_manager.cpp @@ -657,7 +657,7 @@ void FieldManager::registration_ends () g_layout = m_grid->get_2d_vector_layout(size); } else { bool mid = f1_layout.tags().back()==LEV; - g_layout = m_grid->get_3d_vector_layout(mid,size); + g_layout = m_grid->get_3d_vector_layout(mid,size,e2str(CMP)); } FieldIdentifier g_fid(gname,g_layout,nondim,m_grid->name()); diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 4508762f658f..4b66a23d6de5 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -71,7 +71,36 @@ get_vertical_layout (const bool midpoints) const using namespace ShortFieldTagsNames; return midpoints ? FieldLayout ({ LEV},{m_num_vert_levs}) : FieldLayout ({ILEV},{m_num_vert_levs+1}); +} + +FieldLayout +AbstractGrid::get_2d_vector_layout (const int vector_dim) const +{ + using namespace ShortFieldTagsNames; + return get_2d_vector_layout(vector_dim,e2str(CMP)); +} +FieldLayout +AbstractGrid::get_2d_tensor_layout (const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + std::vector names (cmp_dims.size(),e2str(CMP)); + return get_2d_tensor_layout(cmp_dims,names); +} + +FieldLayout +AbstractGrid::get_3d_vector_layout (const bool midpoints, const int vector_dim) const +{ + using namespace ShortFieldTagsNames; + return get_3d_vector_layout(midpoints,vector_dim,e2str(CMP)); +} + +FieldLayout +AbstractGrid::get_3d_tensor_layout (const bool midpoints, const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + std::vector names (cmp_dims.size(),e2str(CMP)); + return get_3d_tensor_layout(midpoints,cmp_dims,names); } bool AbstractGrid::is_unique () const { @@ -154,11 +183,11 @@ is_valid_layout (const FieldLayout& layout) const return true; case LayoutType::Scalar1D: [[fallthrough]]; case LayoutType::Vector1D: - return layout.congruent(get_vertical_layout(midpoints); + return layout.congruent(get_vertical_layout(midpoints)); case LayoutType::Scalar2D: return layout.congruent(get_2d_scalar_layout()); case LayoutType::Scalar3D: - return layout.congruent(get_3d_scalar_layout(midpoints); + return layout.congruent(get_3d_scalar_layout(midpoints)); case LayoutType::Vector2D: return layout.congruent(get_2d_vector_layout(layout.get_vector_dim())); case LayoutType::Vector3D: diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 9a5421a25c66..f969a36b2d04 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -73,12 +73,24 @@ class AbstractGrid : public ekat::enable_shared_from_this // for a vector 3d field on a Point grid it will be (ncols,vector_dim,nlevs) FieldLayout get_vertical_layout (const bool midpoints) const; virtual FieldLayout get_2d_scalar_layout () const = 0; - virtual FieldLayout get_2d_vector_layout (const int vector_dim) const = 0; - virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const = 0; + virtual FieldLayout get_2d_vector_layout (const int vector_dim, const std::string& vec_dim_name) const = 0; + virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims, + const std::vector& cmp_dims_names) const = 0; virtual FieldLayout get_3d_scalar_layout (const bool midpoints) const = 0; - virtual FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const = 0; + virtual FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim, + const std::string& vec_dim_name) const = 0; virtual FieldLayout get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_dims) const = 0; + const std::vector& cmp_dims, + const std::vector& cmp_dims_names) const = 0; + + // Some shortcut versions of the above ones, where the name of the vector/tensor + // components are all equal to e2str(CMP) + FieldLayout get_2d_vector_layout (const int vector_dim) const; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const; + + FieldLayout get_3d_vector_layout (const bool midpoints) const; + FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const; + FieldLayout get_3d_tensor_layout (const bool midpoints, const std::vector& cmp_dims) const; int get_num_vertical_levels () const { return m_num_vert_levs; } diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index ceb7680992c6..70b2ce81e03a 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -48,25 +48,34 @@ PointGrid::get_2d_scalar_layout () const } FieldLayout -PointGrid::get_2d_vector_layout (const int vector_dim) const +PointGrid::get_2d_vector_layout (const int vector_dim, const std::string& vec_dim_name) const { using namespace ShortFieldTagsNames; - return FieldLayout({COL,CMP},{get_num_local_dofs(),vector_dim}); + FieldLayout fl({COL,CMP},{get_num_local_dofs(),vector_dim}); + fl.rename_dim(1,vec_dim_name); + return fl; } FieldLayout -PointGrid::get_2d_tensor_layout (const std::vector& cmp_tags, - const std::vector& cmp_dims) const +PointGrid::get_2d_tensor_layout (const std::vector& cmp_dims, + const std::vector& cmp_names) const { + EKAT_REQUIRE_MSG (cmp_names.size()==cmp_dims.size(), + "[PointGrid::get_2d_tensor_layout] Input vector dimensions mismatch.\n" + " - grid name: " + name() + "\n" + " - cmp_names: " + ekat::join(cmp_names,",") + "\n" + " - cmp_dims : " + ekat::join(cmp_dims,",") + "\n"); using namespace ShortFieldTagsNames; - std::vector tags = {COL}; - std::vector dims = {get_num_local_dofs()}; + FieldLayout fl; - tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); - dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); - return FieldLayout(tags,dims); + fl.append_dim(COL, get_num_local_dofs()); + for (size_t i=0; iget_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - return FieldLayout({COL,CMP,VL},{get_num_local_dofs(),vector_dim,nvl}); + FieldLayout fl({COL,CMP,VL},{get_num_local_dofs(),vector_dim,nvl}); + fl.rename_dim(1,vec_dim_name); + return fl; } FieldLayout PointGrid::get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_tags, - const std::vector& cmp_dims) const + const std::vector& cmp_dims, + const std::vector& cmp_names) const { + EKAT_REQUIRE_MSG (cmp_names.size()==cmp_dims.size(), + "[PointGrid::get_2d_tensor_layout] Input vector dimensions mismatch.\n" + " - grid name: " + name() + "\n" + " - cmp_names: " + ekat::join(cmp_names,",") + "\n" + " - cmp_dims : " + ekat::join(cmp_dims,",") + "\n"); using namespace ShortFieldTagsNames; int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - std::vector tags = {COL}; - std::vector dims = {get_num_local_dofs()}; + FieldLayout fl; + + fl.append_dim(COL, get_num_local_dofs()); + for (size_t i=0; i diff --git a/components/eamxx/src/share/grid/point_grid.hpp b/components/eamxx/src/share/grid/point_grid.hpp index 89e72b5be9f0..771c6b55847d 100644 --- a/components/eamxx/src/share/grid/point_grid.hpp +++ b/components/eamxx/src/share/grid/point_grid.hpp @@ -44,12 +44,16 @@ class PointGrid : public AbstractGrid // Native layout of a dof. This is the natural way to index a dof in the grid. // E.g., for a 2d structured grid, this could be a set of 2 indices. FieldLayout get_2d_scalar_layout () const override; - FieldLayout get_2d_vector_layout (const int vector_dim) const override; - FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims) const override; + FieldLayout get_2d_vector_layout (const int vector_dim, + const std::string& vec_dim_name) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims, + const std::vector& cmp_names) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; - FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const override; + FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim, + const std::string& vec_dim_name) const override; FieldLayout get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_dims) const override; + const std::vector& cmp_dims, + const std::vector& cmp_names) const override; FieldTag get_partitioned_dim_tag () const override { return FieldTag::Column; diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 676b315ee6f7..1707ab4dcdc2 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -118,42 +118,27 @@ FieldLayout HorizInterpRemapperBase:: create_layout (const FieldLayout& fl_in, const grid_ptr_type& grid) const { - using namespace ShortFieldTagsNames; auto fl_out = FieldLayout::invalid(); + using namespace ShortFieldTagsNames; const bool midpoints = fl_in.has_tag(LEV); - const bool is3d = fl_in.has_tag(LEV) or fl_in.has_tag(ILEV); switch (fl_in.type()) { case LayoutType::Scalar2D: - fl_out = m_tgt_grid->get_2d_scalar_layout(); + fl_out = grid->get_2d_scalar_layout(); break; case LayoutType::Vector2D: - fl_out = m_tgt_grid->get_2d_vector_layout(fl_in.dim(CMP)); + fl_out = grid->get_2d_vector_layout(fl_in.get_vector_dim()); break; case LayoutType::Tensor2D: - { - std::vector tdims; - for (auto idx : fl_in.get_tensor_dims()) { - tdims.push_back(fl_in.dim(idx)); - } - - fl_out = m_tgt_grid->get_2d_tensor_layout(fl_in.dim(CMP)); + fl_out = grid->get_2d_tensor_layout(fl_in.get_tensor_dims()); break; - } case LayoutType::Scalar3D: fl_out = grid->get_3d_scalar_layout(midpoints); break; case LayoutType::Vector3D: - fl_out = grid->get_3d_vector_layout(midpoints,fl_in.dim(CMP)); + fl_out = grid->get_3d_vector_layout(midpoints,fl_in.get_vector_dim()); break; case LayoutType::Tensor3D: - { - std::vector tdims; - for (auto idx : fl_in.get_tensor_dims()) { - tdims.push_back(fl_in.dim(idx)); - } - fl_out = grid->get_3d_tensor_layout(midpoints,tdims); - break; - } + fl_out = grid->get_3d_tensor_layout(midpoints,fl_in.get_tensor_dims()); default: EKAT_ERROR_MSG ("Layout not supported by HorizInterpRemapperBase:\n" " - layout: " + to_string(fl_in) + "\n"); @@ -213,7 +198,7 @@ void HorizInterpRemapperBase::create_ov_fields () for (int i=0; iget_vertical_layout(true); break; - case LayoutType::Vector2D: - tgt = m_tgt_grid->get_2d_vector_layout(fl_in.dim(CMP)); - break; case LayoutType::Scalar3D: fl_out = grid_out->get_3d_scalar_layout(true); break; case LayoutType::Vector3D: - fl_out = grid_out->get_3d_vector_layout(true,vec_dim,fl_in.dim(CMP)); + fl_out = grid_out->get_3d_vector_layout(true,fl_in.get_vector_dim()); break; default: // NOTE: this also include Tensor3D. We don't really have any atm proc diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index 6181c8e7c497..3ef770cf0c7c 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -40,29 +40,37 @@ SEGrid::get_2d_scalar_layout () const } FieldLayout -SEGrid::get_2d_vector_layout (const int vector_dim) const +SEGrid::get_2d_vector_layout (const int vector_dim, const std::string& vec_dim_name) const { using namespace ShortFieldTagsNames; - return FieldLayout({EL,CMP,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); + FieldLayout fl({EL,CMP,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); + fl.rename_dim(1,vec_dim_name); + return fl; } FieldLayout -SEGrid::get_2d_tensor_layout (const std::vector& cmp_tags, - const std::vector& cmp_dims) const +SEGrid::get_2d_tensor_layout (const std::vector& cmp_dims, + const std::vector& cmp_names) const { + EKAT_REQUIRE_MSG (cmp_names.size()==cmp_dims.size(), + "[SEGrid::get_2d_tensor_layout] Input vector dimensions mismatch.\n" + " - grid name: " + name() + "\n" + " - cmp_names: " + ekat::join(cmp_names,",") + "\n" + " - cmp_dims : " + ekat::join(cmp_dims,",") + "\n"); + using namespace ShortFieldTagsNames; - std::vector tags = {EL}; - std::vector dims = {m_num_local_elem}; + FieldLayout fl; + + fl = fl.append_dim(EL,m_num_local_elem); - tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); - dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); - tags.push_back(GP); - tags.push_back(GP); - dims.push_back(m_num_gp); - dims.push_back(m_num_gp); - return FieldLayout(tags,dims); + for (size_t i=0; iget_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - return FieldLayout({EL,CMP,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); + FieldLayout fl({EL,CMP,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); + fl.rename_dim(1,vec_dim_name); + return fl; } FieldLayout SEGrid::get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_tags, - const std::vector& cmp_dims) const + const std::vector& cmp_dims, + const std::vector& cmp_names) const { + EKAT_REQUIRE_MSG (cmp_names.size()==cmp_dims.size(), + "[SEGrid::get_2d_tensor_layout] Input vector dimensions mismatch.\n" + " - grid name: " + name() + "\n" + " - cmp_names: " + ekat::join(cmp_names,",") + "\n" + " - cmp_dims : " + ekat::join(cmp_dims,",") + "\n"); + using namespace ShortFieldTagsNames; int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - std::vector tags = {EL}; - std::vector dims = {m_num_local_elem}; - - tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); - dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); - tags.push_back(GP); - tags.push_back(GP); - tags.push_back(VL); - dims.push_back(m_num_gp); - dims.push_back(m_num_gp); - dims.push_back(nvl); - return FieldLayout(tags,dims); + FieldLayout fl; + + fl.append_dim(EL,m_num_local_elem); + + for (size_t i=0; i& cmp_dims) const override; + FieldLayout get_2d_vector_layout (const int vector_dim, + const std::string& vec_dim_name) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims, + const std::vector& cmp_names) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; - FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim) const override; + FieldLayout get_3d_vector_layout (const bool midpoints, const int vector_dim, + const std::string& vec_dim_name) const override; FieldLayout get_3d_tensor_layout (const bool midpoints, - const std::vector& cmp_dims) const override; + const std::vector& cmp_dims, + const std::vector& cmp_names) const override; FieldTag get_partitioned_dim_tag () const override { return FieldTag::Element; From c0af96d50616dfa6556c18638aec6a56d8c37b8c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 29 Mar 2024 11:37:35 -0600 Subject: [PATCH 058/476] EAMxx: purge RAD/MAM field tags Use regular CMP tag with rad-specific names --- components/eamxx/src/diagnostics/aodvis.cpp | 5 +- .../src/diagnostics/tests/aodvis_test.cpp | 2 +- .../eamxx/src/physics/cosp/eamxx_cosp.cpp | 51 +++--- .../eamxx_mam_optics_process_interface.cpp | 103 +++++------- .../mam/mam_aerosol_optics_read_tables.hpp | 76 ++++----- .../eamxx_nudging_process_interface.cpp | 5 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 154 +++++++++--------- .../spa/eamxx_spa_process_interface.cpp | 26 +-- .../src/physics/spa/spa_functions_impl.hpp | 8 +- .../eamxx/src/share/field/field_tag.hpp | 66 -------- .../eamxx/src/share/tests/field_tests.cpp | 12 +- 11 files changed, 209 insertions(+), 299 deletions(-) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index 5eea4fcd6a17..efb9b5ba9510 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -23,9 +23,8 @@ void AODVis::set_grids( m_nlevs = grid->get_num_vertical_levels(); // Define layouts we need (both inputs and outputs) - FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, - {m_ncols, m_swbands, m_nlevs}}; - FieldLayout scalar1d_layout{{COL}, {m_ncols}}; + FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,m_swbands,"swband"); + FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 6a2f2f3efd16..140114812215 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -49,7 +49,7 @@ TEST_CASE("aodvis") { auto grid = gm->get_grid("Physics"); // Input (randomized) tau - FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, {ngcols, nbnds, nlevs}}; + FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,nbnds,"swband"); FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, grid->name()); Field tau(tau_fid); diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 478fce989276..d3d146797d1f 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -50,41 +50,42 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) // Define the different field layouts that will be used for this process // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces - FieldLayout scalar2d_layout { {COL}, {m_num_cols} }; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_num_cols,m_num_levs+1} }; - FieldLayout scalar4d_layout_ctptau { {COL,ISCCPTAU,ISCCPPRS}, {m_num_cols,m_num_isccptau,m_num_isccpctp} }; + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); + FieldLayout scalar4d_ctptau ( {COL,CMP,CMP}, + {m_num_cols,m_num_isccptau,m_num_isccpctp}, + {e2str(COL), "ISCCPTAU", "ISCCPPRS"}); // Set of fields used strictly as input // Name in AD Layout Units Grid Group - add_field("surf_radiative_T", scalar2d_layout , K, grid_name); - //add_field("surfelev", scalar2d_layout , m, grid_name); - //add_field("landmask", scalar2d_layout , nondim, grid_name); - add_field("sunlit", scalar2d_layout , nondim, grid_name); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); - add_field("p_int", scalar3d_layout_int, Pa, grid_name); - //add_field("height_mid", scalar3d_layout_mid, m, grid_name); - //add_field("height_int", scalar3d_layout_int, m, grid_name); - add_field("T_mid", scalar3d_layout_mid, K, grid_name); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers"); - add_field("qc", scalar3d_layout_mid, Q, grid_name, "tracers"); - add_field("qi", scalar3d_layout_mid, Q, grid_name, "tracers"); - add_field("cldfrac_rad", scalar3d_layout_mid, nondim, grid_name); + add_field("surf_radiative_T", scalar2d , K, grid_name); + //add_field("surfelev", scalar2d , m, grid_name); + //add_field("landmask", scalar2d , nondim, grid_name); + add_field("sunlit", scalar2d , nondim, grid_name); + add_field("p_mid", scalar3d_mid, Pa, grid_name); + add_field("p_int", scalar3d_int, Pa, grid_name); + //add_field("height_mid", scalar3d_mid, m, grid_name); + //add_field("height_int", scalar3d_int, m, grid_name); + add_field("T_mid", scalar3d_mid, K, grid_name); + add_field("qv", scalar3d_mid, Q, grid_name, "tracers"); + add_field("qc", scalar3d_mid, Q, grid_name, "tracers"); + add_field("qi", scalar3d_mid, Q, grid_name, "tracers"); + add_field("cldfrac_rad", scalar3d_mid, nondim, grid_name); // Optical properties, should be computed in radiation interface - add_field("dtau067", scalar3d_layout_mid, nondim, grid_name); // 0.67 micron optical depth - add_field("dtau105", scalar3d_layout_mid, nondim, grid_name); // 10.5 micron optical depth + add_field("dtau067", scalar3d_mid, nondim, grid_name); // 0.67 micron optical depth + add_field("dtau105", scalar3d_mid, nondim, grid_name); // 10.5 micron optical depth // Effective radii, should be computed in either microphysics or radiation interface // TODO: should these be meters or microns? Was meters before, but using "m" instead // of "micron" seemed to cause prim_model_finalize to throw error with the following: // ABORTING WITH ERROR: Error! prim_init_model_f90 was not called yet (or prim_finalize_f90 was already called). // P3 defines this field with micron instead of meters units, so is this a unit conversion issue? - add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name); - add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name); + add_field("eff_radius_qc", scalar3d_mid, micron, grid_name); + add_field("eff_radius_qi", scalar3d_mid, micron, grid_name); // Set of fields used strictly as output - add_field("isccp_cldtot", scalar2d_layout, percent, grid_name); - add_field("isccp_ctptau", scalar4d_layout_ctptau, percent, grid_name, 1); - add_field("isccp_mask" , scalar2d_layout, nondim, grid_name); - + add_field("isccp_cldtot", scalar2d, percent, grid_name); + add_field("isccp_ctptau", scalar4d_ctptau, percent, grid_name, 1); + add_field("isccp_mask" , scalar2d, nondim, grid_name); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index a827347c5385..f25bb8cf7281 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -42,78 +42,60 @@ void MAMOptics::set_grids( // Define aerosol optics fields computed by this process. auto nondim = Units::nondimensional(); - // 3D layout for shortwave aerosol fields: columns, number of shortwave, and nlev - FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, - {ncol_, nswbands_, nlev_}}; - - // 3D layout for longwave aerosol fields: columns, number of shortwave, and nlev +1 s - FieldLayout scalar3d_lwband_layout{{COL, LWBND, LEV}, - {ncol_, nlwbands_, nlev_}}; - - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - - // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; - - // layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - add_field("omega", scalar3d_layout_mid, Pa / s, - grid_name); // vertical pressure velocity - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // Temperature - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // total pressure - - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); - add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name); - - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice wet mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + // 3D layout for short/longwave aerosol fields: columns, number of short/longwave band, nlev + FieldLayout scalar3d_swband = grid_->get_3d_vector_layout(true,nswbands_,"swband"); + FieldLayout scalar3d_lwband = grid_->get_3d_vector_layout(true,nlwbands_,"lwband"); + + // layout for 3D (2d horiz X 1d vertical) variables at level midpoints/interfaces + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); + + // layout for 2D (1d horiz X 1d vertical) variables + FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + + add_field("omega", scalar3d_mid, Pa / s, grid_name); // vertical pressure velocity + add_field("T_mid", scalar3d_mid, K, grid_name); // Temperature + add_field("p_mid", scalar3d_mid, Pa, grid_name); // total pressure + add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + add_field("pseudo_density_dry", scalar3d_mid, Pa, grid_name); + add_field("qv", scalar3d_mid, q_unit, grid_name,"tracers"); // specific humidity + add_field("qi", scalar3d_mid, q_unit, grid_name,"tracers"); // ice wet mixing ratio + add_field("ni", scalar3d_mid, n_unit, grid_name,"tracers"); // ice number mixing ratio // droplet activation can alter cloud liquid and number mixing ratios - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // cloud liquid wet mixing ratio - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - - add_field("phis", scalar2d_layout_col, m2 / s2, grid_name); - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); // cloud fraction - add_field("pbl_height", scalar2d_layout_col, m, - grid_name); // planetary boundary layer height + add_field("qc", scalar3d_mid, q_unit, grid_name,"tracers"); // cloud liquid wet mixing ratio + add_field("nc", scalar3d_mid, n_unit, grid_name,"tracers"); // cloud liquid wet number mixing ratio + + add_field("phis", scalar2d, m2 / s2, grid_name); + add_field("cldfrac_tot", scalar3d_mid, nondim,grid_name); // cloud fraction + add_field("pbl_height", scalar2d, m,grid_name); // planetary boundary layer height + // shortwave aerosol scattering asymmetry parameter [unitless] - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("aero_g_sw", scalar3d_swband, nondim, grid_name); + // shortwave aerosol single-scattering albedo [unitless] - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, - grid_name); + add_field("aero_ssa_sw", scalar3d_swband, nondim,grid_name); + // shortwave aerosol extinction optical depth - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, - grid_name); + add_field("aero_tau_sw", scalar3d_swband, nondim, grid_name); + //longwave aerosol extinction optical depth [unitless] - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); + add_field("aero_tau_lw", scalar3d_lwband, nondim, grid_name); - add_field("aodvis", scalar2d_layout_col, nondim, grid_name); + add_field("aodvis", scalar2d, nondim, grid_name); // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); + add_field(int_nmr_field_name, scalar3d_mid, n_unit,grid_name, "tracers"); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_mid, q_unit,grid_name, "tracers"); } } } @@ -121,15 +103,13 @@ void MAMOptics::set_grids( for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } } @@ -137,8 +117,7 @@ void MAMOptics::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } } diff --git a/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp b/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp index d1d54e97fda0..a62954fe553a 100644 --- a/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp +++ b/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp @@ -48,6 +48,13 @@ inline void set_parameters_table( constexpr int refindex_im = mam4::modal_aer_opt::refindex_im; constexpr int coef_number = mam4::modal_aer_opt::coef_number; + auto make_layout = [](const std::vector& extents, + const std::vector& names) + { + std::vector tags(extents.size(),CMP); + return FieldLayout(tags,extents,names); + }; + auto refindex_real_lw_host = view_2d_host("refrtablw_real_host", nlwbands, refindex_real); auto refindex_im_lw_host = @@ -78,24 +85,21 @@ inline void set_parameters_table( aerosol_optics_host_data.extpsw_host = extpsw_host; aerosol_optics_host_data.abspsw_host = abspsw_host; - FieldLayout scalar_refindex_real_lw_layout{{LWBND, NREFINDEX_REAL}, - {nlwbands, refindex_real}}; - - FieldLayout scalar_refindex_im_lw_layout{{LWBND, NREFINDEX_IM}, - {nlwbands, refindex_im}}; - FieldLayout scalar_refindex_real_sw_layout{{SWBND, NREFINDEX_REAL}, - {nswbands, refindex_real}}; - - FieldLayout scalar_refindex_im_sw_layout{{SWBND, NREFINDEX_IM}, - {nswbands, refindex_im}}; - - FieldLayout scalar_absplw_layout{ - {LWBND, MODE, NREFINDEX_IM, NREFINDEX_REAL, NCOEF_NUMBER}, - {nlwbands, 1, refindex_im, refindex_real, coef_number}}; + auto refindex_real_lw_layout = make_layout({nlwbands, refindex_real}, + {"lwband","refindex_real"}); + auto refindex_im_lw_layout = make_layout({nlwbands, refindex_im}, + {"lwband","refindex_im"}); + auto refindex_real_sw_layout = make_layout({nswbands, refindex_real}, + {"swband","refindex_real"}); + auto refindex_im_sw_layout = make_layout({nswbands, refindex_im}, + {"swband","refindex_im"}); + auto absplw_layout = + make_layout({nlwbands, 1, refindex_im, refindex_real, coef_number}, + {"lwband","mode","refindex_im","refindex_real","coef_number"}); // use also for extpsw, abspsw - FieldLayout scalar_asmpsw_layout{ - {SWBND, MODE, NREFINDEX_IM, NREFINDEX_REAL, NCOEF_NUMBER}, - {nswbands, 1, refindex_im, refindex_real, coef_number}}; + auto asmpsw_layout = + make_layout({nswbands, 1, refindex_im, refindex_real, coef_number}, + {"swband","mode","refindex_im","refindex_real","coef_number"}); rrtmg_params.set( "Field Names", @@ -124,14 +128,14 @@ inline void set_parameters_table( host_views["abspsw"] = view_1d_host(abspsw_host.data(), abspsw_host.size()); - layouts.emplace("refindex_real_lw", scalar_refindex_real_lw_layout); - layouts.emplace("refindex_im_lw", scalar_refindex_im_lw_layout); - layouts.emplace("refindex_real_sw", scalar_refindex_real_sw_layout); - layouts.emplace("refindex_im_sw", scalar_refindex_im_sw_layout); - layouts.emplace("absplw", scalar_absplw_layout); - layouts.emplace("asmpsw", scalar_asmpsw_layout); - layouts.emplace("extpsw", scalar_asmpsw_layout); - layouts.emplace("abspsw", scalar_asmpsw_layout); + layouts.emplace("refindex_real_lw", refindex_real_lw_layout); + layouts.emplace("refindex_im_lw", refindex_im_lw_layout); + layouts.emplace("refindex_real_sw", refindex_real_sw_layout); + layouts.emplace("refindex_im_sw", refindex_im_sw_layout); + layouts.emplace("absplw", absplw_layout); + layouts.emplace("asmpsw", asmpsw_layout); + layouts.emplace("extpsw", asmpsw_layout); + layouts.emplace("abspsw", asmpsw_layout); } // KOKKOS_INLINE_FUNCTION inline void read_rrtmg_table( @@ -287,17 +291,13 @@ inline void read_water_refindex(const std::string &table_filename, // defines layouts std::map layouts_water; - FieldLayout scalar_refindex_water_sw_layout{{SWBND}, {nswbands}}; - FieldLayout scalar_refindex_water_lw_layout{{LWBND}, {nlwbands}}; - - layouts_water.emplace("refindex_im_water_sw", - scalar_refindex_water_sw_layout); - layouts_water.emplace("refindex_real_water_sw", - scalar_refindex_water_sw_layout); - layouts_water.emplace("refindex_im_water_lw", - scalar_refindex_water_lw_layout); - layouts_water.emplace("refindex_real_water_lw", - scalar_refindex_water_lw_layout); + FieldLayout refindex_water_sw_layout{{CMP}, {nswbands}, {"swband"}}; + FieldLayout refindex_water_lw_layout{{CMP}, {nlwbands}, {"lwband"}}; + + layouts_water.emplace("refindex_im_water_sw", refindex_water_sw_layout); + layouts_water.emplace("refindex_real_water_sw",refindex_water_sw_layout); + layouts_water.emplace("refindex_im_water_lw", refindex_water_lw_layout); + layouts_water.emplace("refindex_real_water_lw",refindex_water_lw_layout); // create a object to read data AtmosphereInput refindex_water(params, grid, host_views_water, layouts_water); @@ -345,8 +345,8 @@ inline void set_refindex_names(std::string surname, ekat::ParameterList ¶ms, host_views[refindex_real_lw] = view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = view_1d_host(refindex_im_lw, nlwbands); - FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; - FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; + FieldLayout scalar_refindex_sw_layout{{CMP}, {nswbands}, {"swband"}}; + FieldLayout scalar_refindex_lw_layout{{CMP}, {nlwbands}, {"lwband"}}; layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 3fefd1c84123..e6b7ac5be7df 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -58,15 +58,14 @@ Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) void Nudging::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; - using namespace ShortFieldTagsNames; m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; - FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; + FieldLayout scalar3d_layout_mid = m_grid->get_3d_scalar_layout_mid(); + FieldLayout horiz_wind_layout = m_grid->get_3d_vector_layout_mid(2); constexpr int ps = 1; auto Q = kg/kg; diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 16811885a3b9..bc5625e690cd 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -54,8 +54,6 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ auto molmol = mol/mol; molmol.set_string("mol/mol"); - using namespace ShortFieldTagsNames; - m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); m_ncol = m_grid->get_num_local_dofs(); @@ -78,49 +76,49 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ // Set up dimension layouts m_nswgpts = m_params.get("nswgpts",112); m_nlwgpts = m_params.get("nlwgpts",128); - FieldLayout scalar2d_layout { {COL }, {m_ncol } }; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_ncol,m_nlay} }; - FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_ncol,m_nlay+1} }; - FieldLayout scalar3d_swband_layout { {COL,SWBND,LEV}, {m_ncol, m_nswbands, m_nlay} }; - FieldLayout scalar3d_lwband_layout { {COL,LWBND,LEV}, {m_ncol, m_nlwbands, m_nlay} }; - FieldLayout scalar3d_swgpts_layout { {COL,SWGPT,LEV}, {m_ncol, m_nswgpts, m_nlay} }; - FieldLayout scalar3d_lwgpts_layout { {COL,LWGPT,LEV}, {m_ncol, m_nlwgpts, m_nlay} }; + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); + FieldLayout scalar3d_swband = m_grid->get_3d_vector_layout(true,m_nswbands,"swband"); + FieldLayout scalar3d_lwband = m_grid->get_3d_vector_layout(true,m_nlwbands,"lwband"); + FieldLayout scalar3d_swgpts = m_grid->get_3d_vector_layout(true,m_nswgpts,"swgpt"); + FieldLayout scalar3d_lwgpts = m_grid->get_3d_vector_layout(true,m_nlwgpts,"lwgpt"); // Set required (input) fields here - add_field("p_mid" , scalar3d_layout_mid, Pa, grid_name); - add_field("p_int", scalar3d_layout_int, Pa, grid_name); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); - add_field("sfc_alb_dir_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dir_nir", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_nir", scalar2d_layout, nondim, grid_name); - add_field("qc", scalar3d_layout_mid, kgkg, grid_name); - add_field("nc", scalar3d_layout_mid, 1/kg, grid_name); - add_field("qi", scalar3d_layout_mid, kgkg, grid_name); - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); - add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name); - add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name); - add_field("qv",scalar3d_layout_mid,kgkg,grid_name); - add_field("surf_lw_flux_up",scalar2d_layout,W/(m*m),grid_name); + add_field("p_mid" , scalar3d_mid, Pa, grid_name); + add_field("p_int", scalar3d_int, Pa, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dir_nir", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_nir", scalar2d, nondim, grid_name); + add_field("qc", scalar3d_mid, kgkg, grid_name); + add_field("nc", scalar3d_mid, 1/kg, grid_name); + add_field("qi", scalar3d_mid, kgkg, grid_name); + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + add_field("eff_radius_qc", scalar3d_mid, micron, grid_name); + add_field("eff_radius_qi", scalar3d_mid, micron, grid_name); + add_field("qv",scalar3d_mid,kgkg,grid_name); + add_field("surf_lw_flux_up",scalar2d,W/(m*m),grid_name); // Set of required gas concentration fields for (auto& it : m_gas_names) { // Add gas VOLUME mixing ratios (moles of gas / moles of air; what actually gets input to RRTMGP) if (it == "o3") { // o3 is read from file, or computed by chemistry - add_field(it + "_volume_mix_ratio", scalar3d_layout_mid, molmol, grid_name); + add_field(it + "_volume_mix_ratio", scalar3d_mid, molmol, grid_name); } else { // the rest are computed by RRTMGP from prescribed surface values // NOTE: this may change at some point - add_field(it + "_volume_mix_ratio", scalar3d_layout_mid, molmol, grid_name); + add_field(it + "_volume_mix_ratio", scalar3d_mid, molmol, grid_name); } } // Required aerosol optical properties from SPA m_do_aerosol_rad = m_params.get("do_aerosol_rad",true); if (m_do_aerosol_rad) { - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); - add_field("aero_g_sw" , scalar3d_swband_layout, nondim, grid_name); - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); + add_field("aero_tau_sw", scalar3d_swband, nondim, grid_name); + add_field("aero_ssa_sw", scalar3d_swband, nondim, grid_name); + add_field("aero_g_sw" , scalar3d_swband, nondim, grid_name); + add_field("aero_tau_lw", scalar3d_lwband, nondim, grid_name); } // Whether we do extra clean/clear sky calculations @@ -128,47 +126,47 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ m_extra_clnsky_diag = m_params.get("extra_clnsky_diag", false); // Set computed (output) fields - add_field("T_mid" , scalar3d_layout_mid, K , grid_name); - add_field("SW_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clnclrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clnclrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clnclrsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clrsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clnsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clnsky_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("SW_clnsky_flux_dn_dir", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_clnclrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_clnclrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_clrsky_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_clrsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_clnsky_flux_up", scalar3d_layout_int, Wm2, grid_name); - add_field("LW_clnsky_flux_dn", scalar3d_layout_int, Wm2, grid_name); - add_field("rad_heating_pdel", scalar3d_layout_mid, Pa*K/s, grid_name); + add_field("T_mid" , scalar3d_mid, K , grid_name); + add_field("SW_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("SW_flux_up", scalar3d_int, Wm2, grid_name); + add_field("SW_flux_dn_dir", scalar3d_int, Wm2, grid_name); + add_field("LW_flux_up", scalar3d_int, Wm2, grid_name); + add_field("LW_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("SW_clnclrsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("SW_clnclrsky_flux_up", scalar3d_int, Wm2, grid_name); + add_field("SW_clnclrsky_flux_dn_dir", scalar3d_int, Wm2, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_int, Wm2, grid_name); + add_field("SW_clrsky_flux_dn_dir", scalar3d_int, Wm2, grid_name); + add_field("SW_clnsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("SW_clnsky_flux_up", scalar3d_int, Wm2, grid_name); + add_field("SW_clnsky_flux_dn_dir", scalar3d_int, Wm2, grid_name); + add_field("LW_clnclrsky_flux_up", scalar3d_int, Wm2, grid_name); + add_field("LW_clnclrsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("LW_clrsky_flux_up", scalar3d_int, Wm2, grid_name); + add_field("LW_clrsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("LW_clnsky_flux_up", scalar3d_int, Wm2, grid_name); + add_field("LW_clnsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("rad_heating_pdel", scalar3d_mid, Pa*K/s, grid_name); // Cloud properties added as computed fields for diagnostic purposes - add_field("cldlow" , scalar2d_layout, nondim, grid_name); - add_field("cldmed" , scalar2d_layout, nondim, grid_name); - add_field("cldhgh" , scalar2d_layout, nondim, grid_name); - add_field("cldtot" , scalar2d_layout, nondim, grid_name); + add_field("cldlow" , scalar2d, nondim, grid_name); + add_field("cldmed" , scalar2d, nondim, grid_name); + add_field("cldhgh" , scalar2d, nondim, grid_name); + add_field("cldtot" , scalar2d, nondim, grid_name); // 0.67 micron and 10.5 micron optical depth (needed for COSP) - add_field("dtau067" , scalar3d_layout_mid, nondim, grid_name); - add_field("dtau105" , scalar3d_layout_mid, nondim, grid_name); - add_field("sunlit" , scalar2d_layout , nondim, grid_name); - add_field("cldfrac_rad" , scalar3d_layout_mid, nondim, grid_name); + add_field("dtau067" , scalar3d_mid, nondim, grid_name); + add_field("dtau105" , scalar3d_mid, nondim, grid_name); + add_field("sunlit" , scalar2d , nondim, grid_name); + add_field("cldfrac_rad" , scalar3d_mid, nondim, grid_name); // Cloud-top diagnostics following AeroCOM recommendation - add_field("T_mid_at_cldtop", scalar2d_layout, K, grid_name); - add_field("p_mid_at_cldtop", scalar2d_layout, Pa, grid_name); - add_field("cldfrac_ice_at_cldtop", scalar2d_layout, nondim, grid_name); - add_field("cldfrac_liq_at_cldtop", scalar2d_layout, nondim, grid_name); - add_field("cldfrac_tot_at_cldtop", scalar2d_layout, nondim, grid_name); - add_field("cdnc_at_cldtop", scalar2d_layout, 1 / (m * m * m), grid_name); - add_field("eff_radius_qc_at_cldtop", scalar2d_layout, micron, grid_name); - add_field("eff_radius_qi_at_cldtop", scalar2d_layout, micron, grid_name); + add_field("T_mid_at_cldtop", scalar2d, K, grid_name); + add_field("p_mid_at_cldtop", scalar2d, Pa, grid_name); + add_field("cldfrac_ice_at_cldtop", scalar2d, nondim, grid_name); + add_field("cldfrac_liq_at_cldtop", scalar2d, nondim, grid_name); + add_field("cldfrac_tot_at_cldtop", scalar2d, nondim, grid_name); + add_field("cdnc_at_cldtop", scalar2d, 1 / (m * m * m), grid_name); + add_field("eff_radius_qc_at_cldtop", scalar2d, micron, grid_name); + add_field("eff_radius_qi_at_cldtop", scalar2d, micron, grid_name); // Translation of variables from EAM // -------------------------------------------------------------- @@ -181,19 +179,19 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ // netsw sfc_flux_sw_net net (down - up) SW flux at surface // flwds sfc_flux_lw_dn downwelling LW flux at surface // -------------------------------------------------------------- - add_field("sfc_flux_dir_nir", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_dir_vis", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_dif_nir", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_dif_vis", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_sw_net" , scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_lw_dn" , scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_dir_nir", scalar2d, Wm2, grid_name); + add_field("sfc_flux_dir_vis", scalar2d, Wm2, grid_name); + add_field("sfc_flux_dif_nir", scalar2d, Wm2, grid_name); + add_field("sfc_flux_dif_vis", scalar2d, Wm2, grid_name); + add_field("sfc_flux_sw_net" , scalar2d, Wm2, grid_name); + add_field("sfc_flux_lw_dn" , scalar2d, Wm2, grid_name); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d_layout, kg/m2/s, grid_name); - add_field("water_flux", scalar2d_layout, m/s, grid_name); - add_field("ice_flux", scalar2d_layout, m/s, grid_name); - add_field("heat_flux", scalar2d_layout, W/m2, grid_name); + add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); + add_field("water_flux", scalar2d, m/s, grid_name); + add_field("ice_flux", scalar2d, m/s, grid_name); + add_field("heat_flux", scalar2d, W/m2, grid_name); } } // RRTMGPRadiation::set_grids diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index f244a2262f9c..89f521e3d966 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -40,23 +40,23 @@ void SPA::set_grids(const std::shared_ptr grids_manager) // Define the different field layouts that will be used for this process // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; - FieldLayout scalar2d_layout { {COL}, {m_num_cols} }; - FieldLayout scalar1d_layout_mid { {LEV}, {m_num_levs} }; + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + FieldLayout scalar1d_mid = m_grid->get_vertical_layout(true); // Use VAR field tag for gases for now; consider adding a tag? - FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {m_num_cols, m_nswbands, m_num_levs} }; - FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {m_num_cols, m_nlwbands, m_num_levs} }; + FieldLayout scalar3d_swband = m_grid->get_3d_vector_layout(true,m_nswbands,"swband"); + FieldLayout scalar3d_lwband = m_grid->get_3d_vector_layout(true,m_nlwbands,"lwband"); // Set of fields used strictly as input constexpr int ps = Spack::n; - add_field("p_mid" , scalar3d_layout_mid, Pa, grid_name, ps); + add_field("p_mid" , scalar3d_mid, Pa, grid_name, ps); // Set of fields used strictly as output - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name, ps); - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name, ps); - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name, ps); - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name, ps); + add_field("nccn", scalar3d_mid, 1/kg, grid_name, ps); + add_field("aero_g_sw", scalar3d_swband, nondim, grid_name, ps); + add_field("aero_ssa_sw", scalar3d_swband, nondim, grid_name, ps); + add_field("aero_tau_sw", scalar3d_swband, nondim, grid_name, ps); + add_field("aero_tau_lw", scalar3d_lwband, nondim, grid_name, ps); // We can already create some of the spa structures @@ -75,8 +75,8 @@ void SPA::set_grids(const std::shared_ptr grids_manager) SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop!=nullptr); // Grab a sw and lw field from the horiz interp, and check sw/lw dim against what we hardcoded in this class - auto nswbands_data = SPAHorizInterp->get_src_field(4).get_header().get_identifier().get_layout().dim(SWBND); - auto nlwbands_data = SPAHorizInterp->get_src_field(5).get_header().get_identifier().get_layout().dim(LWBND); + auto nswbands_data = SPAHorizInterp->get_src_field(4).get_header().get_identifier().get_layout().dim("swband"); + auto nlwbands_data = SPAHorizInterp->get_src_field(5).get_header().get_identifier().get_layout().dim("lwband"); EKAT_REQUIRE_MSG (nswbands_data==m_nswbands, "Error! Spa data file has a different number of sw bands than the model.\n" " - spa data swbands: " + std::to_string(nswbands_data) + "\n" diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 748983006a19..d9547953d1ed 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -114,8 +114,8 @@ create_horiz_remapper ( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto layout_ccn3 = tgt_grid->get_3d_scalar_layout(true); - const auto layout_sw = tgt_grid->get_3d_vector_layout(true,SWBND,nswbands); - const auto layout_lw = tgt_grid->get_3d_vector_layout(true,LWBND,nlwbands); + const auto layout_sw = tgt_grid->get_3d_vector_layout(true,nswbands,"swband"); + const auto layout_lw = tgt_grid->get_3d_vector_layout(true,nlwbands,"lwband"); const auto nondim = ekat::units::Units::nondimensional(); Field ps (FieldIdentifier("PS", layout_2d, nondim,tgt_grid->name())); @@ -491,8 +491,8 @@ ::update_spa_data_from_file( const int ncols = sw_layout.dim(COL); const int nlevs = sw_layout.dim(LEV); - const int nswbands = sw_layout.dim(SWBND); - const int nlwbands = lw_layout.dim(LWBND); + const int nswbands = sw_layout.dim("swband"); + const int nlwbands = lw_layout.dim("lwband"); Kokkos::deep_copy(spa_input.PS,ps); Kokkos::fence(); diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 0b07e265e666..e5086b567406 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -33,20 +33,6 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - // Added for RRTMGP, TODO: Revisit this approach, is there a better way than adding more field tags? - Gases, - ShortWaveBand, - ShortWaveGpoint, - LongWaveBand, - LongWaveGpoint, - IsccpTau, - IsccpPrs, - // - MAM_NumModes, - MAM_NumRefIndexReal, - MAM_NumRefIndexImag, - MAM_NumCoefficients, - MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -63,21 +49,6 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - // Added for rrtmgp - see TODO item above - constexpr auto NGAS = FieldTag::Gases; - constexpr auto SWBND = FieldTag::ShortWaveBand; - constexpr auto LWBND = FieldTag::LongWaveBand; - constexpr auto SWGPT = FieldTag::ShortWaveGpoint; - constexpr auto LWGPT = FieldTag::LongWaveGpoint; - constexpr auto ISCCPTAU = FieldTag::IsccpTau; - constexpr auto ISCCPPRS = FieldTag::IsccpPrs; - constexpr auto NMODES = FieldTag::MAM_NumModes; - // - constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; - constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; - - constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; - constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -108,43 +79,6 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - // Added for rrtmgp - see TODO item above - case FieldTag::Gases: - name = "ngas"; - break; - case FieldTag::ShortWaveBand: - name = "swband"; - break; - case FieldTag::ShortWaveGpoint: - name = "swgpt"; - break; - case FieldTag::LongWaveBand: - name = "lwband"; - break; - case FieldTag::LongWaveGpoint: - name = "lwgpt"; - break; - case FieldTag::IsccpTau: - name = "ISCCPTAU"; - break; - case FieldTag::IsccpPrs: - name = "ISCCPPRS"; - break; - case FieldTag::MAM_NumModes: - name = "num_modes"; - break; - case FieldTag::MAM_NumRefIndexReal: - name = "refindex_real"; - break; - case FieldTag::MAM_NumRefIndexImag: - name = "refindex_im"; - break; - case FieldTag::MAM_NumCoefficients: - name = "coef_number"; - break; - case FieldTag::MAM_NumModesInFile: - name = "mode"; - break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 69fa64880e82..a1336ae346fb 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -26,10 +26,10 @@ TEST_CASE("field_layout", "") { FieldLayout fl1 ({COL},{1}); FieldLayout fl2 ({COL,CMP},{1,1}); - FieldLayout fl3 ({COL,SWBND,LWBND},{1,1,1}); + FieldLayout fl3 ({COL,CMP,CMP},{1,3,4}); FieldLayout fl4 ({COL,LEV},{1,1}); FieldLayout fl5 ({COL,CMP,LEV},{1,1,1}); - FieldLayout fl6 ({COL,ISCCPTAU,ISCCPPRS,ILEV},{1,1,1,1}); + FieldLayout fl6 ({COL,CMP,CMP,ILEV},{1,5,6,1}); REQUIRE (fl1.type()==LayoutType::Scalar2D); REQUIRE (fl2.type()==LayoutType::Vector2D); @@ -59,10 +59,10 @@ TEST_CASE("field_layout", "") { REQUIRE (fl2.get_vector_dim()==1); REQUIRE (fl5.get_vector_dim()==1); - REQUIRE (fl3.get_tensor_tags()==TVec{SWBND,LWBND}); - REQUIRE (fl6.get_tensor_tags()==TVec{ISCCPTAU,ISCCPPRS}); - REQUIRE (fl3.get_tensor_dims()==IVec{1,2}); - REQUIRE (fl6.get_tensor_dims()==IVec{1,2}); + REQUIRE (fl3.get_tensor_tags()==TVec{CMP,CMP}); + REQUIRE (fl6.get_tensor_components_ids()==IVec{1,2}); + REQUIRE (fl3.get_tensor_dims()==IVec{3,4}); + REQUIRE (fl6.get_tensor_dims()==IVec{5,6}); } TEST_CASE("field_identifier", "") { From 63fb3f85f9f4928459dd30ee03bf53476cd131a6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 29 Mar 2024 11:38:06 -0600 Subject: [PATCH 059/476] EAMxx: use grid to retrieve layouts, rather than building manually --- .../homme/eamxx_homme_process_interface.cpp | 45 ++++++++------- .../src/dynamics/homme/tests/dyn_grid_io.cpp | 32 +++++------ .../eamxx_mam_optics_process_interface.cpp | 1 - .../eamxx_ml_correction_process_interface.cpp | 24 ++++---- .../eamxx_nudging_process_interface.cpp | 4 +- .../shoc/eamxx_shoc_process_interface.cpp | 57 +++++++++---------- .../tms/eamxx_tms_process_interface.cpp | 28 ++++----- .../src/share/io/tests/output_restart.cpp | 47 ++++++--------- 8 files changed, 113 insertions(+), 125 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index 6bf400f2bce7..b550348fa49b 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -108,7 +108,6 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m using namespace ShortFieldTagsNames; using namespace ekat::units; - using FL = FieldLayout; constexpr int NGP = HOMMEXX_NP; constexpr int NTL = HOMMEXX_NUM_TIME_LEVELS; @@ -119,7 +118,6 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m auto Q = kg/kg; Q.set_string("kg/kg"); - const int ncols = m_phys_grid->get_num_local_dofs(); const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const int nlev_mid = m_dyn_grid->get_num_vertical_levels(); const int nlev_int = nlev_mid+1; @@ -168,18 +166,22 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m // Note: qv is needed to transform T<->Theta const auto& pgn = m_phys_grid->name(); - add_field ("horiz_winds", FL({COL,CMP, LEV},{ncols,2,nlev_mid}),m/s, pgn,N); - add_field ("T_mid", FL({COL, LEV},{ncols, nlev_mid}),K, pgn,N); - add_field("pseudo_density",FL({COL, LEV},{ncols, nlev_mid}),Pa, pgn,N); - add_field("pseudo_density_dry",FL({COL, LEV},{ncols, nlev_mid}),Pa, pgn,N); - add_field ("ps", FL({COL },{ncols }),Pa, pgn); - add_field("qv", FL({COL, LEV},{ncols, nlev_mid}),Q, pgn,"tracers",N); - add_field("phis", FL({COL },{ncols }),m2/s2, pgn); - add_field("p_int", FL({COL, ILEV},{ncols, nlev_int}),Pa, pgn,N); - add_field("p_mid", FL({COL, LEV},{ncols, nlev_mid}),Pa, pgn,N); - add_field("p_dry_int", FL({COL, ILEV},{ncols, nlev_int}),Pa, pgn,N); - add_field("p_dry_mid", FL({COL, LEV},{ncols, nlev_mid}),Pa, pgn,N); - add_field("omega", FL({COL, LEV},{ncols, nlev_mid}),Pa/s, pgn,N); + auto pg_scalar2d = m_phys_grid->get_2d_scalar_layout(); + auto pg_scalar3d_mid = m_phys_grid->get_3d_scalar_layout(true); + auto pg_scalar3d_int = m_phys_grid->get_3d_scalar_layout(false); + auto pg_vector3d_mid = m_phys_grid->get_3d_vector_layout(true,2); + add_field ("horiz_winds", pg_vector3d_mid, m/s, pgn,N); + add_field ("T_mid", pg_scalar3d_mid, K, pgn,N); + add_field("pseudo_density", pg_scalar3d_mid, Pa, pgn,N); + add_field("pseudo_density_dry", pg_scalar3d_mid, Pa, pgn,N); + add_field ("ps", pg_scalar2d , Pa, pgn); + add_field("qv", pg_scalar3d_mid, Q, pgn,"tracers",N); + add_field("phis", pg_scalar2d , m2/s2, pgn); + add_field("p_int", pg_scalar3d_int, Pa, pgn,N); + add_field("p_mid", pg_scalar3d_mid, Pa, pgn,N); + add_field("p_dry_int", pg_scalar3d_int, Pa, pgn,N); + add_field("p_dry_mid", pg_scalar3d_mid, Pa, pgn,N); + add_field("omega", pg_scalar3d_mid, Pa/s, pgn,N); add_group("tracers",pgn,N, Bundling::Required); if (fv_phys_active()) { @@ -188,15 +190,18 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m // doing so may conflict with additional IC mechanisms in the AD, e.g., // init'ing a field to a constant. const auto& rgn = m_cgll_grid->name(); - const auto nc = m_cgll_grid->get_num_local_dofs(); - add_field("horiz_winds", FL({COL,CMP, LEV},{nc,2,nlev_mid}),m/s, rgn,N); - add_field("T_mid", FL({COL, LEV},{nc, nlev_mid}),K, rgn,N); - add_field("ps", FL({COL },{nc }),Pa, rgn); - add_field("phis", FL({COL },{nc }),m2/s2, rgn); + auto rg_scalar2d = m_cgll_grid->get_2d_scalar_layout(); + auto rg_scalar3d_mid = m_cgll_grid->get_3d_scalar_layout(true); + auto rg_vector3d_mid = m_cgll_grid->get_3d_vector_layout(true,2); + + add_field("horiz_winds", rg_vector3d_mid,m/s, rgn,N); + add_field("T_mid", rg_scalar3d_mid,K, rgn,N); + add_field("ps", rg_scalar2d ,Pa, rgn); + add_field("phis", rg_scalar2d ,m2/s2, rgn); add_group("tracers",rgn,N, Bundling::Required, DerivationType::Import, "tracers", pgn); fv_phys_rrtmgp_active_gases_init(grids_manager); // This is needed for the dp_ref init in initialize_homme_state. - add_field("pseudo_density",FL({COL, LEV},{nc, nlev_mid}),Pa, rgn,N); + add_field("pseudo_density",rg_scalar3d_mid,Pa, rgn,N); } // Dynamics grid states diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index a2534c049c26..bb09daa1c49a 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -37,8 +37,6 @@ TEST_CASE("dyn_grid_io") { using namespace scream; using namespace ShortFieldTagsNames; - constexpr int np = HOMMEXX_NP; - constexpr int nlev = HOMMEXX_NUM_PHYSICAL_LEV; ekat::Comm comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. @@ -68,28 +66,26 @@ TEST_CASE("dyn_grid_io") auto phys_grid = gm->get_grid("Physics GLL"); // Local counters - const int nelems = get_num_local_elems_f90(); - const int ncols = phys_grid->get_num_local_dofs(); - EKAT_REQUIRE_MSG(ncols>0, "Internal test error! Fix dyn_grid_io, please.\n"); - EKAT_REQUIRE_MSG(nelems>0, "Internal test error! Fix dyn_grid_io, please.\n"); + EKAT_REQUIRE_MSG(phys_grid->get_num_local_dofs()>0, "Internal test error! Fix dyn_grid_io, please.\n"); + EKAT_REQUIRE_MSG(get_num_local_elems_f90()>0, "Internal test error! Fix dyn_grid_io, please.\n"); // Create physics and dynamics Fields - FieldLayout layout_dyn_1({EL,GP,GP,LEV},{nelems,np,np,nlev}); - FieldLayout layout_dyn_2({EL,CMP,GP,GP,LEV},{nelems,2,np,np,nlev}); - FieldLayout layout_dyn_3({EL,GP,GP},{nelems,np,np}); + auto dyn_scalar3d_mid = dyn_grid->get_3d_scalar_layout(true); + auto dyn_vector3d_mid = dyn_grid->get_3d_vector_layout(true,2); + auto dyn_scalar2d = dyn_grid->get_2d_scalar_layout(); - FieldLayout layout_phys_1({COL,LEV},{ncols,nlev}); - FieldLayout layout_phys_2({COL,CMP,LEV},{ncols,2,nlev}); - FieldLayout layout_phys_3({COL},{ncols}); + auto phys_scalar3d_mid = phys_grid->get_3d_scalar_layout(true); + auto phys_vector3d_mid = phys_grid->get_3d_vector_layout(true,2); + auto phys_scalar2d = phys_grid->get_2d_scalar_layout(); auto nondim = ekat::units::Units::nondimensional(); - FieldIdentifier fid_dyn_1 ("field_1",layout_dyn_1,nondim,dyn_grid->name()); - FieldIdentifier fid_dyn_2 ("field_2",layout_dyn_2,nondim,dyn_grid->name()); - FieldIdentifier fid_dyn_3 ("field_3",layout_dyn_3,nondim,dyn_grid->name()); + FieldIdentifier fid_dyn_1 ("field_1",dyn_scalar3d_mid,nondim,dyn_grid->name()); + FieldIdentifier fid_dyn_2 ("field_2",dyn_vector3d_mid,nondim,dyn_grid->name()); + FieldIdentifier fid_dyn_3 ("field_3",dyn_scalar2d ,nondim,dyn_grid->name()); - FieldIdentifier fid_phys_1 ("field_1",layout_phys_1,nondim,phys_grid->name()); - FieldIdentifier fid_phys_2 ("field_2",layout_phys_2,nondim,phys_grid->name()); - FieldIdentifier fid_phys_3 ("field_3",layout_phys_3,nondim,phys_grid->name()); + FieldIdentifier fid_phys_1 ("field_1",phys_scalar3d_mid,nondim,phys_grid->name()); + FieldIdentifier fid_phys_2 ("field_2",phys_vector3d_mid,nondim,phys_grid->name()); + FieldIdentifier fid_phys_3 ("field_3",phys_scalar2d ,nondim,phys_grid->name()); // The starting FM auto fm_dyn = std::make_shared (dyn_grid); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index f25bb8cf7281..d818890a9b46 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -38,7 +38,6 @@ void MAMOptics::set_grids( nlwbands_ = mam4::modal_aer_opt::nlwbands; // number of longwave bands // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; // Define aerosol optics fields computed by this process. auto nondim = Units::nondimensional(); diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 2802e1a590d6..e04f7a0ca3dc 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -35,20 +35,20 @@ void MLCorrection::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - FieldLayout scalar2d_layout{ {COL}, {m_num_cols}}; - FieldLayout scalar3d_layout_mid{{COL, LEV}, {m_num_cols, m_num_levs}}; - FieldLayout scalar3d_layout_int{{COL, ILEV}, {m_num_cols, m_num_levs+1}}; - FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout_mid(); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout_int(); + FieldLayout vector3d_mid = m_grid->get_3d_vector_layout_mid(2); if (not m_ML_correction_unit_test) { const auto m2 = m*m; const auto s2 = s*s; auto Wm2 = W / m / m; auto nondim = m/m; - add_field("phis", scalar2d_layout, m2/s2, grid_name); - add_field("SW_flux_dn", scalar3d_layout_int, Wm2, grid_name, ps); - add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_flux_sw_net", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_lw_dn", scalar2d_layout, Wm2, grid_name); + add_field("phis", scalar2d, m2/s2, grid_name); + add_field("SW_flux_dn", scalar3d_int, Wm2, grid_name, ps); + add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); + add_field("sfc_flux_sw_net", scalar2d, Wm2, grid_name); + add_field("sfc_flux_lw_dn", scalar2d, Wm2, grid_name); m_lat = m_grid->get_geometry_data("lat"); m_lon = m_grid->get_geometry_data("lon"); } @@ -59,9 +59,9 @@ void MLCorrection::set_grids( * is adapting the infrastructure to allow for a generic "add_field" call * to be used here which we can then setup using the m_fields_ml_output_variables variable */ - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("T_mid", scalar3d_mid, K, grid_name, ps); + add_field("qv", scalar3d_mid, Q, grid_name, "tracers", ps); + add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); /* ----------------------- WARNING --------------------------------*/ add_group("tracers", grid_name, 1, Bundling::Required); } diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index e6b7ac5be7df..40925c9edd1a 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -64,8 +64,8 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - FieldLayout scalar3d_layout_mid = m_grid->get_3d_scalar_layout_mid(); - FieldLayout horiz_wind_layout = m_grid->get_3d_vector_layout_mid(2); + FieldLayout scalar3d_layout_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout horiz_wind_layout = m_grid->get_3d_vector_layout(true,2); constexpr int ps = 1; auto Q = kg/kg; diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index f0c368549400..2504c7ddd8af 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -36,20 +36,19 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; // Layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_layout_col{ {COL}, {m_num_cols} }; + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); // Layout for surf_mom_flux - FieldLayout surf_mom_flux_layout { {COL, CMP}, {m_num_cols, 2} }; + FieldLayout vector2d = m_grid->get_2d_vector_layout(2); // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; - FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_num_cols,m_num_levs+1} }; + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); // Layout for horiz_wind field - FieldLayout horiz_wind_layout { {COL,CMP,LEV}, {m_num_cols,2,m_num_levs} }; + FieldLayout vector3d_mid = m_grid->get_3d_vector_layout(true,2); // Define fields needed in SHOC. // Note: shoc_main is organized by a set of 5 structures, variables below are organized @@ -61,46 +60,46 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids const auto s2 = s*s; // These variables are needed by the interface, but not actually passed to shoc_main. - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name, ps); - add_field("surf_sens_flux", scalar2d_layout_col, W/m2, grid_name); - add_field("surf_mom_flux", surf_mom_flux_layout, N/m2, grid_name); + add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); + add_field("surf_sens_flux", scalar2d , W/m2, grid_name); + add_field("surf_mom_flux", vector2d , N/m2, grid_name); - add_field("surf_evap", scalar2d_layout_col, kg/m2/s, grid_name); - add_field ("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field ("qv", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("surf_evap", scalar2d , kg/m2/s, grid_name); + add_field ("T_mid", scalar3d_mid, K, grid_name, ps); + add_field ("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { - add_field("surf_drag_coeff_tms", scalar2d_layout_col, kg/s/m2, grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); } // Input variables - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_int", scalar3d_layout_int, Pa, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("phis", scalar2d_layout_col, m2/s2, grid_name, ps); + add_field("p_mid", scalar3d_mid, Pa, grid_name, ps); + add_field("p_int", scalar3d_int, Pa, grid_name, ps); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); + add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d_layout_col, m, grid_name); - add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d_layout_col, kg/m2/s, grid_name); - add_field("water_flux", scalar2d_layout_col, m/s, grid_name); - add_field("ice_flux", scalar2d_layout_col, m/s, grid_name); - add_field("heat_flux", scalar2d_layout_col, W/m2, grid_name); + add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); + add_field("water_flux", scalar2d, m/s, grid_name); + add_field("ice_flux", scalar2d, m/s, grid_name); + add_field("heat_flux", scalar2d, W/m2, grid_name); } } diff --git a/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp index c06b87a966b1..87e145917987 100644 --- a/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp +++ b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp @@ -42,22 +42,22 @@ void TurbulentMountainStress::set_grids(const std::shared_ptrget_num_vertical_levels(); // Number of levels per column // Add required/computed fields - FieldLayout scalar2d_layout{ {COL}, {m_ncols} }, - scalar3d_midpoint_layout{ {COL, LEV}, {m_ncols, m_nlevs} }, - vector2d_layout{ {COL, CMP}, {m_ncols, 2} }, - vector3d_midpoint_layout{ {COL, CMP, LEV}, {m_ncols, 2, m_nlevs} }; + auto scalar2d = m_grid->get_2d_scalar_layout(); + auto vector2d = m_grid->get_2d_vector_layout(2); + auto scalar3d_mid = m_grid->get_3d_scalar_layout(true); + auto vector3d_mid = m_grid->get_3d_vector_layout(true,2); constexpr int ps = Spack::n; - add_field("horiz_winds", vector3d_midpoint_layout, m/s, grid_name, ps); - add_field("T_mid", scalar3d_midpoint_layout, K, grid_name, ps); - add_field("p_mid", scalar3d_midpoint_layout, Pa, grid_name, ps); - add_field("pseudo_density", scalar3d_midpoint_layout, Pa, grid_name, ps); - add_field("qv", scalar3d_midpoint_layout, Qunit, grid_name, "tracers", ps); - add_field("sgh30", scalar2d_layout, m, grid_name); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - - add_field("surf_drag_coeff_tms", scalar2d_layout, kg/s/m2, grid_name); - add_field("wind_stress_tms", vector2d_layout, N/m2, grid_name); + add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); + add_field("T_mid", scalar3d_mid, K, grid_name, ps); + add_field("p_mid", scalar3d_mid, Pa, grid_name, ps); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); + add_field("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("sgh30", scalar2d , m, grid_name); + add_field("landfrac", scalar2d , nondim, grid_name); + + add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); + add_field("wind_stress_tms", vector2d, N/m2, grid_name); } // ========================================================================================= diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 62676450efee..7e59574b1b9b 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -160,36 +160,25 @@ get_test_fm(const std::shared_ptr& grid) using namespace ShortFieldTagsNames; using namespace ekat::units; - using FL = FieldLayout; using FR = FieldRequest; using SL = std::list; // Create a fm auto fm = std::make_shared(grid); - const int num_lcols = grid->get_num_local_dofs(); - const int num_levs = grid->get_num_vertical_levels(); - - // Create some fields for this fm - std::vector tag_h = {COL}; - std::vector tag_v = {LEV}; - std::vector tag_2d = {COL,LEV}; - std::vector tag_3d = {COL,CMP,LEV}; - std::vector tag_bnd = {COL,SWBND,LEV}; - - std::vector dims_h = {num_lcols}; - std::vector dims_v = {num_levs}; - std::vector dims_2d = {num_lcols,num_levs}; - std::vector dims_3d = {num_lcols,2,num_levs}; - std::vector dims_bnd = {num_lcols,3,num_levs}; + auto scalar_1d = grid->get_vertical_layout(true); + auto scalar_2d = grid->get_2d_scalar_layout(); + auto scalar_3d = grid->get_3d_scalar_layout(true); + auto vector_3d = grid->get_3d_vector_layout(true,2); + auto rad_vector_3d = grid->get_3d_vector_layout(true,3,"SWBND"); const std::string& gn = grid->name(); - FieldIdentifier fid1("field_1",FL{tag_h,dims_h},m,gn); - FieldIdentifier fid2("field_2",FL{tag_v,dims_v},kg,gn); - FieldIdentifier fid3("field_3",FL{tag_2d,dims_2d},kg/m,gn); - FieldIdentifier fid4("field_4",FL{tag_3d,dims_3d},kg/m,gn); - FieldIdentifier fid5("field_5",FL{tag_bnd,dims_bnd},m*m,gn); + FieldIdentifier fid1("field_1",scalar_2d, m, gn); + FieldIdentifier fid2("field_2",scalar_1d, kg, gn); + FieldIdentifier fid3("field_3",scalar_3d, kg/m,gn); + FieldIdentifier fid4("field_4",vector_3d, kg/m,gn); + FieldIdentifier fid5("field_5",rad_vector_3d,m*m, gn); // Register fields with fm fm->registration_begins(); @@ -284,14 +273,14 @@ void time_advance (const FieldManager& fm, for (int i=0; i Date: Thu, 30 Nov 2023 17:39:32 -0700 Subject: [PATCH 060/476] EAMxx: fix how grid returns then name of a tag The grid must now query the layout, in case it stores a special name for that dimension --- components/eamxx/src/share/grid/abstract_grid.hpp | 5 +++-- components/eamxx/src/share/io/scorpio_input.cpp | 7 +++---- components/eamxx/src/share/io/scorpio_output.cpp | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index f969a36b2d04..26cde401f4ff 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -180,8 +180,9 @@ class AbstractGrid : public ekat::enable_shared_from_this virtual bool check_valid_lid_to_idx () const { return true; } void reset_field_tag_name (const FieldTag t, const std::string& s) { m_special_tag_names[t] = s; } - std::string get_dim_name (const FieldTag t) const { - return m_special_tag_names.count(t)==1 ? m_special_tag_names.at(t) : e2str(t); + std::string get_dim_name (const FieldLayout& lt, const int idim) const { + const auto t = lt.tag(idim); + return m_special_tag_names.count(t)==1 ? m_special_tag_names.at(t) : lt.names()[idim]; } // This member is used mostly by IO: if a field exists on multiple grids diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index af0886588699..245b1e05b0e4 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -409,10 +409,9 @@ AtmosphereInput::get_vec_of_dims(const FieldLayout& layout) dims_names.reserve(layout.rank()); for (int i=0; iget_dim_name(layout,i)); if (t==CMP) { - dims_names.push_back("dim" + std::to_string(layout.dim(i))); - } else { - dims_names.push_back(m_io_grid->get_dim_name(t)); + dims_names.back() += std::to_string(layout.dim(i)); } } @@ -428,7 +427,7 @@ get_io_decomp(const FieldLayout& layout) std::vector range(layout.rank()); std::iota(range.begin(),range.end(),0); auto tag_and_dim = [&](int i) { - return m_io_grid->get_dim_name(layout.tag(i)) + + return m_io_grid->get_dim_name(layout,i) + std::to_string(layout.dim(i)); }; diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 3943e59d5da1..c7484422a73f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -758,7 +758,7 @@ void AtmosphereOutput::register_dimensions(const std::string& name) // check tag against m_dims map. If not in there, then add it. const auto& tags = layout.tags(); const auto& dims = layout.dims(); - auto tag_name = m_io_grid->get_dim_name(tags[i]); + auto tag_name = m_io_grid->get_dim_name(layout,i); if (tags[i]==CMP) { tag_name += std::to_string(dims[i]); } @@ -855,7 +855,7 @@ void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const Field if (m_track_avg_cnt) { std::string avg_cnt_name = "avg_count" + avg_cnt_suffix; for (int ii=0; iiget_dim_name(layout.tag(ii)); + auto tag_name = m_io_grid->get_dim_name(layout,ii); avg_cnt_name += "_" + tag_name; } if (std::find(m_avg_cnt_names.begin(),m_avg_cnt_names.end(),avg_cnt_name)==m_avg_cnt_names.end()) { @@ -919,7 +919,7 @@ register_variables(const std::string& filename, std::vector range(layout.rank()); std::iota(range.begin(),range.end(),0); auto tag_and_dim = [&](int i) { - return m_io_grid->get_dim_name(layout.tag(i)) + + return m_io_grid->get_dim_name(layout,i) + std::to_string(layout.dim(i)); }; @@ -934,7 +934,7 @@ register_variables(const std::string& filename, auto set_vec_of_dims = [&](const FieldLayout& layout) { std::vector vec_of_dims; for (int i=0; iget_dim_name(layout.tag(i)); + auto tag_name = m_io_grid->get_dim_name(layout,i); if (layout.tag(i)==CMP) { tag_name += std::to_string(layout.dim(i)); } From 71296d9ca4b42451b864aeaaa06e93d130c5b878 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 Nov 2023 18:34:59 -0700 Subject: [PATCH 061/476] EAMxx: add dim extent to IO dim name only if tag name is 'dim' --- components/eamxx/src/share/io/scorpio_input.cpp | 3 +-- components/eamxx/src/share/io/scorpio_output.cpp | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 245b1e05b0e4..35bb0d86647a 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -408,9 +408,8 @@ AtmosphereInput::get_vec_of_dims(const FieldLayout& layout) std::vector dims_names; dims_names.reserve(layout.rank()); for (int i=0; iget_dim_name(layout,i)); - if (t==CMP) { + if (dims_names.back()=="dim") { dims_names.back() += std::to_string(layout.dim(i)); } } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index c7484422a73f..3c3ad925a68e 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -759,7 +759,7 @@ void AtmosphereOutput::register_dimensions(const std::string& name) const auto& tags = layout.tags(); const auto& dims = layout.dims(); auto tag_name = m_io_grid->get_dim_name(layout,i); - if (tags[i]==CMP) { + if (tag_name=="dim") { tag_name += std::to_string(dims[i]); } auto tag_loc = m_dims.find(tag_name); @@ -935,7 +935,7 @@ register_variables(const std::string& filename, std::vector vec_of_dims; for (int i=0; iget_dim_name(layout,i); - if (layout.tag(i)==CMP) { + if (tag_name=="dim") { tag_name += std::to_string(layout.dim(i)); } vec_of_dims.push_back(tag_name); // Add dimensions string to vector of dims. From 90a507cb54cfde58237fc0ae2ac0074fd6867c8c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 29 Mar 2024 17:54:29 -0600 Subject: [PATCH 062/476] EAMxx: fix a few bugs --- components/eamxx/src/share/field/field_layout.cpp | 15 +++++++++++---- components/eamxx/src/share/grid/abstract_grid.cpp | 8 ++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 14cc20551257..b917d8179363 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -26,7 +26,6 @@ FieldLayout::FieldLayout (const std::vector& tags, , m_tags (tags) , m_names(names) , m_dims (dims) - , m_extents ("",tags.size()) { EKAT_REQUIRE_MSG (dims.size()==tags.size(), "Error! Tags and dims vectors dimensions mismatch.\n" @@ -111,7 +110,7 @@ std::vector FieldLayout::get_tensor_dims () const { } std::vector FieldLayout::get_tensor_tags () const { - auto idx = get_tensor_dims(); + auto idx = get_tensor_components_ids(); return {m_tags[idx[0]], m_tags[idx[1]]}; } @@ -186,12 +185,13 @@ FieldLayout& FieldLayout::reset_dim (const int idim, const int extent) { EKAT_REQUIRE_MSG(idim>=0 && idim // 2) + // 3) if ( is_lev_tag(tags[1]) && (tags[0]==CMP || tags[0]==TL)) { m_type = LayoutType::Vector3D; } else if (tags[0]==TL && tags[1]==CMP ) { m_type = LayoutType::Tensor2D; + } else if (tags[0]==CMP and tags[1]==CMP) { + m_type = LayoutType::Tensor2D; } break; case 3: - // The only supported scenario is: + // The only supported scenarios are: // 1) + // 2) if ( tags[0]==TL && tags[1]==CMP && is_lev_tag(tags[2])) { m_type = LayoutType::Tensor3D; + } else if ( tags[0]==CMP && tags[1]==CMP && is_lev_tag(tags[2])) { + m_type = LayoutType::Tensor3D; } + break; default: // If nothing worked, this type is not recognized diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 4b66a23d6de5..0ff9688c191a 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -186,12 +186,16 @@ is_valid_layout (const FieldLayout& layout) const return layout.congruent(get_vertical_layout(midpoints)); case LayoutType::Scalar2D: return layout.congruent(get_2d_scalar_layout()); - case LayoutType::Scalar3D: - return layout.congruent(get_3d_scalar_layout(midpoints)); case LayoutType::Vector2D: return layout.congruent(get_2d_vector_layout(layout.get_vector_dim())); + case LayoutType::Tensor2D: + return layout.congruent(get_2d_tensor_layout(layout.get_tensor_dims())); + case LayoutType::Scalar3D: + return layout.congruent(get_3d_scalar_layout(midpoints)); case LayoutType::Vector3D: return layout.congruent(get_3d_vector_layout(midpoints,layout.get_vector_dim())); + case LayoutType::Tensor3D: + return layout.congruent(get_3d_tensor_layout(midpoints,layout.get_tensor_dims())); default: // Anything else is probably not ok return false; From 821445287ca90c912a57c9ae361544fcabb7cee9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 1 Apr 2024 10:18:02 -0600 Subject: [PATCH 063/476] EAMxx: add missing include --- components/eamxx/src/share/field/field_tag.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index e5086b567406..673208d6ae86 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -4,6 +4,7 @@ #include "ekat/ekat_assert.hpp" #include +#include namespace scream { From ae3ca60e0f41b8bc97a507b19fe59b9948fecc13 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 1 Apr 2024 18:41:29 -0600 Subject: [PATCH 064/476] EAMxx: allow to call strip_dim on layout without checking if tag is in the layout This allows to strip dim "only if it's in the layout" without checks --- components/eamxx/src/share/field/field_layout.cpp | 9 ++++++--- components/eamxx/src/share/field/field_layout.hpp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index b917d8179363..ada4de5342d2 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -114,11 +114,14 @@ std::vector FieldLayout::get_tensor_tags () const { return {m_tags[idx[0]], m_tags[idx[1]]}; } -FieldLayout& FieldLayout::strip_dim (const FieldTag tag) { +FieldLayout& FieldLayout::strip_dim (const FieldTag tag, const bool throw_if_not_found) { auto it = ekat::find(m_tags,tag); - // Check if found - EKAT_REQUIRE_MSG(it!=m_tags.end(), "Error! Tag '" + e2str(tag) + "' not found.\n"); + if (it==m_tags.end()) { + // Check if found + EKAT_REQUIRE_MSG(not throw_if_not_found, "Error! Tag '" + e2str(tag) + "' not found.\n"); + return *this; + } // Check only one tag (no ambiguity) EKAT_REQUIRE_MSG(ekat::count(m_tags,tag)==1, diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index c088ee33b0b5..66c1c1da9ff3 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -122,7 +122,7 @@ class FieldLayout { std::vector get_tensor_tags () const; // Returns a copy of this layout with a given dimension stripped - FieldLayout& strip_dim (const FieldTag tag); + FieldLayout& strip_dim (const FieldTag tag, const bool throw_if_not_found = true); FieldLayout& strip_dim (const int idim); FieldLayout& append_dim (const FieldTag t, const int extent); FieldLayout& append_dim (const FieldTag t, const int extent, const std::string& name); From b933cf25cb56134c1b3c06edf0da8e43cd417df6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 1 Apr 2024 18:31:48 -0600 Subject: [PATCH 065/476] EAMxx: fix layout related stuff in remappers --- .../share/grid/remap/abstract_remapper.hpp | 4 +-- .../grid/remap/horiz_interp_remapper_base.cpp | 18 +++++++--- .../grid/remap/horiz_interp_remapper_base.hpp | 3 +- .../share/grid/remap/vertical_remapper.hpp | 36 +++++-------------- 4 files changed, 26 insertions(+), 35 deletions(-) diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp index af80889add1b..68367eeb07f1 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp @@ -163,8 +163,8 @@ class AbstractRemapper virtual bool compatible_layouts (const layout_type& src, const layout_type& tgt) const { - // By default, the only compatible layouts are identical - return src==tgt; + // By default, the only compatible layouts are congruent + return src.congruent(tgt); } virtual bool is_valid_src_layout (const layout_type& layout) const { diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 1707ab4dcdc2..71f5a52a50e8 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -121,24 +121,34 @@ create_layout (const FieldLayout& fl_in, auto fl_out = FieldLayout::invalid(); using namespace ShortFieldTagsNames; const bool midpoints = fl_in.has_tag(LEV); + std::vector tdims_names; + std::string vdim_name; switch (fl_in.type()) { case LayoutType::Scalar2D: fl_out = grid->get_2d_scalar_layout(); break; case LayoutType::Vector2D: - fl_out = grid->get_2d_vector_layout(fl_in.get_vector_dim()); + vdim_name = fl_in.names()[fl_in.get_vector_component_idx()]; + fl_out = grid->get_2d_vector_layout(fl_in.get_vector_dim(),vdim_name); break; case LayoutType::Tensor2D: - fl_out = grid->get_2d_tensor_layout(fl_in.get_tensor_dims()); + for (auto idx : fl_in.get_tensor_components_ids()) { + tdims_names.push_back(fl_in.names()[idx]); + } + fl_out = grid->get_2d_tensor_layout(fl_in.get_tensor_dims(),tdims_names); break; case LayoutType::Scalar3D: fl_out = grid->get_3d_scalar_layout(midpoints); break; case LayoutType::Vector3D: - fl_out = grid->get_3d_vector_layout(midpoints,fl_in.get_vector_dim()); + vdim_name = fl_in.names()[fl_in.get_vector_component_idx()]; + fl_out = grid->get_3d_vector_layout(midpoints,fl_in.get_vector_dim(),vdim_name); break; case LayoutType::Tensor3D: - fl_out = grid->get_3d_tensor_layout(midpoints,fl_in.get_tensor_dims()); + for (auto idx : fl_in.get_tensor_components_ids()) { + tdims_names.push_back(fl_in.names()[idx]); + } + fl_out = grid->get_3d_tensor_layout(midpoints,fl_in.get_tensor_dims(),tdims_names); default: EKAT_ERROR_MSG ("Layout not supported by HorizInterpRemapperBase:\n" " - layout: " + to_string(fl_in) + "\n"); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp index df8ef0b4ab73..40e53fb3ceae 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp @@ -34,7 +34,8 @@ class HorizInterpRemapperBase : public AbstractRemapper // be 0 src/tgt gids on some ranks, which means src/tgt.dim(0)=0. using namespace ShortFieldTagsNames; - return src.clone().strip_dim(COL)==tgt.clone().strip_dim(COL); + // Use congruence, since we don't really care about dimension names, only tags/extents + return src.clone().strip_dim(COL).congruent(tgt.clone().strip_dim(COL)); } protected: diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp index e3e5c0781047..f9950fad0f3a 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp @@ -36,36 +36,16 @@ class VerticalRemapper : public AbstractRemapper bool compatible_layouts (const layout_type& src, const layout_type& tgt) const override { - // Same type of layout, and same sizes except for possibly the first one - // Note: we can't do tgt.size()/tgt.dim(0), since there may be 0 tgt gids - // on some ranks, which means tgt.dim(0)=0. - // Note: for vertical remapping we strip out the LEV or ILEV dimension when - // calculating the size. - auto src_dims = src.dims(); - auto tgt_dims = tgt.dims(); - auto src_size = src.rank(); - auto tgt_size = tgt.rank(); + // Strip the LEV/ILEV tags, and check if they are the same + // Also, check rank compatibility, in case one has LEV/ILEV and the other doesn't + // NOTE: tgt layouts always use LEV (not ILEV), while src can have ILEV or LEV. using namespace ShortFieldTagsNames; - if (src.has_tag(LEV) || src.has_tag(ILEV)) { - // Then we ignore the last dimension: - src_size -= 1; - } - if (tgt.has_tag(LEV) || tgt.has_tag(ILEV)) { - // Then we ignore the last dimension: - tgt_size -= 1; - } - - int src_col_size = 1; - for (int i=0; i Date: Tue, 16 Apr 2024 20:35:16 -0600 Subject: [PATCH 066/476] EAMxx: remove some warnings in MAM folder --- .../eamxx_mam_microphysics_process_interface.cpp | 5 ----- .../mam/impl/compute_o3_column_density.cpp | 13 ++++++------- .../src/physics/mam/impl/gas_phase_chemistry.cpp | 16 +++++----------- .../eamxx/src/physics/mam/mam_coupling.hpp | 3 --- 4 files changed, 11 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7bfe86ef713f..4ad61b96fe07 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -223,9 +223,6 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { dry_atm_.w_updraft = buffer_.w_updraft; dry_atm_.z_surf = 0.0; // FIXME: for now - const auto& tracers = get_group_out("tracers"); - const auto& tracers_info = tracers.m_info; - // perform any initialization work if (run_type==RunType::Initial) { } @@ -324,7 +321,6 @@ void MAMMicrophysics::run_impl(const double dt) { // fetch column-specific atmosphere state data auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); auto z_iface = ekat::subview(dry_atm_.z_iface, icol); - Real z_surf = dry_atm_.z_surf; Real phis = dry_atm_.phis(icol); // set surface state data @@ -365,7 +361,6 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { constexpr int num_modes = mam4::AeroConfig::num_modes(); - constexpr int num_aero_ids = mam4::AeroConfig::num_aerosol_ids(); constexpr int gas_pcnst = mam_coupling::gas_pcnst(); constexpr int nqtendbb = mam_coupling::nqtendbb(); diff --git a/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp index b31537eef83d..ceb992bc8863 100644 --- a/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp +++ b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp @@ -3,19 +3,18 @@ namespace scream::impl { KOKKOS_INLINE_FUNCTION void compute_o3_column_density(const ThreadTeam& team, const haero::Atmosphere& atm, const mam4::Prognostics &progs, ColumnView o3_col_dens) { - constexpr int nabscol = mam4::gas_chemistry::nabscol; // number of absorbing densities constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" - constexpr Real mwdry = 1.0/haero::Constants::molec_weight_dry_air; + // constexpr Real mwdry = 1.0/haero::Constants::molec_weight_dry_air; Real o3_col_deltas[mam4::nlev+1] = {}; // o3 column density above model [1/cm^2] // NOTE: if we need o2 column densities, set_ub_col and setcol must be changed Kokkos::parallel_for(Kokkos::TeamThreadRange(team, atm.num_levels()), [&](const int k) { - Real temp = atm.temperature(k); - Real pmid = atm.pressure(k); + // Real temp = atm.temperature(k); + // Real pmid = atm.pressure(k); Real pdel = atm.hydrostatic_dp(k); - Real qv = atm.vapor_mixing_ratio(k); + // Real qv = atm.vapor_mixing_ratio(k); // ... map incoming mass mixing ratios to working array Real q[gas_pcnst], qqcw[gas_pcnst]; @@ -23,8 +22,8 @@ void compute_o3_column_density(const ThreadTeam& team, const haero::Atmosphere& // ... set atmosphere mean mass to the molecular weight of dry air // and compute water vapor vmr - Real mbar = mwdry; - Real h2ovmr = mam4::conversions::vmr_from_mmr(qv, mbar); + // Real mbar = mwdry; + // Real h2ovmr = mam4::conversions::vmr_from_mmr(qv, mbar); // ... Xform from mmr to vmr Real vmr[gas_pcnst], vmrcw[gas_pcnst]; diff --git a/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp index 98e33b3c2b7c..5401fa5e3da2 100644 --- a/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp +++ b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp @@ -265,14 +265,13 @@ void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real const Real extfrc[mam4::gas_chemistry::extcnt], // in Real q[mam4::gas_chemistry::gas_pcnst], // VMRs, inout Real invariants[mam4::gas_chemistry::nfs]) { // out - constexpr Real rga = 1.0/haero::Constants::gravity; - constexpr Real m2km = 0.01; // converts m -> km + // constexpr Real rga = 1.0/haero::Constants::gravity; + // constexpr Real m2km = 0.01; // converts m -> km // The following things are chemical mechanism dependent! See mam4xx/src/mam4xx/gas_chem_mechanism.hpp) constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species constexpr int rxntot = mam4::gas_chemistry::rxntot; // number of chemical reactions constexpr int extcnt = mam4::gas_chemistry::extcnt; // number of species with external forcing - constexpr int nabscol = mam4::gas_chemistry::nabscol; // number of "absorbing column densities" constexpr int indexm = 0; // FIXME: index of total atm density in invariants array constexpr int phtcnt = mam4::mo_photo::phtcnt; // number of photolysis reactions @@ -296,7 +295,6 @@ void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real // "so4_a3", "bc_a3", "pom_a3", "soa_a3", "mom_a3", // "num_a3", "pom_a4", "bc_a4", "mom_a4", "num_a4"}; constexpr int ndx_h2so4 = 2; - constexpr int o3_ndx = 0; // std::string extfrc_list[] = {"SO2", "so4_a1", "so4_a2", "pom_a4", "bc_a4", // "num_a1", "num_a2", "num_a3", "num_a4", "SOAG"}; constexpr int synoz_ndx = -1; @@ -305,16 +303,13 @@ void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real // FIXME: For now, we fix the zenith angle. At length, we need to compute it // FIXME: from EAMxx's current set of orbital parameters, which requires some // FIXME: conversation with the EAMxx team. - Real zen_angle = 0.0; // [deg] // xform geopotential height from m to km and pressure from Pa to mb - Real zsurf = rga * phis; - Real zintr = m2km * zi; - Real zmid = m2km * (zm + zsurf); - Real zint = m2km * (zi + zsurf); + // Real zsurf = rga * phis; + // Real zmid = m2km * (zm + zsurf); // ... compute the column's invariants - Real h2ovmr = q[0]; + // Real h2ovmr = q[0]; // setinv(invariants, temp, h2ovmr, q, pmid); FIXME: not ported yet // ... set rates for "tabular" and user specified reactions @@ -349,7 +344,6 @@ void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real //sethet(het_rates, pmid, zmid, phis, temp, cmfdqr, prain, nevapr, delt, // invariants[indexm], q); - int ltrop_sol = 0; // apply solver to all levels // save h2so4 before gas phase chem (for later new particle nucleation) Real del_h2so4_gasprod = q[ndx_h2so4]; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index e8a2d127c0d2..e4a8a3dff9c8 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -829,7 +829,6 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; @@ -862,7 +861,6 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; @@ -901,7 +899,6 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } From 2eed7293e556dfadfc065131f33a37ea4461aa27 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 11:57:10 -0600 Subject: [PATCH 067/476] EAMxx: turn to_string free function into FieldLayout method --- .../eamxx/src/diagnostics/field_at_height.cpp | 4 ++-- .../eamxx/src/diagnostics/field_at_level.cpp | 4 ++-- .../diagnostics/field_at_pressure_level.cpp | 4 ++-- .../homme/physics_dynamics_remapper.cpp | 4 ++-- .../eamxx/src/share/field/field_impl.hpp | 6 +++--- .../eamxx/src/share/field/field_layout.cpp | 19 +++++++------------ .../eamxx/src/share/field/field_layout.hpp | 3 ++- .../eamxx/src/share/field/field_manager.cpp | 2 +- .../eamxx/src/share/field/field_utils.hpp | 6 +++--- .../src/share/field/field_utils_impl.hpp | 9 ++++----- .../eamxx/src/share/grid/abstract_grid.cpp | 4 ++-- .../share/grid/remap/abstract_remapper.cpp | 8 ++++---- .../share/grid/remap/coarsening_remapper.cpp | 8 ++++---- .../share/grid/remap/do_nothing_remapper.hpp | 4 ++-- .../grid/remap/horiz_interp_remapper_base.cpp | 8 ++++---- .../share/grid/remap/identity_remapper.hpp | 4 ++-- .../share/grid/remap/vertical_remapper.cpp | 8 ++++---- .../eamxx/src/share/io/scorpio_input.cpp | 4 ++-- .../eamxx/src/share/io/scorpio_output.cpp | 2 +- .../share/tests/coarsening_remapper_tests.cpp | 4 ++-- .../eamxx/src/share/tests/field_utils.cpp | 6 +++--- .../share/tests/vertical_remapper_tests.cpp | 6 +++--- 22 files changed, 61 insertions(+), 66 deletions(-) diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index 39be5b1442e0..3982ed9a3ac9 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -89,12 +89,12 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, "Error! Field rank not supported by FieldAtHeight.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtHeight diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" " - field name : " + fid.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); // Figure out the z value m_z_suffix = tag==LEV ? "_mid" : "_int"; diff --git a/components/eamxx/src/diagnostics/field_at_level.cpp b/components/eamxx/src/diagnostics/field_at_level.cpp index 7c36a508a8e9..87ecf7ad9103 100644 --- a/components/eamxx/src/diagnostics/field_at_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_level.cpp @@ -33,12 +33,12 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>1 && layout.rank()<=6, "Error! Field rank not supported by FieldAtLevel.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" " - field name : " + fid.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); // Figure out the level const auto& location = m_params.get("vertical_location"); diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index 27e97cc90c0a..132de63971fe 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -66,12 +66,12 @@ initialize_impl (const RunType /*run_type*/) EKAT_REQUIRE_MSG (layout.rank()>=2 && layout.rank()<=3, "Error! Field rank not supported by FieldAtPressureLevel.\n" " - field name: " + fid.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); const auto tag = layout.tags().back(); EKAT_REQUIRE_MSG (tag==LEV || tag==ILEV, "Error! FieldAtPressureLevel diagnostic expects a layout ending with 'LEV'/'ILEV' tag.\n" " - field name : " + fid.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); // All good, create the diag output FieldIdentifier d_fid (m_diag_name,layout.clone().strip_dim(tag),fid.get_units(),fid.get_grid_name()); diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp index 9ff24886cf46..531a513fc708 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp @@ -68,7 +68,7 @@ create_src_layout (const FieldLayout& tgt_layout) const { EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[PhysicsDynamicsRemapper] Error! Input target layout is not valid for this remapper.\n" - " - input layout: " + to_string(tgt_layout)); + " - input layout: " + tgt_layout.to_string()); auto tags = tgt_layout.tags(); auto dims = tgt_layout.dims(); @@ -102,7 +102,7 @@ create_tgt_layout (const FieldLayout& src_layout) const { EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), "[PhysicsDynamicsRemapper] Error! Input source layout is not valid for this remapper.\n" - " - input layout: " + to_string(src_layout)); + " - input layout: " + src_layout.to_string()); auto tags = src_layout.tags(); auto dims = src_layout.dims(); diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 0efe46771422..1964622d0057 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -73,7 +73,7 @@ Field (const identifier_type& id, // Create an unmanaged dev view, and its host mirror const auto view_dim = alloc_prop.get_alloc_size(); char* data = reinterpret_cast(view_d.data()); - std::cout << "fl: " << to_string(fl) << "\n" + std::cout << "fl: " << fl.to_string() << "\n" << "view dim: " << view_dim << "\n"; m_data.d_view = decltype(m_data.d_view)(data,view_dim); m_data.h_view = Kokkos::create_mirror_view(m_data.d_view); @@ -593,8 +593,8 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) "Error! Incompatible layouts for update_field.\n" " - x name: " + x.name() + "\n" " - y name: " + name() + "\n" - " - x layout: " + to_string(x_l) + "\n" - " - y layout: " + to_string(y_l) + "\n"); + " - x layout: " + x_l.to_string() + "\n" + " - y layout: " + y_l.to_string() + "\n"); using device_t = typename Field::get_device; using exec_space = typename device_t::execution_space; diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index ada4de5342d2..563e9d857bc6 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -78,7 +78,7 @@ FieldTag FieldLayout::get_vector_tag () const { std::vector FieldLayout::get_tensor_components_ids () const { EKAT_REQUIRE_MSG (is_tensor_layout(), "Error! 'get_tensor_dims' available only for tensor layouts.\n" - " Current layout: " + to_string(*this) + "\n" + " Current layout: " + to_string() + "\n" " Layout type : " + e2str(m_type) + "\n"); using namespace ShortFieldTagsNames; @@ -95,7 +95,7 @@ std::vector FieldLayout::get_tensor_components_ids () const { EKAT_REQUIRE_MSG (idx.size()==2, "Error! Could not find a two tensor tags in the layout.\n" - " - layout: " + to_string(*this) + "\n" + " - layout: " + to_string() + "\n" " - detected tags indices: " + ekat::join(idx,",") + "\n"); return idx; @@ -294,18 +294,13 @@ void FieldLayout::compute_type () { } } -std::string to_string (const FieldLayout& layout) +std::string FieldLayout::to_string () const { - if (layout.rank()==0) { - return "<>()"; - } - std::string s; - s += "<" + e2str(layout.tags()[0]); - for (int dim=1; dim"; + s += "(" + ekat::join(m_dims,",") + ")"; return s; } diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 66c1c1da9ff3..8f4deb7eb7d7 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -136,6 +136,8 @@ class FieldLayout { FieldLayout& rename_dim (const int idim, const std::string& n); FieldLayout& rename_dim (const FieldTag tag, const std::string& n); FieldLayout& reset_dim (const int idim, const int extent); + // For printing purposes + std::string to_string () const; protected: void compute_type (); @@ -151,7 +153,6 @@ class FieldLayout { }; bool operator== (const FieldLayout& fl1, const FieldLayout& fl2); -std::string to_string (const FieldLayout& l); // ========================== IMPLEMENTATION ======================= // diff --git a/components/eamxx/src/share/field/field_manager.cpp b/components/eamxx/src/share/field/field_manager.cpp index 1ea13c8e8dd0..4db9fd0fa15e 100644 --- a/components/eamxx/src/share/field/field_manager.cpp +++ b/components/eamxx/src/share/field/field_manager.cpp @@ -736,7 +736,7 @@ void FieldManager::add_field (const Field& f) { "Error! Input field to 'add_field' has a layout not compatible with the stored grid.\n" " - input field name : " + f.name() + "\n" " - field manager grid: " + m_grid->name() + "\n" - " - input field layout: " + to_string(f.get_header().get_identifier().get_layout()) + "\n"); + " - input field layout: " + f.get_header().get_identifier().get_layout().to_string() + "\n"); EKAT_REQUIRE_MSG (not has_field(f.name()), "Error! The method 'add_field' requires the input field to not be already existing.\n" " - field name: " + f.get_header().get_identifier().name() + "\n"); diff --git a/components/eamxx/src/share/field/field_utils.hpp b/components/eamxx/src/share/field/field_utils.hpp index b58b60f40bfb..8f977a7caa1b 100644 --- a/components/eamxx/src/share/field/field_utils.hpp +++ b/components/eamxx/src/share/field/field_utils.hpp @@ -87,7 +87,7 @@ void perturb (const Field& f, "Error! Trying to perturb field \""+f.name()+"\", but field " "does not have LEV or ILEV as last dimension.\n" " - field name: " + f.name() + "\n" - " - field layout: " + to_string(fl) + "\n"); + " - field layout: " + fl.to_string() + "\n"); if (fl.has_tag(COL)) { // If field has a column dimension, it should be the first dimension @@ -95,7 +95,7 @@ void perturb (const Field& f, "Error! Trying to perturb field \""+f.name()+"\", but field " "does not have COL as first dimension.\n" " - field name: " + f.name() + "\n" - " - field layout: " + to_string(fl) + "\n"); + " - field layout: " + fl.to_string() + "\n"); const auto& dof_gids_fl = dof_gids.get_header().get_identifier().get_layout(); EKAT_REQUIRE_MSG(dof_gids_fl.dim(0) == fl.dim(COL), @@ -103,7 +103,7 @@ void perturb (const Field& f, "perturbed field's column dimension.\n" " - dof_gids dim: " + std::to_string(dof_gids_fl.dim(0)) + "\n" " - field name: " + f.name() + "\n" - " - field layout: " + to_string(fl) + "\n"); + " - field layout: " + fl.to_string() + "\n"); EKAT_REQUIRE_MSG(dof_gids.data_type() == DataType::IntType, "Error! DoF GIDs field must have \"int\" as data type.\n"); } diff --git a/components/eamxx/src/share/field/field_utils_impl.hpp b/components/eamxx/src/share/field/field_utils_impl.hpp index 58e333f6cc20..222ff1bb49ed 100644 --- a/components/eamxx/src/share/field/field_utils_impl.hpp +++ b/components/eamxx/src/share/field/field_utils_impl.hpp @@ -742,7 +742,7 @@ void print_field_hyperslab (const Field& f, f.sync_to_host(); const int rank = layout.rank(); - out << " " << f.name() << to_string(orig_layout) << "\n\n"; + out << " " << f.name() << orig_layout.to_string() << "\n\n"; switch (rank) { case 0: { @@ -830,8 +830,7 @@ void print_field_hyperslab (const Field& f, EKAT_ERROR_MSG ( "Unsupported rank in print_field_hyperslab.\n" " - field name : " + f.name() + "\n" - " - field layout (upon slicing): " + to_string(layout) + "\n"); - + " - field layout (upon slicing): " + layout.to_string() + "\n"); } } else { auto tag = tags[curr_idx]; @@ -841,14 +840,14 @@ void print_field_hyperslab (const Field& f, EKAT_REQUIRE_MSG (it!=layout.tags().end(), "Error! Something went wrong while slicing field.\n" " - field name : " + f.name() + "\n" - " - field layout: " + to_string(layout) + "\n" + " - field layout: " + layout.to_string() + "\n" " - curr tag : " + e2str(tag) + "\n"); auto idim = std::distance(layout.tags().begin(),it); EKAT_REQUIRE_MSG (idim==0 || idim==1, "Error! Cannot subview field for printing.\n" " - field name : " + f.name() + "\n" - " - field layout: " + to_string(layout) + "\n" + " - field layout: " + layout.to_string() + "\n" " - loc tags : <" + ekat::join(tags,",") + ">\n" " - loc indices : (" + ekat::join(indices,",") + ")\n"); diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 0ff9688c191a..033e48cd88eb 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -259,8 +259,8 @@ AbstractGrid::create_geometry_data (const FieldIdentifier& fid) "Error! Cannot create geometry data, since it already exists.\n" " - grid name: " + this->name() + "\n" " - geo data name: " + name + "\n" - " - geo data layout: " + to_string(m_geo_fields.at(name).get_header().get_identifier().get_layout()) + "\n" - " - input layout: " + to_string(fid.get_layout()) + "\n"); + " - geo data layout: " + m_geo_fields.at(name).get_header().get_identifier().get_layout().to_string() + "\n" + " - input layout: " + fid.get_layout().to_string() + "\n"); // Create field and the read only copy as well auto& f = m_geo_fields[name] = Field(fid); diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp index 80df739aa4f4..260d1c2e5f17 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp @@ -33,18 +33,18 @@ register_field (const identifier_type& src, const identifier_type& tgt) { EKAT_REQUIRE_MSG(is_valid_src_layout(src.get_layout()), "Error! Source field has an invalid layout.\n" " - field name : " + src.name() + "\n" - " - field layout: " + to_string(src.get_layout()) + "\n"); + " - field layout: " + src.get_layout().to_string() + "\n"); EKAT_REQUIRE_MSG(is_valid_tgt_layout(tgt.get_layout()), "Error! Source field has an invalid layout.\n" " - field name : " + tgt.name() + "\n" - " - field layout: " + to_string(tgt.get_layout()) + "\n"); + " - field layout: " + tgt.get_layout().to_string() + "\n"); EKAT_REQUIRE_MSG(compatible_layouts(src.get_layout(),tgt.get_layout()), "Error! Source and target layouts are not compatible.\n" " - src name: " + src.name() + "\n" " - tgt name: " + tgt.name() + "\n" - " - src layout: " + to_string(src.get_layout()) + "\n" - " - tgt layout: " + to_string(tgt.get_layout()) + "\n"); + " - src layout: " + src.get_layout().to_string() + "\n" + " - tgt layout: " + tgt.get_layout().to_string() + "\n"); do_register_field (src,tgt); diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index eccbee51c2a4..2ae6f3110e7c 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -133,13 +133,13 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) EKAT_REQUIRE_MSG(f_lt.has_tag(COL) == m_lt.has_tag(COL), "Error! Incompatible field and mask layouts.\n" " - field name: " + src.name() + "\n" - " - field layout: " + to_string(f_lt) + "\n" - " - mask layout: " + to_string(m_lt) + "\n"); + " - field layout: " + f_lt.to_string() + "\n" + " - mask layout: " + m_lt.to_string() + "\n"); EKAT_REQUIRE_MSG(f_lt.has_tag(LEV) == m_lt.has_tag(LEV), "Error! Incompatible field and mask layouts.\n" " - field name: " + src.name() + "\n" - " - field layout: " + to_string(f_lt) + "\n" - " - mask layout: " + to_string(m_lt) + "\n"); + " - field layout: " + f_lt.to_string() + "\n" + " - mask layout: " + m_lt.to_string() + "\n"); } } HorizInterpRemapperBase::do_bind_field(ifield,src,tgt); diff --git a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp index 2bce51a14a95..42dbc3782424 100644 --- a/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/do_nothing_remapper.hpp @@ -54,7 +54,7 @@ class DoNothingRemapper : public AbstractRemapper default: EKAT_ERROR_MSG ( "[DoNothingRemapper] Error! Input target layout is not valid for this remapper.\n" - " - input layout: " + to_string(tgt)); + " - input layout: " + tgt.to_string()); } return src; } @@ -79,7 +79,7 @@ class DoNothingRemapper : public AbstractRemapper default: EKAT_ERROR_MSG ( "[DoNothingRemapper] Error! Input source layout is not valid for this remapper.\n" - " - input layout: " + to_string(src)); + " - input layout: " + src.to_string()); } return tgt; } diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index 71f5a52a50e8..f55e0f56ae8e 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -96,7 +96,7 @@ create_src_layout (const FieldLayout& tgt_layout) const EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[HorizInterpRemapperBase] Error! Input target layout is not valid for this remapper.\n" - " - input layout: " + to_string(tgt_layout)); + " - input layout: " + tgt_layout.to_string()); return create_layout (tgt_layout, m_src_grid); } @@ -109,7 +109,7 @@ create_tgt_layout (const FieldLayout& src_layout) const EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), "[HorizInterpRemapperBase] Error! Input source layout is not valid for this remapper.\n" - " - input layout: " + to_string(src_layout)); + " - input layout: " + src_layout.to_string()); return create_layout (src_layout, m_tgt_grid); } @@ -151,7 +151,7 @@ create_layout (const FieldLayout& fl_in, fl_out = grid->get_3d_tensor_layout(midpoints,fl_in.get_tensor_dims(),tdims_names); default: EKAT_ERROR_MSG ("Layout not supported by HorizInterpRemapperBase:\n" - " - layout: " + to_string(fl_in) + "\n"); + " - layout: " + fl_in.to_string() + "\n"); } return fl_out; } @@ -171,7 +171,7 @@ do_register_field (const identifier_type& src, const identifier_type& tgt) EKAT_REQUIRE_MSG (src.get_layout().has_tag(COL), "Error! Cannot register a field without COL tag in RefiningRemapperP2P.\n" " - field name: " + src.name() + "\n" - " - field layout: " + to_string(src.get_layout()) + "\n"); + " - field layout: " + src.get_layout().to_string() + "\n"); m_src_fields.push_back(field_type(src)); m_tgt_fields.push_back(field_type(tgt)); } diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 047bd90ef835..2106e1b93696 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -49,7 +49,7 @@ class IdentityRemapper : public AbstractRemapper FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override { EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[IdentityRemapper] Error! Input target layout is not valid for this remapper.\n" - " - input layout: " + to_string(tgt_layout)); + " - input layout: " + tgt_layout.to_string()); // Src and tgt grids are the same, so return the input return tgt_layout; @@ -57,7 +57,7 @@ class IdentityRemapper : public AbstractRemapper FieldLayout create_tgt_layout (const FieldLayout& src_layout) const override { EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), "[IdentityRemapper] Error! Input source layout is not valid for this remapper.\n" - " - input layout: " + to_string(src_layout)); + " - input layout: " + src_layout.to_string()); // Src and tgt grids are the same, so return the input return src_layout; diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index cc4ef5f68a58..78d18e086a65 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -80,7 +80,7 @@ create_src_layout (const FieldLayout& tgt_layout) const EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[VerticalRemapper] Error! Input target layout is not valid for this remapper.\n" - " - input layout: " + to_string(tgt_layout)); + " - input layout: " + tgt_layout.to_string()); return create_layout(tgt_layout,m_src_grid); } @@ -92,7 +92,7 @@ create_tgt_layout (const FieldLayout& src_layout) const EKAT_REQUIRE_MSG (is_valid_src_layout(src_layout), "[VerticalRemapper] Error! Input source layout is not valid for this remapper.\n" - " - input layout: " + to_string(src_layout)); + " - input layout: " + src_layout.to_string()); return create_layout(src_layout,m_tgt_grid); } @@ -129,7 +129,7 @@ create_layout (const FieldLayout& fl_in, // that needs to handle a tensor3d quantity, so no need to add it EKAT_ERROR_MSG ( "[VerticalRemapper] Error! Layout not supported by VerticalRemapper.\n" - " - input layout: " + to_string(fl_in) + "\n"); + " - input layout: " + fl_in.to_string() + "\n"); } return fl_out; } @@ -179,7 +179,7 @@ register_vertical_source_field(const Field& src) EKAT_REQUIRE_MSG (vert_tag==LEV or vert_tag==ILEV, "Error! Input vertical level field does not have a vertical level tag at the end.\n" " - field name: " + src.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); if (vert_tag==LEV) { m_src_mid = src; diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 35bb0d86647a..0c04a612a020 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -138,8 +138,8 @@ set_field_manager (const std::shared_ptr& field_mgr) auto lay_curr = field_curr.get_header().get_identifier().get_layout(); auto lay_new = field_new.get_header().get_identifier().get_layout(); EKAT_REQUIRE_MSG(lay_curr==lay_new,"ERROR!! AtmosphereInput::set_field_manager - setting new field manager which has different layout for field " << name <<"\n" - << " Old Layout: " << to_string(lay_curr) << "\n" - << " New Layout: " << to_string(lay_new) << "\n"); + << " Old Layout: " << lay_curr.to_string() << "\n" + << " New Layout: " << lay_new.to_string() << "\n"); } } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 3c3ad925a68e..42f0f38278d4 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1462,7 +1462,7 @@ update_avg_cnt_view(const Field& field, view_1d_dev& dev_view) { EKAT_ERROR_MSG ( "Error! Field rank not not supported by AtmosphereOutput.\n" " - field name: " + field.name() + "\n" - " - field layout: " + to_string(layout) + "\n"); + " - field layout: " + layout.to_string() + "\n"); } } diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index 6201463842c7..7f1a3c20c7b0 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -376,9 +376,9 @@ TEST_CASE("coarsening_remap") auto gtgt = all_gather_field(tgt_f[ifield],comm); const auto& l = gsrc.get_header().get_identifier().get_layout(); - const auto ls = to_string(l); + const auto ls = l.to_string(); std::string dots (30-ls.size(),'.'); - auto msg = " -> Checking field with layout " + to_string(l) + " " + dots; + auto msg = " -> Checking field with layout " + ls + " " + dots; root_print (msg + "\n",comm); bool ok = true; switch (l.type()) { diff --git a/components/eamxx/src/share/tests/field_utils.cpp b/components/eamxx/src/share/tests/field_utils.cpp index b8c901c9f390..1d57d867f602 100644 --- a/components/eamxx/src/share/tests/field_utils.cpp +++ b/components/eamxx/src/share/tests/field_utils.cpp @@ -327,7 +327,7 @@ TEST_CASE ("print_field_hyperslab") { print_field_hyperslab(f,loc_tags,loc_idxs,out); std::stringstream expected; - expected << " f" << to_string(fid.get_layout()) << "\n\n"; + expected << " f" << fid.get_layout().to_string() << "\n\n"; for (int gp1=0; gp1 Checking field with source layout " + to_string(lsrc) +" " + dots + "\n",comm); + print (" -> Checking field with source layout " + ls +" " + dots + "\n",comm); f.sync_to_host(); @@ -383,7 +383,7 @@ TEST_CASE ("vertical_remap") { EKAT_ERROR_MSG ("Unexpected layout.\n"); } - print (" -> Checking field with source layout " + to_string(lsrc) + " " + dots + " OK!\n",comm); + print (" -> Checking field with source layout " + ls + " " + dots + " OK!\n",comm); } print ("check tgt fields ... done!\n",comm); } From 2bcd98066579615c28de5bc20898f9196e137830 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 12:03:03 -0600 Subject: [PATCH 068/476] EAMxx: change behavior of FieldLayout modification methods When providing a FieldTag, *all* matches will be processed --- .../eamxx/src/share/field/field_layout.cpp | 65 ++++++++++++++----- .../eamxx/src/share/field/field_layout.hpp | 13 ++-- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 563e9d857bc6..132423770646 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -114,22 +114,22 @@ std::vector FieldLayout::get_tensor_tags () const { return {m_tags[idx[0]], m_tags[idx[1]]}; } -FieldLayout& FieldLayout::strip_dim (const FieldTag tag, const bool throw_if_not_found) { - auto it = ekat::find(m_tags,tag); - - if (it==m_tags.end()) { - // Check if found - EKAT_REQUIRE_MSG(not throw_if_not_found, "Error! Tag '" + e2str(tag) + "' not found.\n"); - return *this; +FieldLayout& FieldLayout::strip_dim (const FieldTag t, const bool throw_if_not_found) { + bool any_match = false; + for (int i=0; i get_tensor_dims () const; std::vector get_tensor_tags () const; - // Returns a copy of this layout with a given dimension stripped + // Change this layout by adding/removing a dimension or changing its extent/name + // NOTE: the strip_dim/rename_dim/reset_dim overloads with FieldTag will alter *all* + // dimension matching the input tag FieldLayout& strip_dim (const FieldTag tag, const bool throw_if_not_found = true); FieldLayout& strip_dim (const int idim); FieldLayout& append_dim (const FieldTag t, const int extent); FieldLayout& append_dim (const FieldTag t, const int extent, const std::string& name); + FieldLayout& rename_dim (const int idim, const std::string& n); + FieldLayout& rename_dim (const FieldTag tag, const std::string& n, const bool throw_if_not_found = true); + FieldLayout& reset_dim (const int idim, const int extent); + FieldLayout& reset_dim (const FieldTag t, const int extent, const bool throw_if_not_found = true); + FieldLayout clone() const; // NOTE: congruent does not check the tags names. It only checks // rank, m_tags, and m_dims. Use operator== if names are important bool congruent (const FieldLayout& rhs) const; - // Change the name of a dimension - FieldLayout& rename_dim (const int idim, const std::string& n); - FieldLayout& rename_dim (const FieldTag tag, const std::string& n); - FieldLayout& reset_dim (const int idim, const int extent); // For printing purposes std::string to_string () const; From 583f4eee32feee78626f4761ef99775df2f2c9e5 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 12:03:24 -0600 Subject: [PATCH 069/476] EAMxx: allow to change multiple tag names at once in FieldLayout --- components/eamxx/src/share/field/field_layout.cpp | 10 ++++++++++ components/eamxx/src/share/field/field_layout.hpp | 1 + 2 files changed, 11 insertions(+) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 132423770646..12ad26729e63 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -197,6 +197,16 @@ FieldLayout& FieldLayout::rename_dim (const FieldTag t, const std::string& n, co return *this; } + +FieldLayout& FieldLayout::rename_dims (const std::map& new_names) +{ + for (const auto& it : new_names) { + rename_dim(it.first,it.second,false); + } + + return *this; +} + FieldLayout& FieldLayout::reset_dim (const int idim, const int extent) { EKAT_REQUIRE_MSG(idim>=0 && idim& new_names); // Does not throw if not found FieldLayout& reset_dim (const int idim, const int extent); FieldLayout& reset_dim (const FieldTag t, const int extent, const bool throw_if_not_found = true); From 0b1c337e86a9d72a0111cf6ddfe82f67898a3c83 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 12:03:51 -0600 Subject: [PATCH 070/476] EAMxx: fix dimension names when returning layouts from grids --- components/eamxx/src/share/grid/abstract_grid.cpp | 5 +++-- components/eamxx/src/share/grid/abstract_grid.hpp | 1 - components/eamxx/src/share/grid/point_grid.cpp | 12 ++++++------ components/eamxx/src/share/grid/se_grid.cpp | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 033e48cd88eb..3e6b9fd0a0c8 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -69,8 +69,9 @@ FieldLayout AbstractGrid:: get_vertical_layout (const bool midpoints) const { using namespace ShortFieldTagsNames; - return midpoints ? FieldLayout ({ LEV},{m_num_vert_levs}) - : FieldLayout ({ILEV},{m_num_vert_levs+1}); + const auto t = midpoints ? LEV : ILEV; + const auto d = m_num_vert_levs + (midpoints ? 0 : 1); + return FieldLayout({t},{d}).rename_dims(m_special_tag_names); } FieldLayout diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 26cde401f4ff..00abba1571a7 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -203,7 +203,6 @@ class AbstractGrid : public ekat::enable_shared_from_this // since it calls get_2d_scalar_layout. void create_dof_fields (const int scalar2d_layout_rank); -private: // The grid name and type GridType m_type; diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index 70b2ce81e03a..8cf73e21f2a0 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -44,7 +44,7 @@ PointGrid::get_2d_scalar_layout () const { using namespace ShortFieldTagsNames; - return FieldLayout({COL},{get_num_local_dofs()}); + return FieldLayout({COL},{get_num_local_dofs()}).rename_dims(m_special_tag_names); } FieldLayout @@ -54,7 +54,7 @@ PointGrid::get_2d_vector_layout (const int vector_dim, const std::string& vec_di FieldLayout fl({COL,CMP},{get_num_local_dofs(),vector_dim}); fl.rename_dim(1,vec_dim_name); - return fl; + return fl.rename_dims(m_special_tag_names); } FieldLayout @@ -75,7 +75,7 @@ PointGrid::get_2d_tensor_layout (const std::vector& cmp_dims, fl.append_dim(CMP,cmp_dims[i],cmp_names[i]); } - return fl; + return fl.rename_dims(m_special_tag_names); } FieldLayout @@ -86,7 +86,7 @@ PointGrid::get_3d_scalar_layout (const bool midpoints) const int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - return FieldLayout({COL,VL},{get_num_local_dofs(),nvl}); + return FieldLayout({COL,VL},{get_num_local_dofs(),nvl}).rename_dims(m_special_tag_names); } FieldLayout @@ -100,7 +100,7 @@ PointGrid::get_3d_vector_layout (const bool midpoints, const int vector_dim, FieldLayout fl({COL,CMP,VL},{get_num_local_dofs(),vector_dim,nvl}); fl.rename_dim(1,vec_dim_name); - return fl; + return fl.rename_dims(m_special_tag_names); } FieldLayout @@ -126,7 +126,7 @@ PointGrid::get_3d_tensor_layout (const bool midpoints, } fl.append_dim(VL,nvl); - return fl; + return fl.rename_dims(m_special_tag_names); } std::shared_ptr diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index 3ef770cf0c7c..741bdb4e640d 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -36,7 +36,7 @@ SEGrid::get_2d_scalar_layout () const { using namespace ShortFieldTagsNames; - return FieldLayout({EL,GP,GP},{m_num_local_elem,m_num_gp,m_num_gp}); + return FieldLayout({EL,GP,GP},{m_num_local_elem,m_num_gp,m_num_gp}).rename_dims(m_special_tag_names); } FieldLayout @@ -46,7 +46,7 @@ SEGrid::get_2d_vector_layout (const int vector_dim, const std::string& vec_dim_n FieldLayout fl({EL,CMP,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); fl.rename_dim(1,vec_dim_name); - return fl; + return fl.rename_dims(m_special_tag_names); } FieldLayout @@ -70,7 +70,7 @@ SEGrid::get_2d_tensor_layout (const std::vector& cmp_dims, } fl.append_dim(GP,m_num_gp); - return fl; + return fl.rename_dims(m_special_tag_names); } FieldLayout @@ -81,7 +81,7 @@ SEGrid::get_3d_scalar_layout (const bool midpoints) const int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); auto VL = midpoints ? LEV : ILEV; - return FieldLayout({EL,GP,GP,VL},{m_num_local_elem,m_num_gp,m_num_gp,nvl}); + return FieldLayout({EL,GP,GP,VL},{m_num_local_elem,m_num_gp,m_num_gp,nvl}).rename_dims(m_special_tag_names); } FieldLayout @@ -95,7 +95,7 @@ SEGrid::get_3d_vector_layout (const bool midpoints, const int vector_dim, FieldLayout fl({EL,CMP,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); fl.rename_dim(1,vec_dim_name); - return fl; + return fl.rename_dims(m_special_tag_names); } FieldLayout @@ -124,7 +124,7 @@ SEGrid::get_3d_tensor_layout (const bool midpoints, fl.append_dim(GP,m_num_gp); fl.append_dim(VL,nvl); - return fl; + return fl.rename_dims(m_special_tag_names); } Field SEGrid::get_cg_dofs_gids () From 11894e0adbb2672945e5ff1705871b1527704ba7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 12:04:45 -0600 Subject: [PATCH 071/476] EAMxx: remove lib linked to a unit test for no reason --- .../homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt index c3fa3c7bc2a9..52dcc3a47ac0 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt @@ -10,7 +10,7 @@ CreateDynamicsLib("theta-l_kokkos" 4 72 10) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS dynamics tms shoc cld p3 rrtmgp physics pg2 - LIBS cld_fraction nudging tms shoc p3 scream_rrtmgp ${dynLibName} diagnostics + LIBS cld_fraction tms shoc p3 scream_rrtmgp ${dynLibName} diagnostics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) From ef857ae3a16cf4802d35845e14ae625ba5b89f7d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 18:25:23 -0600 Subject: [PATCH 072/476] EAMxx: fix creation of io layouts in IOP The tags names must be preserved --- .../eamxx/src/control/intensive_observation_period.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 9f9a3474caf5..a374c595102b 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -498,9 +498,8 @@ read_fields_from_file_for_iop (const std::string& file_name, "as first dim tag.\n"); // Set first dimension to match input file - auto dims = fm_fid.get_layout().dims(); - dims[0] = io_grid->get_num_local_dofs(); - FieldLayout io_fl(fm_fid.get_layout().tags(), dims); + FieldLayout io_fl = fm_fid.get_layout(); + io_fl.reset_dim(0,io_grid->get_num_local_dofs()); FieldIdentifier io_fid(fm_fid.name(), io_fl, fm_fid.get_units(), io_grid->name()); Field io_field(io_fid); io_field.allocate_view(); From d1d8e3f63975665697291aac0eec4deb58c01500 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 24 Apr 2024 12:59:06 -0500 Subject: [PATCH 073/476] refine and refocus the rad diags text --- components/eamxx/docs/user/clean_clear_sky.md | 47 +------------------ 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/components/eamxx/docs/user/clean_clear_sky.md b/components/eamxx/docs/user/clean_clear_sky.md index 56ae29bf9337..0cde752028a2 100644 --- a/components/eamxx/docs/user/clean_clear_sky.md +++ b/components/eamxx/docs/user/clean_clear_sky.md @@ -6,8 +6,6 @@ These extra diagnostics are optionally added to the main radiation call. The ext - Clean-clear-sky fluxes: the fluxes that would be present if there were neither aerosols nor clouds, and are calculated by adding an additional radiation call at the very beginning of the logic before the optics class is endowed with aerosol and cloud properties. - Clean-sky fluxes: the fluxes that would be present if there were no aerosols, and are calculated by adding an additional radiation call after substantiating an additional optics class, but not endowing it with aerosol properties. -It was necessary to add an additional optics class because the original optics class is endowed with aerosols before clouds (in order to calculate the clear-sky fluxes). - ## Example setup (current as of April 2024) The extra calls are controlled by runtime flags `extra_clnclrsky_diag` and `extra_clnsky_diag` (they take either `true` or `false` as their values). @@ -17,7 +15,7 @@ The extra calls are controlled by runtime flags `extra_clnclrsky_diag` and `extr ./atmchange extra_clnsky_diag=true ``` -An example output file with the additional radiation diagnostics requested. +Below is an example output file to output the extra (clean and clean-clear-sky) radiation diagnostics atop the atmosphere. ```yaml %YAML 1.1 @@ -28,53 +26,10 @@ Max Snapshots Per File: 1 Fields: Physics PG2: Field Names: - #2D vars - - SW_flux_up_at_model_top - - SW_flux_dn_at_model_top - - LW_flux_up_at_model_top - SW_clnclrsky_flux_up_at_model_top - LW_clnclrsky_flux_up_at_model_top - - SW_clrsky_flux_up_at_model_top - - LW_clrsky_flux_up_at_model_top - SW_clnsky_flux_up_at_model_top - LW_clnsky_flux_up_at_model_top - - SW_flux_dn_at_model_bot - - SW_clnclrsky_flux_dn_at_model_bot - - SW_clrsky_flux_dn_at_model_bot - - SW_clnsky_flux_dn_at_model_bot - - SW_flux_up_at_model_bot - - SW_clnclrsky_flux_up_at_model_bot - - SW_clrsky_flux_up_at_model_bot - - SW_clnsky_flux_up_at_model_bot - - LW_flux_dn_at_model_bot - - LW_clnclrsky_flux_dn_at_model_bot - - LW_clrsky_flux_dn_at_model_bot - - LW_clnsky_flux_dn_at_model_bot - - LW_flux_up_at_model_bot - - LongwaveCloudForcing - - ShortwaveCloudForcing - - ps - - SeaLevelPressure - - T_2m - - qv_2m - - surf_radiative_T - - VapWaterPath - - IceWaterPath - - LiqWaterPath - - RainWaterPath - - ZonalVapFlux - - MeridionalVapFlux - - surf_evap - - surf_sens_flux - - surface_upward_latent_heat_flux - - precip_liq_surf_mass_flux - - precip_ice_surf_mass_flux - - landfrac - - ocnfrac - - PotentialTemperature_at_700hPa - - PotentialTemperature_at_1000hPa - - omega_at_500hPa - - RelativeHumidity_at_700hPa output_control: Frequency: 1 frequency_units: nmonths From 0b6d21c142f2eb46643137655826ba8a7ebfa0fa Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 24 Apr 2024 13:01:22 -0500 Subject: [PATCH 074/476] more descriptive text for these nudging docs --- .../{coarse_nudging.md => nudging_from_coarse_reanalysis.md} | 0 components/eamxx/mkdocs.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename components/eamxx/docs/user/{coarse_nudging.md => nudging_from_coarse_reanalysis.md} (100%) diff --git a/components/eamxx/docs/user/coarse_nudging.md b/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md similarity index 100% rename from components/eamxx/docs/user/coarse_nudging.md rename to components/eamxx/docs/user/nudging_from_coarse_reanalysis.md diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index 6cc8fdeae0a8..85e8e54dfe9a 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -8,7 +8,7 @@ nav: - 'Model output': 'user/model_output.md' - 'Model input': 'user/model_input.md' - 'Runtime parameters': 'common/eamxx_params.md' - - 'Coarse nudging': 'user/coarse_nudging.md' + - 'Nudging from coarse reanalysis': 'user/nudging_from_coarse_reanalysis.md' - 'Extra radiation calls': 'user/clean_clear_sky.md' - 'Developer Guide': - 'Overview': 'developer/index.md' From 3dfee227546b3045523a4f716fc15e127cda352b Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 24 Apr 2024 13:13:41 -0500 Subject: [PATCH 075/476] point to the code, not yet-to-exist docs --- components/eamxx/docs/user/nudging_from_coarse_reanalysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md b/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md index 6dbf4d7eeb46..3b8768554356 100644 --- a/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md +++ b/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md @@ -24,4 +24,4 @@ The following options are needed to specify the nudging. ./atmchange nudging::nudging_refine_remap_mapfile="${PSCRATCH}/map_ne45pg2_to_ne256pg2.trbilin.20240410.nc" ``` -To gain a deeper understanding of these parameters and options, please refer to the overarching nudging documentation. +To gain a deeper understanding of these parameters and options, please refer to code implementation of the nudging process. From e10406331c02b73073f80d572dc34aba67fa9901 Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Wed, 24 Apr 2024 11:42:16 -0700 Subject: [PATCH 076/476] slimmed down descriptions in top level intro --- components/eamxx/docs/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/docs/index.md b/components/eamxx/docs/index.md index d10d4d630fb4..32ace14eb554 100644 --- a/components/eamxx/docs/index.md +++ b/components/eamxx/docs/index.md @@ -4,8 +4,8 @@ EAMxx is an entirely new atmosphere model for E3SM. It is written in C++ using t Like the documentation for other component models, EAMxx documentation is divided into: -* The [User Guide](user/index.md) which explains how to run EAMxx and describes all the compile-time and run-time options available for modifying a simulation -* The [Developer Guide](developer/index.md) which contains all the information needed to contribute to the development of EAMxx -* The [Technical Guide](technical/index.md) which describes the equations and numerical methods used by each of the specific process implementations and diagnostics available in the current version of EAMxx +* The [User Guide](user/index.md) - info about running EAMxx and all options for modifying a simulation +* The [Developer Guide](developer/index.md) - information needed to contribute to EAMxx development +* The [Technical Guide](technical/index.md) - equations and numerical methods used in EAMxx Put another way, all information about how to customize runs without changing code is included in the User Guide, general information about software design which is needed for intelligently modifying code goes in the Developer Guide, and details about the specific process implementations in the current model version are included in the Technical Guide. From 7167a9913d35434d35a67314571c1da719cf1215 Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Thu, 25 Apr 2024 08:27:33 -0700 Subject: [PATCH 077/476] cleaned up relation to older EAMs --- components/eamxx/docs/index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/docs/index.md b/components/eamxx/docs/index.md index 32ace14eb554..d243f3c27560 100644 --- a/components/eamxx/docs/index.md +++ b/components/eamxx/docs/index.md @@ -1,6 +1,10 @@ # The E3SM Atmosphere Model in C++ (EAMxx) -EAMxx is an entirely new atmosphere model for E3SM. It is written in C++ using the Kokkos performance portability library to enable it to run efficiently on CPUs, GPUs, and (hopefully) whatever comes next. Currently only the km-scale explicit-convection version called SCREAM (the Simple Cloud-Resolving E3SM Atmosphere Model) is available, but a low-resolution version is in the works. +EAMxx +EAMxx is almost completely different in all ways from the atmosphere model used for E3SM versions 1-3. + + +EAMxx was built from the ground up using C++ in order to embrace modern software practices and to allow "performance portability" across various supercomputers. The latter goal is achieved by using the Kokkos library. EAMxx is a "clean-start" model with almost no similarity to the E3SM atmosphere model used in versions 1-3. Currently only the km-scale explicit-convection version called SCREAM (the Simple Cloud-Resolving E3SM Atmosphere Model) is available, but a low-resolution version is in the works. Like the documentation for other component models, EAMxx documentation is divided into: From 517105fcbd15e961fe9b011bd98fb1f5d1b2aeea Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Thu, 25 Apr 2024 08:38:11 -0700 Subject: [PATCH 078/476] cleaned up tech guide intro --- components/eamxx/docs/technical/index.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 895df86b97c3..0dbad183d0a2 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -10,14 +10,15 @@ Currently, EAMxx is only configured for km-scale convection-permitting runs. In 2. gravity-wave drag 3. energy fixer -The only configuration of EAMxx that is currently implemented is the convection-permitting version, commonly known as the Simple Cloud-Resolving E3SM Atmosphere Model (SCREAM). Parameterizations in EAMxx-SCREAM are: +The only configuration of EAMxx that is currently implemented is the convection-permitting version, commonly known as the Simple Cloud-Resolving E3SM Atmosphere Model (SCREAM). Processes in EAMxx-SCREAM are: 1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions [@Taylor_et20] with semi-Lagrangian tracer advection as described by [@Bradley_et22] -2. the **Simple Higher-Order Closure (SHOC)** parameterization from [@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction -3. **turbulent mountain stress** is crudely parameterized following [@Fiedler_Panofsky72] to reduce excessive winds around topography +2. **turbulent mountain stress** is crudely parameterized following [@Fiedler_Panofsky72] to reduce excessive winds around topography +3. the **Simple Higher-Order Closure (SHOC)** parameterization from [@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction 4. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. -5. the effects of aerosol are prescribed via the Simple Prescribed Aerosol (SPA) scheme, which is very similar to MACv2-SP [@Stevens_et17] -5. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC -6. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance +5. the effects of aerosol are prescribed via the **Simple Prescribed Aerosol (SPA)** scheme, which is very similar to MACv2-SP [@Stevens_et17] +6. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC +7. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance +8. the **CFMIP Observation Simulator Package (COSP)** is also integrated into EAMxx, but currently only the ISCCP output is enabled -By default processes are called in this order, but which processes to include and in what order is modifiable at run time. These parameterizations are described in more detail in Caldwell et al 2021. The CFMIP Observation Simulator Package (COSP) is also integrated into EAMxx, but currently only the ISCCP output is enabled. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. +By default processes are called in this order, but which processes to include and in what order is modifiable at run time. After all atmospheric processes are called, output is written. Surface components are then called before the next atmosphere step starts. These processes are described in more detail in [@Caldwell_et21]. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. From 4232e64bba9516d756f1ed6fc80f73ea5cac8411 Mon Sep 17 00:00:00 2001 From: Noel Keen Date: Thu, 25 Apr 2024 13:54:40 -0400 Subject: [PATCH 079/476] upgrade to rocm 5.4 for frontier-scream --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b0e083c6f518..87dfacd7fd41 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1433,7 +1433,7 @@ PrgEnv-cray craype-accel-amd-gfx90a - rocm/5.1.0 + rocm/5.4.0 libunwind/1.6.2 From 37d402374567949ea8b2bff1cc4650d4d0975213 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 25 Apr 2024 12:12:06 -0600 Subject: [PATCH 080/476] mam4xx - removing hard-coded aerosol species names and using mam4xx map instead. --- .../eamxx/src/physics/mam/mam_coupling.hpp | 22 ++++--------------- externals/mam4xx | 2 +- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index e8a2d127c0d2..69fccddccdbf 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -87,22 +87,6 @@ const char* aero_mode_name(const int mode) { return mode_names[mode]; } -// Given a MAM aerosol species ID, returns a string denoting the symbolic -// name of the species. -KOKKOS_INLINE_FUNCTION -const char* aero_species_name(const int species_id) { - static const char *species_names[num_aero_species()] = { - "soa", - "so4", - "pom", - "bc", - "nacl", - "dst", - "mom", - }; - return species_names[species_id]; -} - // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. KOKKOS_INLINE_FUNCTION @@ -220,7 +204,8 @@ const char* int_aero_mmr_field_name(const int mode, const int species) { if (!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); if (aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), + auto aerosol_species_name =mam4::aero_id_short_name(aero_id); + concat_3_strings(aerosol_species_name.c_str(), "_a", aero_mode_name(mode), int_aero_mmr_names(mode, species)); } @@ -238,7 +223,8 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { if (!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); if (aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), + auto aerosol_species_name =mam4::aero_id_short_name(aero_id); + concat_3_strings(aerosol_species_name.c_str(), "_c", aero_mode_name(mode), cld_aero_mmr_names(mode, species)); } diff --git a/externals/mam4xx b/externals/mam4xx index 168c5c07bf6f..19289b5d4c06 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 168c5c07bf6f881269bf39089049ec8726f21ed8 +Subproject commit 19289b5d4c06b8738698442cd5a25477c9f94ed4 From 2a6f5f5a5e161c90ab9ec18bcbd221957285af22 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 25 Apr 2024 16:41:47 -0500 Subject: [PATCH 081/476] add NumberPath diags --- .../eamxx/src/diagnostics/CMakeLists.txt | 1 + .../eamxx/src/diagnostics/number_path.cpp | 96 +++++++ .../eamxx/src/diagnostics/number_path.hpp | 42 +++ .../src/diagnostics/register_diagnostics.hpp | 2 + .../src/diagnostics/tests/CMakeLists.txt | 4 + .../diagnostics/tests/number_paths_tests.cpp | 243 ++++++++++++++++++ .../eamxx/src/share/io/scorpio_output.cpp | 6 + 7 files changed, 394 insertions(+) create mode 100644 components/eamxx/src/diagnostics/number_path.cpp create mode 100644 components/eamxx/src/diagnostics/number_path.hpp create mode 100644 components/eamxx/src/diagnostics/tests/number_paths_tests.cpp diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 7acd3662afd5..064b0553f6a5 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -18,6 +18,7 @@ set(DIAGNOSTIC_SRCS water_path.cpp wind_speed.cpp aodvis.cpp + number_path.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/number_path.cpp b/components/eamxx/src/diagnostics/number_path.cpp new file mode 100644 index 000000000000..658c4a0a2e93 --- /dev/null +++ b/components/eamxx/src/diagnostics/number_path.cpp @@ -0,0 +1,96 @@ +#include "diagnostics/number_path.hpp" + +#include + +#include "physics/share/physics_constants.hpp" + +namespace scream { + +NumberPathDiagnostic::NumberPathDiagnostic(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + EKAT_REQUIRE_MSG(params.isParameter("Number Kind"), + "Error! NumberPathDiagnostic requires 'Number Kind' in its " + "input parameters.\n"); + + m_kind = m_params.get("Number Kind"); + if(m_kind == "Liq") { + m_qname = "qc"; + m_nname = "nc"; + } else if(m_kind == "Ice") { + m_qname = "qi"; + m_nname = "ni"; + } else if(m_kind == "Rain") { + m_qname = "qr"; + m_nname = "nr"; + } else { + EKAT_ERROR_MSG( + "Error! Invalid choice for 'NumberKind' in NumberPathDiagnostic.\n" + " - input value: " + + m_kind + + "\n" + " - valid values: Liq, Ice, Rain\n"); + } +} + +std::string NumberPathDiagnostic::name() const { return m_kind + "NumberPath"; } + +void NumberPathDiagnostic::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + const auto m2 = m * m; + auto Q = kg / kg; + Q.set_string("kg/kg"); + + auto grid = grids_manager->get_grid("Physics"); + const auto &grid_name = grid->name(); + m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank + m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column + + auto scalar2d = grid->get_2d_scalar_layout(); + auto scalar3d = grid->get_3d_scalar_layout(true); + + // The fields required for this diagnostic to be computed + add_field("pseudo_density", scalar3d, Pa, grid_name); + add_field(m_qname, scalar3d, kg / kg, grid_name); + add_field(m_nname, scalar3d, 1 / kg, grid_name); + + // Construct and allocate the diagnostic field + FieldIdentifier fid(name(), scalar2d, 1 / m2, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); +} + +void NumberPathDiagnostic::compute_diagnostic_impl() { + using PC = scream::physics::Constants; + using KT = KokkosTypes; + using MT = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + constexpr Real g = PC::gravit; + + const auto np = m_diagnostic_output.get_view(); + const auto q = get_field_in(m_qname).get_view(); + const auto n = get_field_in(m_nname).get_view(); + const auto rho = get_field_in("pseudo_density").get_view(); + + const auto num_levs = m_num_levs; + const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); + Kokkos::parallel_for( + "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { + const int icol = team.league_rank(); + auto q_icol = ekat::subview(q, icol); + auto n_icol = ekat::subview(n, icol); + auto rho_icol = ekat::subview(rho, icol); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(team, num_levs), + [&](const int &ilev, Real &lsum) { + lsum += q_icol(ilev) * n_icol(ilev) * rho_icol(ilev) / g; + }, + np(icol)); + team.team_barrier(); + }); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/number_path.hpp b/components/eamxx/src/diagnostics/number_path.hpp new file mode 100644 index 000000000000..4888d3601f44 --- /dev/null +++ b/components/eamxx/src/diagnostics/number_path.hpp @@ -0,0 +1,42 @@ +#ifndef EAMXX_NUMBER_PATH_DIAGNOSTIC_HPP +#define EAMXX_NUMBER_PATH_DIAGNOSTIC_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream { + +/* + * This diagnostic will produce the "number" path. + */ + +class NumberPathDiagnostic : public AtmosphereDiagnostic { + public: + // Constructors + NumberPathDiagnostic(const ekat::Comm &comm, + const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + protected: + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + std::string m_qname; + std::string m_nname; + std::string m_kind; +}; // class NumberPathDiagnostic + +} // namespace scream + +#endif // EAMXX_NUMBER_PATH_DIAGNOSTIC_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index b57a04bd3486..bc9d0940fc44 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -21,6 +21,7 @@ #include "diagnostics/surf_upward_latent_heat_flux.hpp" #include "diagnostics/wind_speed.hpp" #include "diagnostics/aodvis.hpp" +#include "diagnostics/number_path.hpp" namespace scream { @@ -49,6 +50,7 @@ inline void register_diagnostics () { diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); + diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 3e17d1122a7a..bbaf666a247c 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -64,4 +64,8 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test AODVIS CreateDiagTest(aodvis "aodvis_test.cpp") + + # Test "number" paths + CreateDiagTest(number_paths "number_paths_tests.cpp") + endif() diff --git a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp new file mode 100644 index 000000000000..f42b79b0101a --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp @@ -0,0 +1,243 @@ +#include + +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "ekat/kokkos/ekat_kokkos_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_common_physics_functions.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_utils.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +//-----------------------------------------------------------------------------------------------// +template +void run(std::mt19937_64 &engine) { + using PC = scream::physics::Constants; + using KT = ekat::KokkosTypes; + using ESU = ekat::ExeSpaceUtils; + using MemberType = typename KT::MemberType; + using view_1d = typename KT::template view_1d; + + constexpr int num_levs = 33; + constexpr Real gravit = PC::gravit; + constexpr Real macheps = PC::macheps; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // Create a grids manager - single column for these tests + const int ncols = 10; + auto gm = create_gm(comm, ncols, num_levs); + + // Kokkos Policy + auto policy = ESU::get_default_team_policy(ncols, num_levs); + + // Input (randomized) views + view_1d pseudo_density("pseudo_density", num_levs), qc("qc", num_levs), + nc("nc", num_levs), qr("qr", num_levs), nr("nr", num_levs), + qi("qi", num_levs), ni("ni", num_levs); + + // Construct random input data + using RPDF = std::uniform_real_distribution; + RPDF pdf_qx(0.0, 1e-3), pdf_pseudodens(1.0, 100.0); + + // A time stamp + util::TimeStamp t0({2022, 1, 1}, {0, 0, 0}); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + ekat::ParameterList params; + + REQUIRE_THROWS( + diag_factory.create("NumberPath", comm, params)); // No 'Number Kind' + params.set("Number Kind", "Foo"); + REQUIRE_THROWS(diag_factory.create("NumberPath", comm, + params)); // Invalid 'Number Kind' + + // Liquid + params.set("Number Kind", "Liq"); + auto diag_liq = diag_factory.create("NumberPath", comm, params); + diag_liq->set_grids(gm); + diags.emplace("lnp", diag_liq); + // Ice + params.set("Number Kind", "Ice"); + auto diag_ice = diag_factory.create("NumberPath", comm, params); + diag_ice->set_grids(gm); + diags.emplace("inp", diag_ice); + // Rain + params.set("Number Kind", "Rain"); + auto diag_rain = diag_factory.create("NumberPath", comm, params); + diag_rain->set_grids(gm); + diags.emplace("rnp", diag_rain); + + // Set the required fields for the diagnostic. + std::map input_fields; + for(const auto &dd : diags) { + const auto &diag = dd.second; + for(const auto &req : diag->get_required_field_requests()) { + if(input_fields.find(req.fid.name()) == input_fields.end()) { + Field f(req.fid); + f.allocate_view(); + f.get_header().get_tracking().update_time_stamp(t0); + input_fields.emplace(f.name(), f); + } + const auto &f = input_fields.at(req.fid.name()); + diag->set_required_field(f.get_const()); + REQUIRE_THROWS(diag->set_computed_field(f)); + } + // Initialize the diagnostic + diag->initialize(t0, RunType::Initial); + } + + // Run tests + { + // Construct random data to use for test + // Get views of input data and set to random values + const auto &qc_f = input_fields.at("qc"); + const auto &qc_v = qc_f.get_view(); + const auto &nc_f = input_fields.at("nc"); + const auto &nc_v = nc_f.get_view(); + const auto &qi_f = input_fields.at("qi"); + const auto &qi_v = qi_f.get_view(); + const auto &ni_f = input_fields.at("ni"); + const auto &ni_v = ni_f.get_view(); + const auto &qr_f = input_fields.at("qr"); + const auto &qr_v = qr_f.get_view(); + const auto &nr_f = input_fields.at("nr"); + const auto &nr_v = nr_f.get_view(); + const auto &pseudo_dens_f = input_fields.at("pseudo_density"); + const auto &pseudo_dens_v = pseudo_dens_f.get_view(); + for(int icol = 0; icol < ncols; icol++) { + const auto &qc_sub = ekat::subview(qc_v, icol); + const auto &nc_sub = ekat::subview(nc_v, icol); + const auto &qi_sub = ekat::subview(qi_v, icol); + const auto &ni_sub = ekat::subview(ni_v, icol); + const auto &qr_sub = ekat::subview(qr_v, icol); + const auto &nr_sub = ekat::subview(nr_v, icol); + const auto &dp_sub = ekat::subview(pseudo_dens_v, icol); + ekat::genRandArray(pseudo_density, engine, pdf_pseudodens); + Kokkos::deep_copy(dp_sub, pseudo_density); + + ekat::genRandArray(qc, engine, pdf_qx); + ekat::genRandArray(nc, engine, pdf_qx); + ekat::genRandArray(qi, engine, pdf_qx); + ekat::genRandArray(ni, engine, pdf_qx); + ekat::genRandArray(qr, engine, pdf_qx); + ekat::genRandArray(nr, engine, pdf_qx); + Kokkos::deep_copy(qc_sub, qc); + Kokkos::deep_copy(nc_sub, nc); + Kokkos::deep_copy(qi_sub, qi); + Kokkos::deep_copy(ni_sub, ni); + Kokkos::deep_copy(qr_sub, qr); + Kokkos::deep_copy(nr_sub, nr); + } + // Grab views for each of the number path diagnostics + const auto &lnp = diags["lnp"]->get_diagnostic(); + const auto &inp = diags["inp"]->get_diagnostic(); + const auto &rnp = diags["rnp"]->get_diagnostic(); + const auto &lnp_v = lnp.get_view(); + const auto &inp_v = inp.get_view(); + const auto &rnp_v = rnp.get_view(); + const auto &lnp_h = lnp.get_view(); + const auto &inp_h = inp.get_view(); + const auto &rnp_h = rnp.get_view(); + + for(const auto &dd : diags) { + dd.second->compute_diagnostic(); + } + // test manual calculation vs one provided by diags + { + Kokkos::parallel_for( + "", policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); + auto dp_icol = ekat::subview(pseudo_dens_v, icol); + // Liquid + Real qndc_prod = 0.0; + auto qc_icol = ekat::subview(qc_v, icol); + auto nc_icol = ekat::subview(nc_v, icol); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(team, num_levs), + [&](Int idx, Real &lsum) { + lsum += nc_icol(idx) * qc_icol(idx) * dp_icol(idx) / gravit; + }, + qndc_prod); + REQUIRE(std::abs(lnp_v(icol) - qndc_prod) < macheps); + team.team_barrier(); + // Ice + Real qndi_prod = 0.0; + auto qi_icol = ekat::subview(qi_v, icol); + auto ni_icol = ekat::subview(ni_v, icol); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(team, num_levs), + [&](Int idx, Real &lsum) { + lsum += ni_icol(idx) * qi_icol(idx) * dp_icol(idx) / gravit; + }, + qndi_prod); + inp.sync_to_host(); + REQUIRE(std::abs(inp_v(icol) - qndi_prod) < macheps); + team.team_barrier(); + // Rain + Real qndr_prod = 0.0; + auto qr_icol = ekat::subview(qr_v, icol); + auto nr_icol = ekat::subview(nr_v, icol); + Kokkos::parallel_reduce( + Kokkos::TeamVectorRange(team, num_levs), + [&](Int idx, Real &lsum) { + lsum += nr_icol(idx) * qr_icol(idx) * dp_icol(idx) / gravit; + }, + qndr_prod); + REQUIRE(std::abs(rnp_v(icol) - qndr_prod) < macheps); + team.team_barrier(); + }); + Kokkos::fence(); + } + } + + // Finalize the diagnostic + for(const auto &dd : diags) { + const auto &diag = dd.second; + diag->finalize(); + } + +} // run() + +TEST_CASE("number_path_test", "number_path_test]") { + using Device = scream::DefaultDevice; + + constexpr int num_runs = 10; + + auto engine = scream::setup_random_test(); + + printf(" -> Number of randomized runs: %d\n\n", num_runs); + + for(int irun = 0; irun < num_runs; ++irun) { + run(engine); + } +} + +} // namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 42f0f38278d4..2762c7fcf8aa 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1319,6 +1319,12 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "WaterPath"; // split will return the list [X, ''], with X being whatever is before 'WaterPath' params.set("Water Kind",ekat::split(diag_field_name,"WaterPath").front()); + } else if (diag_field_name=="IceNumberPath" or + diag_field_name=="LiqNumberPath" or + diag_field_name=="RainNumberPath") { + diag_name = "NumberPath"; + // split will return the list [X, ''], with X being whatever is before 'NumberPath' + params.set("Number Kind",ekat::split(diag_field_name,"NumberPath").front()); } else if (diag_field_name=="MeridionalVapFlux" or diag_field_name=="ZonalVapFlux") { diag_name = "VaporFlux"; From 3133733989f62eded00d2bc6a22cb6d1159938e9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 25 Apr 2024 15:55:26 -0600 Subject: [PATCH 082/476] Progress ob validation --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 79 ++++++++++++++++++- .../src/physics/rrtmgp/rrtmgp_test_utils.cpp | 4 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 11 +-- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 12 +-- 5 files changed, 83 insertions(+), 25 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 374f2bdfbe1e..4c679612ebb0 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 374f2bdfbe1eea0489bd625114347a8463dee407 +Subproject commit 4c679612ebb0b9b188a7c9623ca230de7e40a334 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 6552651658f1..7fddcc77d46a 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -582,6 +582,11 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); + VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_sw, rrtmgp::k_dist_sw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_lw, rrtmgp::k_dist_lw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, rrtmgp::cloud_optics_sw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, rrtmgp::cloud_optics_lw_k); #endif // Set property checks for fields in this process @@ -869,10 +874,10 @@ void RRTMGPRadiation::run_impl (const double dt) { return real1dk(v.data(),ncol); }; auto subview_2dk = [&](const real2dk v) -> real2dk { - return real2dk(v.data(),ncol,v.extent(0)); + return real2dk(v.data(),ncol,v.extent(1)); }; auto subview_3dk = [&](const real3dk v) -> real3dk { - return real3dk(v.data(),ncol,v.extent(0),v.extent(1)); + return real3dk(v.data(),ncol,v.extent(1),v.extent(2)); }; auto p_lay_k = subview_2dk(m_buffer.p_lay_k); @@ -1057,6 +1062,29 @@ void RRTMGPRadiation::run_impl (const double dt) { } #endif #ifdef RRTMGP_ENABLE_KOKKOS + mu0_k(i) = d_mu0(i); + sfc_alb_dir_vis_k(i) = d_sfc_alb_dir_vis(icol); + sfc_alb_dir_nir_k(i) = d_sfc_alb_dir_nir(icol); + sfc_alb_dif_vis_k(i) = d_sfc_alb_dif_vis(icol); + sfc_alb_dif_nir_k(i) = d_sfc_alb_dif_nir(icol); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { + p_lay_k(i,k) = d_pmid(icol,k); + t_lay_k(i,k) = d_tmid(icol,k); + z_del_k(i,k) = d_dz(i,k); + p_del_k(i,k) = d_pdel(icol,k); + qc_k(i,k) = d_qc(icol,k); + nc_k(i,k) = d_nc(icol,k); + qi_k(i,k) = d_qi(icol,k); + rel_k(i,k) = d_rel(icol,k); + rei_k(i,k) = d_rei(icol,k); + p_lev_k(i,k) = d_pint(icol,k); + t_lev_k(i,k) = d_tint(i,k); + }); + + p_lev_k(i,nlay) = d_pint(icol,nlay); + t_lev_k(i,nlay) = d_tint(i,nlay); + // Note that RRTMGP expects ordering (col,lay,bnd) but the FM keeps things in (col,bnd,lay) order if (do_aerosol_rad) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nswbands*nlay), [&] (const int&idx) { @@ -1089,6 +1117,11 @@ void RRTMGPRadiation::run_impl (const double dt) { }); } Kokkos::fence(); +#ifdef RRTMGP_ENABLE_KOKKOS + COMPARE_ALL_WRAP(std::vector({aero_tau_sw}), //, aero_ssa_sw, aero_g_sw, aero_tau_lw}), + std::vector({aero_tau_sw_k}));//, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); +#endif + // Populate GasConcs object to pass to RRTMGP driver // set_vmr requires the input array size to have the correct size, @@ -1130,6 +1163,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.set_vmr(name, tmp2d_k); + VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif } @@ -1193,6 +1227,10 @@ void RRTMGPRadiation::run_impl (const double dt) { }); } Kokkos::fence(); +#ifdef RRTMGP_ENABLE_KOKKOS + COMPARE_WRAP(cldfrac_tot, cldfrac_tot_k); +#endif + // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); @@ -1201,6 +1239,8 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); + COMPARE_ALL_WRAP(std::vector({lwp, iwp}), + std::vector({lwp_k, iwp_k})); #endif // Convert to g/m2 (needed by RRTMGP) { @@ -1237,6 +1277,8 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, sfc_alb_dir_k, sfc_alb_dif_k); + COMPARE_ALL_WRAP(std::vector({sfc_alb_dir, sfc_alb_dif}), + std::vector({sfc_alb_dir_k, sfc_alb_dif_k})); #endif // Compute cloud optical properties here? @@ -1284,6 +1326,25 @@ void RRTMGPRadiation::run_impl (const double dt) { eccf, m_atm_logger, m_extra_clnclrsky_diag, m_extra_clnsky_diag ); + COMPARE_ALL_WRAP(std::vector({ + sw_flux_up, sw_flux_dn, sw_flux_dn_dir, lw_flux_up, lw_flux_dn, + sw_clnclrsky_flux_up, sw_clnclrsky_flux_dn, sw_clnclrsky_flux_dn_dir, + sw_clrsky_flux_up, sw_clrsky_flux_dn, sw_clrsky_flux_dn_dir, + sw_clnsky_flux_up, sw_clnsky_flux_dn, sw_clnsky_flux_dn_dir, + lw_clnclrsky_flux_up, lw_clnclrsky_flux_dn, + lw_clrsky_flux_up, lw_clrsky_flux_dn, + lw_clnsky_flux_up, lw_clnsky_flux_dn}), + std::vector({ + sw_flux_up_k, sw_flux_dn_k, sw_flux_dn_dir_k, lw_flux_up_k, lw_flux_dn_k, + sw_clnclrsky_flux_up_k, sw_clnclrsky_flux_dn_k, sw_clnclrsky_flux_dn_dir_k, + sw_clrsky_flux_up_k, sw_clrsky_flux_dn_k, sw_clrsky_flux_dn_dir_k, + sw_clnsky_flux_up_k, sw_clnsky_flux_dn_k, sw_clnsky_flux_dn_dir_k, + lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, + lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, + lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); + + COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), + std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif // Update heating tendency @@ -1332,6 +1393,8 @@ void RRTMGPRadiation::run_impl (const double dt) { }); } Kokkos::fence(); + COMPARE_ALL_WRAP(std::vector({sw_heating, lw_heating}), + std::vector({sw_heating_k, lw_heating_k})); #endif // Index to surface (bottom of model); used to get surface fluxes below @@ -1374,6 +1437,8 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k ); + COMPARE_ALL_WRAP(std::vector({sfc_flux_dir_vis, sfc_flux_dir_nir, sfc_flux_dif_vis, sfc_flux_dif_nir}), + std::vector({sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, sfc_flux_dif_vis_k, sfc_flux_dif_nir_k})); #endif // Compute diagnostic total cloud area (vertically-projected cloud cover) @@ -1408,6 +1473,8 @@ void RRTMGPRadiation::run_impl (const double dt) { rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); + COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), + std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif // Get visible 0.67 micron band for COSP @@ -1447,6 +1514,14 @@ void RRTMGPRadiation::run_impl (const double dt) { nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k); + COMPARE_ALL_WRAP(std::vector({ + T_mid_at_cldtop, p_mid_at_cldtop, cldfrac_ice_at_cldtop, + cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, + eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop}), + std::vector({ + T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, + cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, + eff_radius_qc_at_cldtop_k, eff_radius_qi_at_cldtop_k})); #endif // Copy output data back to FieldManager diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index ea3280a3e035..54a32133da92 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -273,7 +273,7 @@ void write_fluxes( // TODO: This function should instead take values, not file names, // because for the test we do not want to write to file #ifdef RRTMGP_ENABLE_YAKL -int compare(std::string file1, std::string file2) { +int compare_y(std::string file1, std::string file2) { // Read data from baseline and test file real2d sw_flux_up_1; real2d sw_flux_dn_1; @@ -299,7 +299,7 @@ int compare(std::string file1, std::string file2) { } #endif #ifdef RRTMGP_ENABLE_KOKKOS -int compare(std::string file1, std::string file2) { +int compare_k(std::string file1, std::string file2) { // Read data from baseline and test file real2dk sw_flux_up_1; real2dk sw_flux_dn_1; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index e0f22c79eeed..a6d7b6085cf7 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -34,7 +34,7 @@ void finalize_kls() yakl::finalize(); #endif #ifdef RRTMGP_ENABLE_KOKKOS - Kokkos::finalize(); + //Kokkos::finalize(); We do the kokkos finalization elsewhere #endif } @@ -527,7 +527,7 @@ void compute_band_by_band_surface_albedos( auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, - "Error! 1st dimension for wavenumber_limits should be 2."); + "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); @@ -1986,16 +1986,9 @@ void compute_cloud_area( } #endif -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - -int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } -#endif #ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 03e2e8861ed7..19d8aedf55bf 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -261,16 +261,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); GasConcs gas_concs; - int ngas = 8; - string1d gas_names("gas_names",ngas); - gas_names(1) = std::string("h2o"); - gas_names(2) = std::string("co2"); - gas_names(3) = std::string("o3" ); - gas_names(4) = std::string("n2o"); - gas_names(5) = std::string("co" ); - gas_names(6) = std::string("ch4"); - gas_names(7) = std::string("o2" ); - gas_names(8) = std::string("n2" ); + string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); @@ -413,7 +404,6 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux") { logger->info("Free memory...\n"); scream::rrtmgp::rrtmgp_finalize(); gas_concs.reset(); - gas_names.deallocate(); sw_bnd_flux_dir.deallocate(); sw_bnd_flux_dif.deallocate(); sfc_flux_dir_nir.deallocate(); From 570cdbc1f53fe0add9308d095414e96a9190762d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 25 Apr 2024 16:43:09 -0700 Subject: [PATCH 083/476] fix failing test with simpler loops, minor edits --- .../diagnostics/tests/number_paths_tests.cpp | 127 +++++++----------- 1 file changed, 50 insertions(+), 77 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp index f42b79b0101a..a2a9eda50d41 100644 --- a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp @@ -62,7 +62,7 @@ void run(std::mt19937_64 &engine) { // Construct random input data using RPDF = std::uniform_real_distribution; - RPDF pdf_qx(0.0, 1e-3), pdf_pseudodens(1.0, 100.0); + RPDF pdf_qx(0.0, 1e-3), pdf_pd(1.0, 100.0); // A time stamp util::TimeStamp t0({2022, 1, 1}, {0, 0, 0}); @@ -118,20 +118,27 @@ void run(std::mt19937_64 &engine) { { // Construct random data to use for test // Get views of input data and set to random values - const auto &qc_f = input_fields.at("qc"); - const auto &qc_v = qc_f.get_view(); - const auto &nc_f = input_fields.at("nc"); - const auto &nc_v = nc_f.get_view(); - const auto &qi_f = input_fields.at("qi"); - const auto &qi_v = qi_f.get_view(); - const auto &ni_f = input_fields.at("ni"); - const auto &ni_v = ni_f.get_view(); - const auto &qr_f = input_fields.at("qr"); - const auto &qr_v = qr_f.get_view(); - const auto &nr_f = input_fields.at("nr"); - const auto &nr_v = nr_f.get_view(); - const auto &pseudo_dens_f = input_fields.at("pseudo_density"); - const auto &pseudo_dens_v = pseudo_dens_f.get_view(); + const auto &qc_f = input_fields.at("qc"); + const auto &qc_v = qc_f.get_view(); + const auto &qc_h = qc_f.get_view(); + const auto &nc_f = input_fields.at("nc"); + const auto &nc_v = nc_f.get_view(); + const auto &nc_h = nc_f.get_view(); + const auto &qi_f = input_fields.at("qi"); + const auto &qi_v = qi_f.get_view(); + const auto &qi_h = qi_f.get_view(); + const auto &ni_f = input_fields.at("ni"); + const auto &ni_v = ni_f.get_view(); + const auto &ni_h = ni_f.get_view(); + const auto &qr_f = input_fields.at("qr"); + const auto &qr_v = qr_f.get_view(); + const auto &qr_h = qr_f.get_view(); + const auto &nr_f = input_fields.at("nr"); + const auto &nr_v = nr_f.get_view(); + const auto &nr_h = nr_f.get_view(); + const auto &pd_f = input_fields.at("pseudo_density"); + const auto &pd_v = pd_f.get_view(); + const auto &pd_h = pd_f.get_view(); for(int icol = 0; icol < ncols; icol++) { const auto &qc_sub = ekat::subview(qc_v, icol); const auto &nc_sub = ekat::subview(nc_v, icol); @@ -139,30 +146,19 @@ void run(std::mt19937_64 &engine) { const auto &ni_sub = ekat::subview(ni_v, icol); const auto &qr_sub = ekat::subview(qr_v, icol); const auto &nr_sub = ekat::subview(nr_v, icol); - const auto &dp_sub = ekat::subview(pseudo_dens_v, icol); - ekat::genRandArray(pseudo_density, engine, pdf_pseudodens); - Kokkos::deep_copy(dp_sub, pseudo_density); - - ekat::genRandArray(qc, engine, pdf_qx); - ekat::genRandArray(nc, engine, pdf_qx); - ekat::genRandArray(qi, engine, pdf_qx); - ekat::genRandArray(ni, engine, pdf_qx); - ekat::genRandArray(qr, engine, pdf_qx); - ekat::genRandArray(nr, engine, pdf_qx); - Kokkos::deep_copy(qc_sub, qc); - Kokkos::deep_copy(nc_sub, nc); - Kokkos::deep_copy(qi_sub, qi); - Kokkos::deep_copy(ni_sub, ni); - Kokkos::deep_copy(qr_sub, qr); - Kokkos::deep_copy(nr_sub, nr); + const auto &dp_sub = ekat::subview(pd_v, icol); + ekat::genRandArray(qc_sub, engine, pdf_qx); + ekat::genRandArray(nc_sub, engine, pdf_qx); + ekat::genRandArray(qi_sub, engine, pdf_qx); + ekat::genRandArray(ni_sub, engine, pdf_qx); + ekat::genRandArray(qr_sub, engine, pdf_qx); + ekat::genRandArray(nr_sub, engine, pdf_qx); + ekat::genRandArray(dp_sub, engine, pdf_pd); } // Grab views for each of the number path diagnostics const auto &lnp = diags["lnp"]->get_diagnostic(); const auto &inp = diags["inp"]->get_diagnostic(); const auto &rnp = diags["rnp"]->get_diagnostic(); - const auto &lnp_v = lnp.get_view(); - const auto &inp_v = inp.get_view(); - const auto &rnp_v = rnp.get_view(); const auto &lnp_h = lnp.get_view(); const auto &inp_h = inp.get_view(); const auto &rnp_h = rnp.get_view(); @@ -172,49 +168,26 @@ void run(std::mt19937_64 &engine) { } // test manual calculation vs one provided by diags { - Kokkos::parallel_for( - "", policy, KOKKOS_LAMBDA(const MemberType &team) { - const int icol = team.league_rank(); - auto dp_icol = ekat::subview(pseudo_dens_v, icol); - // Liquid - Real qndc_prod = 0.0; - auto qc_icol = ekat::subview(qc_v, icol); - auto nc_icol = ekat::subview(nc_v, icol); - Kokkos::parallel_reduce( - Kokkos::TeamVectorRange(team, num_levs), - [&](Int idx, Real &lsum) { - lsum += nc_icol(idx) * qc_icol(idx) * dp_icol(idx) / gravit; - }, - qndc_prod); - REQUIRE(std::abs(lnp_v(icol) - qndc_prod) < macheps); - team.team_barrier(); - // Ice - Real qndi_prod = 0.0; - auto qi_icol = ekat::subview(qi_v, icol); - auto ni_icol = ekat::subview(ni_v, icol); - Kokkos::parallel_reduce( - Kokkos::TeamVectorRange(team, num_levs), - [&](Int idx, Real &lsum) { - lsum += ni_icol(idx) * qi_icol(idx) * dp_icol(idx) / gravit; - }, - qndi_prod); - inp.sync_to_host(); - REQUIRE(std::abs(inp_v(icol) - qndi_prod) < macheps); - team.team_barrier(); - // Rain - Real qndr_prod = 0.0; - auto qr_icol = ekat::subview(qr_v, icol); - auto nr_icol = ekat::subview(nr_v, icol); - Kokkos::parallel_reduce( - Kokkos::TeamVectorRange(team, num_levs), - [&](Int idx, Real &lsum) { - lsum += nr_icol(idx) * qr_icol(idx) * dp_icol(idx) / gravit; - }, - qndr_prod); - REQUIRE(std::abs(rnp_v(icol) - qndr_prod) < macheps); - team.team_barrier(); - }); - Kokkos::fence(); + for(int icol = 0; icol < ncols; icol++) { + Real qndc_prod = 0.0; + for(int ilev = 0; ilev < num_levs; ++ilev) { + qndc_prod += + nc_h(icol, ilev) * qc_h(icol, ilev) * pd_h(icol, ilev) / gravit; + } + REQUIRE(std::abs(lnp_h(icol) - qndc_prod) < macheps); + Real qndi_prod = 0.0; + for(int ilev = 0; ilev < num_levs; ++ilev) { + qndi_prod += + ni_h(icol, ilev) * qi_h(icol, ilev) * pd_h(icol, ilev) / gravit; + } + REQUIRE(std::abs(inp_h(icol) - qndi_prod) < macheps); + Real qndr_prod = 0.0; + for(int ilev = 0; ilev < num_levs; ++ilev) { + qndr_prod += + nr_h(icol, ilev) * qr_h(icol, ilev) * pd_h(icol, ilev) / gravit; + } + REQUIRE(std::abs(rnp_h(icol) - qndr_prod) < macheps); + } } } From d3b579c084e47dd25f7c97417c686dbc1027f964 Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Fri, 26 Apr 2024 10:19:44 -0700 Subject: [PATCH 084/476] fixed citations --- components/eamxx/docs/technical/index.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 0dbad183d0a2..6a636a481890 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -12,13 +12,13 @@ Currently, EAMxx is only configured for km-scale convection-permitting runs. In The only configuration of EAMxx that is currently implemented is the convection-permitting version, commonly known as the Simple Cloud-Resolving E3SM Atmosphere Model (SCREAM). Processes in EAMxx-SCREAM are: -1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions [@Taylor_et20] with semi-Lagrangian tracer advection as described by [@Bradley_et22] -2. **turbulent mountain stress** is crudely parameterized following [@Fiedler_Panofsky72] to reduce excessive winds around topography -3. the **Simple Higher-Order Closure (SHOC)** parameterization from [@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction +1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions[@Taylor_et20] with semi-Lagrangian tracer advection as described by Bradley et al., (2022)[@Bradley_et22] +2. **turbulent mountain stress** is crudely parameterized following Fiedler and Panofsky (1072)[@Fiedler_Panofsky72] to reduce excessive winds around topography +3. the **Simple Higher-Order Closure (SHOC)** parameterization from Bogenschutz and Krueger (2013)[@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction 4. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. -5. the effects of aerosol are prescribed via the **Simple Prescribed Aerosol (SPA)** scheme, which is very similar to MACv2-SP [@Stevens_et17] -6. the **P3 microphysics** scheme from [@Morrison_Milbrandt15] modified as described by [@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC -7. **RTE/RRTMGP radiation** from [@Pincus_et19] rewritten in C++ for consistency and performance +5. the effects of aerosol are prescribed via the **Simple Prescribed Aerosol (SPA)** scheme, which is very similar to MACv2-SP[@Stevens_et17] +6. the **P3 microphysics** scheme from Morrison and Milbrandt (2015)[@Morrison_Milbrandt15] modified as described by Caldwell et al., (2021)[@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC +7. **RTE/RRTMGP radiation** from Pincus et al., (2019)[@Pincus_et19] rewritten in C++ for consistency and performance 8. the **CFMIP Observation Simulator Package (COSP)** is also integrated into EAMxx, but currently only the ISCCP output is enabled -By default processes are called in this order, but which processes to include and in what order is modifiable at run time. After all atmospheric processes are called, output is written. Surface components are then called before the next atmosphere step starts. These processes are described in more detail in [@Caldwell_et21]. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in [@Hannah_et21]. +By default processes are called in this order, but which processes to include and in what order is modifiable at run time. After all atmospheric processes are called, output is written. Surface components are then called before the next atmosphere step starts. These processes are described in more detail in Caldwell et al., (2021)[@Caldwell_et21]. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in Hannah et al., (2021)[@Hannah_et21]. From 035356f920bfa507ff4b4d5bd6f429fd3dd56d03 Mon Sep 17 00:00:00 2001 From: PeterCaldwell Date: Fri, 26 Apr 2024 11:19:54 -0700 Subject: [PATCH 085/476] clean up refs more --- components/eamxx/docs/technical/index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 6a636a481890..2df88d5cee96 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -12,13 +12,13 @@ Currently, EAMxx is only configured for km-scale convection-permitting runs. In The only configuration of EAMxx that is currently implemented is the convection-permitting version, commonly known as the Simple Cloud-Resolving E3SM Atmosphere Model (SCREAM). Processes in EAMxx-SCREAM are: -1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions[@Taylor_et20] with semi-Lagrangian tracer advection as described by Bradley et al., (2022)[@Bradley_et22] +1. a non-hydrostatic version of the **spectral-element dynamical core** used by other E3SM Atmosphere Model versions[@Taylor_et20] with semi-Lagrangian tracer advection as described by Bradley et al. (2022)[@Bradley_et22] 2. **turbulent mountain stress** is crudely parameterized following Fiedler and Panofsky (1072)[@Fiedler_Panofsky72] to reduce excessive winds around topography 3. the **Simple Higher-Order Closure (SHOC)** parameterization from Bogenschutz and Krueger (2013)[@Bogenschutz_Krueger13], which handles turbulent diffusion, condensation/evaporation, and liquid cloud fraction 4. an **all-or-nothing ice cloud fraction** parameterization that sets ice cloud fraction to 100% whenever cloud ice mass qi is less than a user-specified threshold set by default to 1e-5 kg/kg. This scheme also sets the total cloud fraction (used by microphysics) to the maximum of the liquid and ice cloud fraction. 5. the effects of aerosol are prescribed via the **Simple Prescribed Aerosol (SPA)** scheme, which is very similar to MACv2-SP[@Stevens_et17] -6. the **P3 microphysics** scheme from Morrison and Milbrandt (2015)[@Morrison_Milbrandt15] modified as described by Caldwell et al., (2021)[@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC -7. **RTE/RRTMGP radiation** from Pincus et al., (2019)[@Pincus_et19] rewritten in C++ for consistency and performance +6. the **P3 microphysics** scheme from Morrison and Milbrandt (2015)[@Morrison_Milbrandt15] modified as described by Caldwell et al. (2021)[@Caldwell_et21] to assume instantaneous liquid saturation adjustment for consistency with SHOC +7. **RTE/RRTMGP radiation** from Pincus et al. (2019)[@Pincus_et19] rewritten in C++ for consistency and performance 8. the **CFMIP Observation Simulator Package (COSP)** is also integrated into EAMxx, but currently only the ISCCP output is enabled -By default processes are called in this order, but which processes to include and in what order is modifiable at run time. After all atmospheric processes are called, output is written. Surface components are then called before the next atmosphere step starts. These processes are described in more detail in Caldwell et al., (2021)[@Caldwell_et21]. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in Hannah et al., (2021)[@Hannah_et21]. +By default processes are called in this order, but which processes to include and in what order is modifiable at run time. After all atmospheric processes are called, output is written. Surface components are then called before the next atmosphere step starts. These processes are described in more detail in Caldwell et al. (2021)[@Caldwell_et21]. As in EAM, dynamics operates on a spectral element grid and all other processes use a finite-volume grid that divides each spectral element into 4 quadrilaterals. This physics grid is described in Hannah et al. (2021)[@Hannah_et21]. From e4aabdc89ab462bff8e69c4c088b560f0facee79 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 26 Apr 2024 14:51:37 -0600 Subject: [PATCH 086/476] Runtime fixes --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 2 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 23 ++++++++++++++----- .../rrtmgp/scream_rrtmgp_interface.hpp | 5 ++++ .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 2 ++ 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 4c679612ebb0..fff1f14603e9 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 4c679612ebb0b9b188a7c9623ca230de7e40a334 +Subproject commit fff1f14603e9f7aea84bdf03e9abc0d5189b39a0 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 7fddcc77d46a..112e52b98313 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -365,9 +365,9 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_lw_bnd.totElems(); #endif +#ifdef RRTMGP_ENABLE_KOKKOS mem = reinterpret_cast(buffer_manager.get_memory()); -#ifdef RRTMGP_ENABLE_KOKKOS // 1d arrays m_buffer.mu0_k = decltype(m_buffer.mu0_k)(mem, m_col_chunk_size); mem += m_buffer.mu0_k.size(); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index a6d7b6085cf7..438b65d7cb0c 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -75,6 +75,7 @@ CloudOpticsK cloud_optics_lw_k; #endif bool initialized = false; +bool initialized_k = false; // local functions namespace { @@ -426,13 +427,13 @@ void rrtmgp_initialize( const std::shared_ptr& logger) { // If we've already initialized, just exit - if (initialized) { + if (initialized_k) { if (logger) logger->info("RRTMGP is already initialized; skipping\n"); return; } - // Initialize YAKL + // Initialize Kokkos if (!Kokkos::is_initialized()) { Kokkos::initialize(); } // Load and initialize absorption coefficient data @@ -443,25 +444,35 @@ void rrtmgp_initialize( load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + // initialize kokkos rrtmgp pool allocator + const size_t base_ref = 18000; + const size_t ncol = gas_concs.ncol; + const size_t nlay = gas_concs.nlay; + const size_t nlev = SCREAM_NUM_VERTICAL_LEV; + const size_t my_size_ref = ncol * nlay * nlev; + conv::MemPoolSingleton::init(2e6 * (float(my_size_ref) / base_ref)); + // We are now initialized! - initialized = true; + initialized_k = true; } #endif void rrtmgp_finalize() { - initialized = false; #ifdef RRTMGP_ENABLE_YAKL + initialized = false; k_dist_sw.finalize(); k_dist_lw.finalize(); cloud_optics_sw.finalize(); //~CloudOptics(); cloud_optics_lw.finalize(); //~CloudOptics(); #endif #ifdef RRTMGP_ENABLE_KOKKOS + initialized_k = false; k_dist_sw_k.finalize(); k_dist_lw_k.finalize(); cloud_optics_sw_k.finalize(); //~CloudOptics(); cloud_optics_lw_k.finalize(); //~CloudOptics(); + conv::MemPoolSingleton::finalize(); #endif } @@ -523,7 +534,7 @@ void compute_band_by_band_surface_albedos( real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { - EKAT_ASSERT_MSG(initialized, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, @@ -1247,7 +1258,7 @@ void rrtmgp_sw( auto gas_names = gas_concs.get_gas_names(); GasConcs gas_concs_day; gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 1; igas <= ngas; igas++) { + for (int igas = 0; igas < ngas; igas++) { auto vmr_day = real2d("vmr_day", nday, nlay); auto vmr = real2d("vmr" , ncol, nlay); gas_concs.get_vmr(gas_names[igas], vmr); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index df700264ec4a..4096041b6645 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -49,7 +49,12 @@ extern CloudOpticsK cloud_optics_lw_k; /* * Flag to indicate whether or not we have initialized RRTMGP */ +#ifdef RRTMGP_ENABLE_YAKL extern bool initialized; +#endif +#ifdef RRTMGP_ENABLE_KOKKOS +extern bool initialized_k; +#endif /* * Initialize data for RRTMGP driver diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 19d8aedf55bf..400e3aa19329 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -11,11 +11,13 @@ namespace { +#ifdef RRTMGP_ENABLE_KOKKOS template auto chc(const View& view) { return Kokkos::create_mirror_view_and_copy(HostDevice(), view); } +#endif // Names of input files we will need. std::string coefficients_file_sw = SCREAM_DATA_DIR "/init/rrtmgp-data-sw-g112-210809.nc"; From 628c98a944cbc48f6692c569ca22cf253af8e65f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 26 Apr 2024 14:37:10 -0700 Subject: [PATCH 087/476] sync to host and add proper units --- .../eamxx/src/diagnostics/number_path.cpp | 7 +++--- .../diagnostics/tests/number_paths_tests.cpp | 24 ++++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/components/eamxx/src/diagnostics/number_path.cpp b/components/eamxx/src/diagnostics/number_path.cpp index 658c4a0a2e93..6ce0e773b684 100644 --- a/components/eamxx/src/diagnostics/number_path.cpp +++ b/components/eamxx/src/diagnostics/number_path.cpp @@ -39,9 +39,8 @@ void NumberPathDiagnostic::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - const auto m2 = m * m; - auto Q = kg / kg; - Q.set_string("kg/kg"); + auto out_units = kg / (kg * m * m); + out_units.set_string("kg/(kg m2)"); auto grid = grids_manager->get_grid("Physics"); const auto &grid_name = grid->name(); @@ -57,7 +56,7 @@ void NumberPathDiagnostic::set_grids( add_field(m_nname, scalar3d, 1 / kg, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d, 1 / m2, grid_name); + FieldIdentifier fid(name(), scalar2d, out_units, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp index a2a9eda50d41..850c535529a2 100644 --- a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp @@ -42,7 +42,7 @@ void run(std::mt19937_64 &engine) { using view_1d = typename KT::template view_1d; constexpr int num_levs = 33; - constexpr Real gravit = PC::gravit; + constexpr Real g = PC::gravit; constexpr Real macheps = PC::macheps; // A world comm @@ -162,29 +162,41 @@ void run(std::mt19937_64 &engine) { const auto &lnp_h = lnp.get_view(); const auto &inp_h = inp.get_view(); const auto &rnp_h = rnp.get_view(); - + // Sync to host + qc_f.sync_to_host(); + nc_f.sync_to_host(); + qi_f.sync_to_host(); + ni_f.sync_to_host(); + qr_f.sync_to_host(); + nr_f.sync_to_host(); + pd_f.sync_to_host(); + // Compute for(const auto &dd : diags) { dd.second->compute_diagnostic(); } - // test manual calculation vs one provided by diags + // Sync to host + lnp.sync_to_host(); + inp.sync_to_host(); + rnp.sync_to_host(); + // Test manual calculation vs one provided by diags { for(int icol = 0; icol < ncols; icol++) { Real qndc_prod = 0.0; for(int ilev = 0; ilev < num_levs; ++ilev) { qndc_prod += - nc_h(icol, ilev) * qc_h(icol, ilev) * pd_h(icol, ilev) / gravit; + nc_h(icol, ilev) * qc_h(icol, ilev) * pd_h(icol, ilev) / g; } REQUIRE(std::abs(lnp_h(icol) - qndc_prod) < macheps); Real qndi_prod = 0.0; for(int ilev = 0; ilev < num_levs; ++ilev) { qndi_prod += - ni_h(icol, ilev) * qi_h(icol, ilev) * pd_h(icol, ilev) / gravit; + ni_h(icol, ilev) * qi_h(icol, ilev) * pd_h(icol, ilev) / g; } REQUIRE(std::abs(inp_h(icol) - qndi_prod) < macheps); Real qndr_prod = 0.0; for(int ilev = 0; ilev < num_levs; ++ilev) { qndr_prod += - nr_h(icol, ilev) * qr_h(icol, ilev) * pd_h(icol, ilev) / gravit; + nr_h(icol, ilev) * qr_h(icol, ilev) * pd_h(icol, ilev) / g; } REQUIRE(std::abs(rnp_h(icol) - qndr_prod) < macheps); } From 0a756b615e572e3eed22c16e156918b5c982dc07 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 26 Apr 2024 20:11:59 -0400 Subject: [PATCH 088/476] add citations, generalize nudging --- components/eamxx/docs/refs.bib | 24 +++++++++++++++++ .../eamxx/docs/technical/aerocom_cldtop.md | 5 +--- components/eamxx/docs/user/nudging.md | 25 +++++++++++++++++ .../user/nudging_from_coarse_reanalysis.md | 27 ------------------- 4 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 components/eamxx/docs/user/nudging.md delete mode 100644 components/eamxx/docs/user/nudging_from_coarse_reanalysis.md diff --git a/components/eamxx/docs/refs.bib b/components/eamxx/docs/refs.bib index 6ab4e21c9173..7a8745e07096 100644 --- a/components/eamxx/docs/refs.bib +++ b/components/eamxx/docs/refs.bib @@ -123,3 +123,27 @@ @article{Taylor_et20 note = {e2019MS001783 10.1029/2019MS001783}, year = {2020} } + +@techreport{tiedtke_ecmwf_1979, + address = {Shinfield Park, Reading}, + type = {Technical {Report}}, + title = {{ECMWF} model parameterisation of sub-grid scale processes}, + language = {en}, + institution = {ECMWF}, + author = {Tiedtke, M. and Geleyn, J.-F. and Hollingsworth, A. and Louis, J.-F.}, + month = jan, + year = {1979}, + note = {10}, + pages = {146}, +} + +@article{raisanen2004stochastic, + title={Stochastic generation of subgrid-scale cloudy columns for large-scale models}, + author={R{\"a}is{\"a}nen, Petri and Barker, Howard W and Khairoutdinov, Marat F and Li, Jiangnan and Randall, David A}, + journal={Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography}, + volume={130}, + number={601}, + pages={2047--2067}, + year={2004}, + publisher={Wiley Online Library} +} diff --git a/components/eamxx/docs/technical/aerocom_cldtop.md b/components/eamxx/docs/technical/aerocom_cldtop.md index cda0c46c3b8b..2dfbfbab0d85 100644 --- a/components/eamxx/docs/technical/aerocom_cldtop.md +++ b/components/eamxx/docs/technical/aerocom_cldtop.md @@ -24,7 +24,4 @@ $$c\Phi_{i,k} = \frac{cQ_{i,k}}{iQ_{i,k} + cQ_{i,k}}$$ The thermodynamic phase is used only for cloud properties (e.g., cloud-top cloud droplet number concentration) or cloud content (e.g., cloud liquid content). Further, $X_{i,k}$ is the three-dimensional cloud property of interest which is needed if we are converting a property from three-dimensional ($X$) to its two-dimensional counterpart ($x$). "Other" properties here include temperature and pressure which are not dependent on the thermodynamic phase. -Helpful pointers: - -- Räisänen, P., Barker, H. W., Khairoutdinov, M. F., Li, J., & Randall, D. A. (2004). Stochastic generation of subgrid‐scale cloudy columns for large‐scale models. Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography, 130(601), 2047-2067. -- References in this ECMWF report: Tiedtke, M., Geleyn, J.-F., Hollingsworth, A., and Louis, J.-F.: ECMWF model parameterisation of sub-grid scale processes, Technical Report, ECMWF, Shinfield Park, Reading, 10, 1979. +Most studies in this topic refer a technical report by Tiedtke et al. (1979)[@tiedtke_ecmwf_1979]. Another more recent general reference that may be of interest is that of Räisänen et al. (2004)[@raisanen2004stochastic]. diff --git a/components/eamxx/docs/user/nudging.md b/components/eamxx/docs/user/nudging.md new file mode 100644 index 000000000000..b102cedd26c3 --- /dev/null +++ b/components/eamxx/docs/user/nudging.md @@ -0,0 +1,25 @@ +# Nudging in EAMxx + +Nudging is supported in EAMxx. +Currently, it is possible to nudge EAMxx to reference meterological fields at the run resolution or a coarser one. + +## Example setup (current as of April 2024) + +A user can produce nudging data for EAMxx using a model run or by using reanalysis data. +To enable nudging as a process, one must declare it in the `atm_procs_list` runtime parameter. + +```shell + ./atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp,nudging" +``` + +The following options are needed to specify the nudging. + +```shell +./atmchange nudging::nudging_filenames_patterns="/pathto/nudging_files/*.nc" +./atmchange nudging::source_pressure_type=TIME_DEPENDENT_3D_PROFILE +./atmchange nudging::nudging_fields=U,V +./atmchange nudging::nudging_timescale=21600 +``` + +To gain a deeper understanding of these parameters and options, please refer to code implementation of the nudging process. +In particular, depending on the specific needs, one may utilize weighted nudging (for regionally refined model runs) or nudging from coarse data via specifying a map in the `atmchange` calls above. diff --git a/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md b/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md deleted file mode 100644 index 3b8768554356..000000000000 --- a/components/eamxx/docs/user/nudging_from_coarse_reanalysis.md +++ /dev/null @@ -1,27 +0,0 @@ -# Nudging from coarse data - -Because EAMxx is designed to support ultra-high resolutions (in fact, that was the initial reason for its inception), it is not feasible to produce nudging data at the same resolution. -Instead, in EAMxx, it is possible to nudge from coarse data. -This is done by remapping the coarse data provided by the user to the runtime physics grid of EAMxx. -In order to enable nudging from coarse data, the user must provide nudging data at the coarse resolution desired and an appropriate ncremap-compatible mapping file. - -## Example setup (current as of April 2024) - -A user can produce coarse nudging data from running EAMxx or EAM at a ne30pg2 or any other applicable resolution. -Additionally, several users in the E3SM projects have produced nudging data at the ne30pg2 resolution from the MERRA2 and ERA5 datasets. Then, to enable nudging as a process, one must declare it in the `atm_procs_list` runtime parameter. - -```shell - ./atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp,nudging" -``` - -The following options are needed to specify the nudging. - -```shell - ./atmchange nudging::nudging_filenames_patterns="${PSCRATCH}/mo_ne45pg2_nudg_trim_merra/*.nc" - ./atmchange nudging::source_pressure_type=TIME_DEPENDENT_3D_PROFILE - ./atmchange nudging::nudging_fields=U,V - ./atmchange nudging::nudging_timescale=21600 # 6-hr - ./atmchange nudging::nudging_refine_remap_mapfile="${PSCRATCH}/map_ne45pg2_to_ne256pg2.trbilin.20240410.nc" -``` - -To gain a deeper understanding of these parameters and options, please refer to code implementation of the nudging process. From 7fe672e6ef1f8291cd1e231603edcbbcc67bb106 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 26 Apr 2024 20:14:23 -0400 Subject: [PATCH 089/476] refer to nudging, not old name --- components/eamxx/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index 85e8e54dfe9a..d86bc23f63a3 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -8,7 +8,7 @@ nav: - 'Model output': 'user/model_output.md' - 'Model input': 'user/model_input.md' - 'Runtime parameters': 'common/eamxx_params.md' - - 'Nudging from coarse reanalysis': 'user/nudging_from_coarse_reanalysis.md' + - 'Nudging': 'user/nudging.md' - 'Extra radiation calls': 'user/clean_clear_sky.md' - 'Developer Guide': - 'Overview': 'developer/index.md' From 9c7d36f8b6291bf8d2848741ab3031d568a8cc3a Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 26 Apr 2024 20:21:36 -0400 Subject: [PATCH 090/476] fix technical overview --- components/eamxx/docs/technical/index.md | 2 +- components/eamxx/mkdocs.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/docs/technical/index.md b/components/eamxx/docs/technical/index.md index 2df88d5cee96..0bfd2e5add0d 100644 --- a/components/eamxx/docs/technical/index.md +++ b/components/eamxx/docs/technical/index.md @@ -1,6 +1,6 @@ # EAMxx Technical Guide -The goal of this document is to describe the specific equations, parameterizations, and numerical methods used in the current version of EAMxx. Because our master-branch implementation changes every time we make a new commit, this documentation will also evolve continuously. As such, documentation for master should always be considered to be preliminary and under construction. If you want trustworthy documentation, pull it from an official model release. +The goal of this document is to describe the specific equations, parameterizations, and numerical methods used in the current version of EAMxx. Because our master-branch implementation changes every time we make a new commit, this documentation will also evolve continuously. As such, documentation for master should always be considered to be preliminary and under construction. If you want trustworthy documentation, pull it from an official model release. ## Overview diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index d86bc23f63a3..770362671d34 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -27,6 +27,7 @@ nav: - 'Full model': 'developer/cime_testing.md' - 'CI and Nightly Testing': 'developer/ci_nightly.md' - 'Technical Guide': + - 'Overview': 'technical/index.md' - 'AeroCom cloud top': 'technical/aerocom_cldtop.md' edit_uri: "" From ed8859adf1d0efd0005c2ad7c18d9a746fc45e3e Mon Sep 17 00:00:00 2001 From: Peter Caldwell Date: Mon, 29 Apr 2024 09:53:13 -0700 Subject: [PATCH 091/476] cleaned up nudging.md text --- components/eamxx/docs/user/nudging.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/docs/user/nudging.md b/components/eamxx/docs/user/nudging.md index b102cedd26c3..acb72468d97c 100644 --- a/components/eamxx/docs/user/nudging.md +++ b/components/eamxx/docs/user/nudging.md @@ -1,11 +1,10 @@ # Nudging in EAMxx Nudging is supported in EAMxx. -Currently, it is possible to nudge EAMxx to reference meterological fields at the run resolution or a coarser one. +Currently, it is possible to nudge EAMxx to the output from a different EAMxx run or to reanalysis. Nudging data can be on your model grid or an arbitrary coarser grid. Inline interpolating of finer-grid nudging data to a coarser model resolution isn't implemented yet but may be in the future. ## Example setup (current as of April 2024) -A user can produce nudging data for EAMxx using a model run or by using reanalysis data. To enable nudging as a process, one must declare it in the `atm_procs_list` runtime parameter. ```shell From 32b24f037f4a1f9d11918f689219af833159c43e Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 29 Apr 2024 10:58:57 -0600 Subject: [PATCH 092/476] mam4xx - Switching mam4xx to main branch. --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 19289b5d4c06..42ea4d6dd65d 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 19289b5d4c06b8738698442cd5a25477c9f94ed4 +Subproject commit 42ea4d6dd65d560f79554b2a3c24d18c4ad9dfa7 From a2a9f5c4ddb3d38bedb0419dc78e27c623aec6f4 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Mon, 29 Apr 2024 16:44:09 -0700 Subject: [PATCH 093/476] bring various IOP features out to the namelist --- components/eamxx/cime_config/namelist_defaults_scream.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 6c240f3542da..7f25fc55fc82 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -552,6 +552,10 @@ be lost if SCREAM_HACK_XML is not enabled. true false true + false + false + 10800 + false From 74df2ff553654f8095597cb230c68a4f0aabce4d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 24 Apr 2024 14:50:57 -0500 Subject: [PATCH 094/476] set up skeleton for aerocom diags --- .../eamxx/src/diagnostics/CMakeLists.txt | 1 + .../eamxx/src/diagnostics/aerocom_cld.cpp | 204 ++++++++++++++++++ .../eamxx/src/diagnostics/aerocom_cld.hpp | 43 ++++ .../src/diagnostics/register_diagnostics.hpp | 2 + .../src/diagnostics/tests/CMakeLists.txt | 3 + .../diagnostics/tests/aerocom_cld_test.cpp | 115 ++++++++++ .../eamxx/src/share/io/scorpio_output.cpp | 5 + 7 files changed, 373 insertions(+) create mode 100644 components/eamxx/src/diagnostics/aerocom_cld.cpp create mode 100644 components/eamxx/src/diagnostics/aerocom_cld.hpp create mode 100644 components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 064b0553f6a5..a7a1ee9eaeaf 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -19,6 +19,7 @@ set(DIAGNOSTIC_SRCS wind_speed.cpp aodvis.cpp number_path.cpp + aerocom_cld.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp new file mode 100644 index 000000000000..e878682f9580 --- /dev/null +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -0,0 +1,204 @@ +#include "diagnostics/aerocom_cld.hpp" + +#include + +#include "share/util/scream_common_physics_functions.hpp" + +namespace scream { + +AeroComCld::AeroComCld(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + EKAT_REQUIRE_MSG( + params.isParameter("AeroComCld Kind"), + "Error! NumberPathDiagnostic requires 'AeroComCld Kind' in its " + "input parameters.\n"); + + m_topbot = m_params.get("AeroComCld Kind"); +} + +std::string AeroComCld::name() const { return "AeroComCld" + m_topbot; } + +void AeroComCld::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + auto grid = grids_manager->get_grid("Physics"); + const auto &grid_name = grid->name(); + + const auto nondim = Units::nondimensional(); + const auto micron = m / 1000000; + + m_ncols = grid->get_num_local_dofs(); + m_nlevs = grid->get_num_vertical_levels(); + m_ndiag = 8; + + // Define layouts we need (both inputs and outputs) + FieldLayout scalar2d_layout{{COL, LEV}, {m_ncols, m_nlevs}}; + FieldLayout vector1d_layout{{COL, CMP}, {m_ncols, m_ndiag}}; + + // The fields required for this diagnostic to be computed + add_field("T_mid", scalar2d_layout, K, grid_name); + add_field("pseudo_density", scalar2d_layout, Pa, grid_name); + add_field("p_mid", scalar2d_layout, Pa, grid_name); + add_field("qv", scalar2d_layout, kg / kg, grid_name); + add_field("qc", scalar2d_layout, kg / kg, grid_name); + add_field("qi", scalar2d_layout, kg / kg, grid_name); + add_field("eff_radius_qc", scalar2d_layout, micron, grid_name); + add_field("eff_radius_qi", scalar2d_layout, micron, grid_name); + add_field("cldfrac_tot", scalar2d_layout, nondim, grid_name); + add_field("nc", scalar2d_layout, kg / kg, grid_name); + + // Construct and allocate the aodvis field + FieldIdentifier fid(name(), vector1d_layout, nondim, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); + + for(int i = 1; i < m_nlevs; ++i) { + m_level_vector[i] = m_topbot == "Top" ? i : m_nlevs - 1 - i; + } + + // Self-document the outputs to parse in post-processing + using stratt_t = std::map; + auto d = get_diagnostic(); + auto &metadata = + d.get_header().get_extra_data("io: string attributes"); + metadata["0"] = "T_mid"; + metadata["1"] = "p_mid"; + metadata["2"] = "cldfrac_ice"; + metadata["3"] = "cldfrac_liq"; + metadata["4"] = "cdnc"; + metadata["5"] = "eff_radius_qc"; + metadata["6"] = "eff_radius_qi"; + metadata["7"] = "cldfrac_tot"; +} + +void AeroComCld::compute_diagnostic_impl() { + /* The goal of this routine/impl is to calculate properties at cloud top + * based on the AeroCom recommendation. See reference for routine + * get_subcolumn_mask in rrtmpg, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. The logic is reversed to calculate + * cloud-bottom properties as well. + */ + using KT = KokkosTypes; + using MT = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + + using PF = scream::PhysicsFunctions; + + const auto out = m_diagnostic_output.get_view(); + + // zero out the outputs + Kokkos::deep_copy(out, 0.0); + + // Get the input fields + const auto tmid = get_field_in("T_mid").get_view(); + const auto pden = get_field_in("pseudo_density").get_view(); + const auto pmid = get_field_in("p_mid").get_view(); + const auto qv = get_field_in("qv").get_view(); + const auto qc = get_field_in("qc").get_view(); + const auto qi = get_field_in("qi").get_view(); + const auto rel = get_field_in("eff_radius_qc").get_view(); + const auto rei = get_field_in("eff_radius_qi").get_view(); + const auto cld = get_field_in("cldfrac_tot").get_view(); + const auto nc = get_field_in("nc").get_view(); + + // Hack: phony field to store dz + auto dz_f = get_field_in("pseudo_density").clone("dz"); + auto dz = dz_f.get_view(); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + // TODO: move tunable constant to namelist + constexpr auto q_threshold = 0.0; + // TODO: move tunable constant to namelist + constexpr auto cldfrac_tot_threshold = 0.001; + + const auto num_levs = m_nlevs; + const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); + Kokkos::parallel_for( + "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { + const int icol = team.league_rank(); + // Subview the inputs at icol + auto tmid_icol = ekat::subview(tmid, icol); + auto pden_icol = ekat::subview(pden, icol); + auto pmid_icol = ekat::subview(pmid, icol); + auto qv_icol = ekat::subview(qv, icol); + auto qc_icol = ekat::subview(qc, icol); + auto qi_icol = ekat::subview(qi, icol); + auto rel_icol = ekat::subview(rel, icol); + auto rei_icol = ekat::subview(rei, icol); + auto cld_icol = ekat::subview(cld, icol); + auto nc_icol = ekat::subview(nc, icol); + + // We need dz too + // TODO: deduce dz_icol type, etc. on the spot here w/o field cloning? + auto dz_icol = ekat::subview(dz, icol); + PF::calculate_dz(team, pden_icol, pmid_icol, tmid_icol, qv_icol, + dz_icol); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto clr_icol = 1.0; + + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds for cldtop, but starting + // at m_nlevs-1 for cldbot + + for(int ilay : m_level_vector) { + // Only do the calculation if certain conditions are met + if((qc_icol(ilay) + qi_icol(ilay)) > q_threshold && + (cld_icol(ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + clr_icol * + (1.0 - ekat::impl::max(cld_icol(ilay - 1), cld_icol(ilay))) / + (1.0 - ekat::impl::min(cld_icol(ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = clr_icol - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = qc_icol(ilay) / (qc_icol(ilay) + qi_icol(ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + out(icol, /* T_mid */ 0) += tmid_icol(ilay) * aerocom_wts; + // p_mid_at_cldtop + out(icol, /* p_mid */ 1) += pmid_icol(ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + out(icol, /* cldfrac_ice */ 2) += (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + out(icol, /* cldfrac_liq */ 3) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc_icol(ilay) * pden_icol(ilay) / dz_icol(ilay) / + physconst::gravit / cld_icol(ilay); + out(icol, /* cdnc */ 4) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtopbot + out(icol, /* eff_radius_qc */ 5) += + rel_icol(ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtopbot + out(icol, /* eff_radius_qi */ 6) += + rei_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + clr_icol = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + out(icol, /* cldfrac_tot */ 7) = 1.0 - clr_icol; + }); +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp new file mode 100644 index 000000000000..10cc30a9aff0 --- /dev/null +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -0,0 +1,43 @@ +#ifndef EAMXX_AEROCOMCLD_DIAG +#define EAMXX_AEROCOMCLD_DIAG + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream { + +/* + * This diagnostic will compute the AeroCom diagnostics. + */ + +class AeroComCld : public AtmosphereDiagnostic { + public: + // Constructors + AeroComCld(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids( + const std::shared_ptr grids_manager) override; + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + int m_ncols; + int m_nlevs; + int m_ndiag; + + // bool for bot/top + std::string m_topbot; + + // vector of levels + std::vector m_level_vector; +}; + +} // namespace scream + +#endif // EAMXX_AEROCOMCLD_DIAG diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index bc9d0940fc44..efb55980a2fb 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -22,6 +22,7 @@ #include "diagnostics/wind_speed.hpp" #include "diagnostics/aodvis.hpp" #include "diagnostics/number_path.hpp" +#include "diagnostics/aerocom_cld.hpp" namespace scream { @@ -51,6 +52,7 @@ inline void register_diagnostics () { diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); + diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index bbaf666a247c..6b2db87df0bf 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -68,4 +68,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test "number" paths CreateDiagTest(number_paths "number_paths_tests.cpp") + # Test AEROCOM_CLOUDTOP + CreateDiagTest(aerocom_cldtop "aerocom_cld_test.cpp") + endif() diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp new file mode 100644 index 000000000000..f3ac586dc8cc --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -0,0 +1,115 @@ +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("aerocom_cldtop") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2022, 1, 1}, {0, 0, 0}); + + const auto nondim = Units::nondimensional(); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 33; + const int ngcols = 2 * comm.size(); + + int m_ndiag = 2; + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) qc, nc + FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); + FieldIdentifier nc_fid("nc", scalar2d_layout, kg / kg, grid->name()); + Field qc(qc_fid); + qc.allocate_view(); + qc.get_header().get_tracking().update_time_stamp(t0); + Field nc(nc_fid); + nc.allocate_view(); + nc.get_header().get_tracking().update_time_stamp(t0); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(0, 0.05); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + constexpr int ntests = 5; + for(int itest = 0; itest < ntests; ++itest) { + // Randomize qc, nc + randomize(qc, engine, pdf); + randomize(nc, engine, pdf); + + // Create and set up the diagnostic + ekat::ParameterList params; + auto diag = diag_factory.create("AeroComCldTop", comm, params); + diag->set_grids(gm); + diag->set_required_field(qc); + diag->set_required_field(nc); + diag->initialize(t0, RunType::Initial); + + // Run diag + diag->compute_diagnostic(); + + // Check result + qc.sync_to_host(); + nc.sync_to_host(); + diag->get_diagnostic().sync_to_host(); + + const auto qc_h = qc.get_view(); + const auto nc_h = nc.get_view(); + const auto out_hf = diag->get_diagnostic(); + + Field out_tf = diag->get_diagnostic().clone(); + out_tf.deep_copy(0.0); + auto out_t = out_tf.get_view(); + + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + for(int ilev = 0; ilev < nlevs; ++ilev) { + out_t(icol, 0) += qc_h(icol, ilev); + out_t(icol, 1) += nc_h(icol, ilev); + } + } + out_hf.sync_to_dev(); + out_tf.sync_to_dev(); + // Workaround for non-bfb behavior of view_reduction() in release builds + if(SCREAM_BFB_TESTING) { + REQUIRE(views_are_equal(out_hf, out_tf)); + } + } +} + +} // namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 2762c7fcf8aa..9e255185b42c 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1325,6 +1325,11 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "NumberPath"; // split will return the list [X, ''], with X being whatever is before 'NumberPath' params.set("Number Kind",ekat::split(diag_field_name,"NumberPath").front()); + } else if (diag_field_name=="AeroComCldTop" or + diag_field_name=="AeroComCldBot") { + diag_name = "AeroComCld"; + // split will return the list ['', X], with X being whatever is after 'AeroComCld' + params.set("AeroComCld Kind",ekat::split(diag_field_name,"AeroComCld").back()); } else if (diag_field_name=="MeridionalVapFlux" or diag_field_name=="ZonalVapFlux") { diag_name = "VaporFlux"; From 8029787a655ba983455246827893fba990502502 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 13:32:58 -0500 Subject: [PATCH 095/476] add a check to ensure top/bot only --- components/eamxx/src/diagnostics/aerocom_cld.cpp | 6 +++++- components/eamxx/src/diagnostics/tests/CMakeLists.txt | 4 ++-- .../eamxx/src/diagnostics/tests/aerocom_cld_test.cpp | 8 +++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index e878682f9580..a4ff953fbfb5 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -11,10 +11,14 @@ AeroComCld::AeroComCld(const ekat::Comm &comm, : AtmosphereDiagnostic(comm, params) { EKAT_REQUIRE_MSG( params.isParameter("AeroComCld Kind"), - "Error! NumberPathDiagnostic requires 'AeroComCld Kind' in its " + "Error! AeroComCld requires 'AeroComCld Kind' in its " "input parameters.\n"); m_topbot = m_params.get("AeroComCld Kind"); + // check if m_topbot is "Bot" or "Top", else error out + EKAT_REQUIRE_MSG(m_topbot == "Bot" || m_topbot == "Top", + "Error! AeroComCld requires 'AeroComCld Kind' " + "to be 'Bot' or 'Top' in its input parameters.\n"); } std::string AeroComCld::name() const { return "AeroComCld" + m_topbot; } diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 6b2db87df0bf..2ea6bcffd88a 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -68,7 +68,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test "number" paths CreateDiagTest(number_paths "number_paths_tests.cpp") - # Test AEROCOM_CLOUDTOP - CreateDiagTest(aerocom_cldtop "aerocom_cld_test.cpp") + # Test AEROCOM_CLD + CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index f3ac586dc8cc..5d6d50ae0d8c 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -25,7 +25,7 @@ std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, return gm; } -TEST_CASE("aerocom_cldtop") { +TEST_CASE("aerocom_cld") { using namespace ShortFieldTagsNames; using namespace ekat::units; @@ -67,6 +67,12 @@ TEST_CASE("aerocom_cldtop") { auto &diag_factory = AtmosphereDiagnosticFactory::instance(); register_diagnostics(); + REQUIRE_THROWS( + diag_factory.create("AeroComCld", comm, params)); // No 'AeroComCld Kind' + params.set("AeroComCld Kind", "Foo"); + REQUIRE_THROWS(diag_factory.create("AeroComCld", comm, + params)); // Invalid 'AeroComCld Kind' + constexpr int ntests = 5; for(int itest = 0; itest < ntests; ++itest) { // Randomize qc, nc From 91c2f73ce2b4f9c352ad2aa3add22533501d541d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 14:08:44 -0500 Subject: [PATCH 096/476] fixes and start adding the test --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 22 +++--- .../eamxx/src/diagnostics/aerocom_cld.hpp | 5 +- .../diagnostics/tests/aerocom_cld_test.cpp | 74 +++++++++++++++++-- 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index a4ff953fbfb5..78b4ad8b02a9 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -9,16 +9,15 @@ namespace scream { AeroComCld::AeroComCld(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereDiagnostic(comm, params) { - EKAT_REQUIRE_MSG( - params.isParameter("AeroComCld Kind"), - "Error! AeroComCld requires 'AeroComCld Kind' in its " - "input parameters.\n"); + EKAT_REQUIRE_MSG(params.isParameter("AeroComCld Kind"), + "Error! AeroComCld requires 'AeroComCld Kind' in its " + "input parameters.\n"); m_topbot = m_params.get("AeroComCld Kind"); // check if m_topbot is "Bot" or "Top", else error out EKAT_REQUIRE_MSG(m_topbot == "Bot" || m_topbot == "Top", - "Error! AeroComCld requires 'AeroComCld Kind' " - "to be 'Bot' or 'Top' in its input parameters.\n"); + "Error! AeroComCld requires 'AeroComCld Kind' " + "to be 'Bot' or 'Top' in its input parameters.\n"); } std::string AeroComCld::name() const { return "AeroComCld" + m_topbot; } @@ -59,10 +58,6 @@ void AeroComCld::set_grids( m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); - for(int i = 1; i < m_nlevs; ++i) { - m_level_vector[i] = m_topbot == "Top" ? i : m_nlevs - 1 - i; - } - // Self-document the outputs to parse in post-processing using stratt_t = std::map; auto d = get_diagnostic(); @@ -92,6 +87,11 @@ void AeroComCld::compute_diagnostic_impl() { using PF = scream::PhysicsFunctions; + std::vector level_vector(m_nlevs); + for(int i = 1; i < m_nlevs; ++i) { + level_vector[i] = m_topbot == "Top" ? i : m_nlevs - i; + } + const auto out = m_diagnostic_output.get_view(); // zero out the outputs @@ -151,7 +151,7 @@ void AeroComCld::compute_diagnostic_impl() { // highest is assumed to hav no clouds for cldtop, but starting // at m_nlevs-1 for cldbot - for(int ilay : m_level_vector) { + for(int ilay : level_vector) { // Only do the calculation if certain conditions are met if((qc_icol(ilay) + qi_icol(ilay)) > q_threshold && (cld_icol(ilay) > cldfrac_tot_threshold)) { diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index 10cc30a9aff0..3c5a9e67038c 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -31,11 +31,8 @@ class AeroComCld : public AtmosphereDiagnostic { int m_nlevs; int m_ndiag; - // bool for bot/top + // Bot or Top std::string m_topbot; - - // vector of levels - std::vector m_level_vector; }; } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 5d6d50ae0d8c..eb7d99a5dccc 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -36,23 +36,61 @@ TEST_CASE("aerocom_cld") { util::TimeStamp t0({2022, 1, 1}, {0, 0, 0}); const auto nondim = Units::nondimensional(); + const auto micron = m / 1000000; // Create a grids manager - single column for these tests constexpr int nlevs = 33; const int ngcols = 2 * comm.size(); - int m_ndiag = 2; + int m_ndiag = 8; auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); // Input (randomized) qc, nc FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + + // Create fields + FieldIdentifier tm_fid("T_mid", scalar2d_layout, K, grid->name()); + FieldIdentifier pd_fid("pseudo_density", scalar2d_layout, Pa, grid->name()); + FieldIdentifier pm_fid("p_mid", scalar2d_layout, Pa, grid->name()); + FieldIdentifier qv_fid("qv", scalar2d_layout, kg / kg, grid->name()); FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); + FieldIdentifier qi_fid("qi", scalar2d_layout, kg / kg, grid->name()); + FieldIdentifier ec_fid("eff_radius_qc", scalar2d_layout, micron, + grid->name()); + FieldIdentifier ei_fid("eff_radius_qi", scalar2d_layout, micron, + grid->name()); + FieldIdentifier cd_fid("cldfrac_tot", scalar2d_layout, nondim, grid->name()); FieldIdentifier nc_fid("nc", scalar2d_layout, kg / kg, grid->name()); + + Field tm(tm_fid); + tm.allocate_view(); + tm.get_header().get_tracking().update_time_stamp(t0); + Field pd(pd_fid); + pd.allocate_view(); + pd.get_header().get_tracking().update_time_stamp(t0); + Field pm(pm_fid); + pm.allocate_view(); + pm.get_header().get_tracking().update_time_stamp(t0); + Field qv(qv_fid); + qv.allocate_view(); + qv.get_header().get_tracking().update_time_stamp(t0); Field qc(qc_fid); qc.allocate_view(); qc.get_header().get_tracking().update_time_stamp(t0); + Field qi(qi_fid); + qi.allocate_view(); + qi.get_header().get_tracking().update_time_stamp(t0); + Field ec(ec_fid); + ec.allocate_view(); + ec.get_header().get_tracking().update_time_stamp(t0); + Field ei(ei_fid); + ei.allocate_view(); + ei.get_header().get_tracking().update_time_stamp(t0); + Field cd(cd_fid); + cd.allocate_view(); + cd.get_header().get_tracking().update_time_stamp(t0); Field nc(nc_fid); nc.allocate_view(); nc.get_header().get_tracking().update_time_stamp(t0); @@ -66,6 +104,7 @@ TEST_CASE("aerocom_cld") { std::map> diags; auto &diag_factory = AtmosphereDiagnosticFactory::instance(); register_diagnostics(); + ekat::ParameterList params; REQUIRE_THROWS( diag_factory.create("AeroComCld", comm, params)); // No 'AeroComCld Kind' @@ -75,16 +114,35 @@ TEST_CASE("aerocom_cld") { constexpr int ntests = 5; for(int itest = 0; itest < ntests; ++itest) { - // Randomize qc, nc + // Randomize everything + randomize(tm, engine, pdf); + randomize(pd, engine, pdf); + randomize(pm, engine, pdf); + randomize(qv, engine, pdf); randomize(qc, engine, pdf); + randomize(qi, engine, pdf); + randomize(ec, engine, pdf); + randomize(ei, engine, pdf); + randomize(cd, engine, pdf); randomize(nc, engine, pdf); // Create and set up the diagnostic - ekat::ParameterList params; - auto diag = diag_factory.create("AeroComCldTop", comm, params); + params.set("AeroComCld Kind", "Top"); + auto diag = diag_factory.create("AeroComCld", comm, params); + diag->set_grids(gm); + + diag->set_required_field(tm); + diag->set_required_field(pd); + diag->set_required_field(pm); + diag->set_required_field(qv); diag->set_required_field(qc); + diag->set_required_field(qi); + diag->set_required_field(ec); + diag->set_required_field(ei); + diag->set_required_field(cd); diag->set_required_field(nc); + diag->initialize(t0, RunType::Initial); // Run diag @@ -111,10 +169,10 @@ TEST_CASE("aerocom_cld") { } out_hf.sync_to_dev(); out_tf.sync_to_dev(); - // Workaround for non-bfb behavior of view_reduction() in release builds - if(SCREAM_BFB_TESTING) { - REQUIRE(views_are_equal(out_hf, out_tf)); - } + // // Workaround for non-bfb behavior of view_reduction() in release builds + // if(SCREAM_BFB_TESTING) { + // REQUIRE(views_are_equal(out_hf, out_tf)); + // } } } From c6a57d1dce9c55881ea904db3fb5c31433936520 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 14:34:57 -0500 Subject: [PATCH 097/476] one test, many more to go --- .../diagnostics/tests/aerocom_cld_test.cpp | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index eb7d99a5dccc..ba2ed8194122 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -145,34 +145,17 @@ TEST_CASE("aerocom_cld") { diag->initialize(t0, RunType::Initial); - // Run diag + // Case 1: if the cloud fraction is zero, everything is zero + cd.deep_copy(0.0); diag->compute_diagnostic(); - - // Check result - qc.sync_to_host(); - nc.sync_to_host(); diag->get_diagnostic().sync_to_host(); - - const auto qc_h = qc.get_view(); - const auto nc_h = nc.get_view(); - const auto out_hf = diag->get_diagnostic(); - - Field out_tf = diag->get_diagnostic().clone(); - out_tf.deep_copy(0.0); - auto out_t = out_tf.get_view(); - + Field diag_f = diag->get_diagnostic(); + auto diag_v = diag_f.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int ilev = 0; ilev < nlevs; ++ilev) { - out_t(icol, 0) += qc_h(icol, ilev); - out_t(icol, 1) += nc_h(icol, ilev); + for(int idiag = 0; idiag < m_ndiag; ++idiag) { + REQUIRE(diag_v(icol, idiag)==0.0); } } - out_hf.sync_to_dev(); - out_tf.sync_to_dev(); - // // Workaround for non-bfb behavior of view_reduction() in release builds - // if(SCREAM_BFB_TESTING) { - // REQUIRE(views_are_equal(out_hf, out_tf)); - // } } } From 6596074c990d439b147fbdbce6e9acea12af95eb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 15:19:29 -0500 Subject: [PATCH 098/476] more tests --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 21 +++++---- .../diagnostics/tests/aerocom_cld_test.cpp | 47 +++++++++++++++++-- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 78b4ad8b02a9..04b853633c72 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -87,16 +87,6 @@ void AeroComCld::compute_diagnostic_impl() { using PF = scream::PhysicsFunctions; - std::vector level_vector(m_nlevs); - for(int i = 1; i < m_nlevs; ++i) { - level_vector[i] = m_topbot == "Top" ? i : m_nlevs - i; - } - - const auto out = m_diagnostic_output.get_view(); - - // zero out the outputs - Kokkos::deep_copy(out, 0.0); - // Get the input fields const auto tmid = get_field_in("T_mid").get_view(); const auto pden = get_field_in("pseudo_density").get_view(); @@ -122,6 +112,17 @@ void AeroComCld::compute_diagnostic_impl() { const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); + + std::vector level_vector(m_nlevs); + for(int i = 1; i < m_nlevs; ++i) { + level_vector[i] = m_topbot == "Top" ? i : m_nlevs - i; + } + + const auto out = m_diagnostic_output.get_view(); + + // zero out the outputs + Kokkos::deep_copy(out, 0.0); + Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index ba2ed8194122..b2adc713a908 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -39,8 +39,8 @@ TEST_CASE("aerocom_cld") { const auto micron = m / 1000000; // Create a grids manager - single column for these tests - constexpr int nlevs = 33; - const int ngcols = 2 * comm.size(); + constexpr int nlevs = 9; + const int ngcols = 1 * comm.size(); int m_ndiag = 8; @@ -112,7 +112,7 @@ TEST_CASE("aerocom_cld") { REQUIRE_THROWS(diag_factory.create("AeroComCld", comm, params)); // Invalid 'AeroComCld Kind' - constexpr int ntests = 5; + constexpr int ntests = 2; for(int itest = 0; itest < ntests; ++itest) { // Randomize everything randomize(tm, engine, pdf); @@ -156,6 +156,47 @@ TEST_CASE("aerocom_cld") { REQUIRE(diag_v(icol, idiag)==0.0); } } + + // Case 2: if the cloud fraction is one, everything takes 1 * its value + // Take a moment to set "sensible" for other things... + cd.deep_copy(1.0); + tm.deep_copy(300.0); + pd.deep_copy(10.0); + pm.deep_copy(100.0); + qv.deep_copy(1.0); + qc.deep_copy(1.0); + qi.deep_copy(1.0); + ec.deep_copy(10.0); + ei.deep_copy(10.0); + nc.deep_copy(5.0); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + REQUIRE(diag_v(icol, 0)==300.0); + REQUIRE(diag_v(icol, 1)==100.0); + REQUIRE(diag_v(icol, 2)==0.5); + REQUIRE(diag_v(icol, 3)==0.5); + REQUIRE(diag_v(icol, 4)>0.0); + REQUIRE(diag_v(icol, 5)>0.0); + REQUIRE(diag_v(icol, 6)>0.0); + REQUIRE(diag_v(icol, 7)==1.0); + } + + // Case 3: test the max overlap (if contiguous cloudy layers, then max) + cd.deep_copy(0.0); + auto cd_v = cd.get_view(); + cd_v(0, 1) = 0.5; + cd_v(0, 2) = 0.7; // ------> max! + cd_v(0, 3) = 0.3; + cd_v(0, 4) = 0.2; + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) == 0.7); } } From 3995b2fa1fc7e7be68e1df5d08fd36dbd7476341 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 15:23:46 -0500 Subject: [PATCH 099/476] add another test --- .../diagnostics/tests/aerocom_cld_test.cpp | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index b2adc713a908..23cb0a94bd0d 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -150,10 +150,10 @@ TEST_CASE("aerocom_cld") { diag->compute_diagnostic(); diag->get_diagnostic().sync_to_host(); Field diag_f = diag->get_diagnostic(); - auto diag_v = diag_f.get_view(); + auto diag_v = diag_f.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { for(int idiag = 0; idiag < m_ndiag; ++idiag) { - REQUIRE(diag_v(icol, idiag)==0.0); + REQUIRE(diag_v(icol, idiag) == 0.0); } } @@ -174,21 +174,21 @@ TEST_CASE("aerocom_cld") { diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - REQUIRE(diag_v(icol, 0)==300.0); - REQUIRE(diag_v(icol, 1)==100.0); - REQUIRE(diag_v(icol, 2)==0.5); - REQUIRE(diag_v(icol, 3)==0.5); - REQUIRE(diag_v(icol, 4)>0.0); - REQUIRE(diag_v(icol, 5)>0.0); - REQUIRE(diag_v(icol, 6)>0.0); - REQUIRE(diag_v(icol, 7)==1.0); + REQUIRE(diag_v(icol, 0) == 300.0); + REQUIRE(diag_v(icol, 1) == 100.0); + REQUIRE(diag_v(icol, 2) == 0.5); + REQUIRE(diag_v(icol, 3) == 0.5); + REQUIRE(diag_v(icol, 4) > 0.0); + REQUIRE(diag_v(icol, 5) > 0.0); + REQUIRE(diag_v(icol, 6) > 0.0); + REQUIRE(diag_v(icol, 7) == 1.0); } // Case 3: test the max overlap (if contiguous cloudy layers, then max) cd.deep_copy(0.0); - auto cd_v = cd.get_view(); + auto cd_v = cd.get_view(); cd_v(0, 1) = 0.5; - cd_v(0, 2) = 0.7; // ------> max! + cd_v(0, 2) = 0.7; // ------> max! cd_v(0, 3) = 0.3; cd_v(0, 4) = 0.2; cd.sync_to_dev(); @@ -197,6 +197,19 @@ TEST_CASE("aerocom_cld") { diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) == 0.7); + + // Case 3xtra: test max overlap again + // This case should produce >0.7 due to slight enhancement in the presence + // of a local minimum (0.1 is the local minimum between 0.2 and 0.4) + cd_v(0, 5) = 0.1; + cd_v(0, 6) = 0.4; + cd_v(0, 7) = 0.2; + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) > 0.7); } } From d8b212b0d6063de043c14a6e9a0a75c4ff28d52f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 15:41:02 -0500 Subject: [PATCH 100/476] port all tests from rrtmpg --- .../diagnostics/tests/aerocom_cld_test.cpp | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 23cb0a94bd0d..998e332436a7 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -210,6 +210,78 @@ TEST_CASE("aerocom_cld") { diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) > 0.7); + + // Case 4: test random overlap + // If non-contiguous cloudy layers, then random + cd_v(0, 6) = 0.0; + cd_v(0, 7) = 0.1; + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) > 0.7); // must be larger than the max! + + // Case 5a: test independence of ice and liq fractions + cd_v(0, 3) = 1.0; + cd_v(0, 8) = 1.0; + cd_v(0, 9) = 0.2; + qc.deep_copy(1.0); + qi.deep_copy(0.0); // zero ice! + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) == 1.0); + REQUIRE(diag_v(0, 3) == 1.0); + REQUIRE(diag_v(0, 2) == 0.0); // zero ice! + + // Case 5b: test independence of ice and liq fractions + auto qc_v = qc.get_view(); + auto qi_v = qi.get_view(); + qc.deep_copy(0.0); // zero liq! + qi.deep_copy(1.0); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) == 1.0); + REQUIRE(diag_v(0, 3) == 0.0); // zero liq! + REQUIRE(diag_v(0, 2) == 1.0); + + // Case 6: test independence of ice and liquid fractions + // There is NOT complete independence... + // Essentially, higher ice clouds mask lower liquid clouds + // This can be problematic if the ice clouds are thin... + // We will revisit and validate this assumption later + cd.deep_copy(0.0); + cd_v(0, 1) = 0.5; // ice + cd_v(0, 2) = 0.7; // ice ------> max! + cd_v(0, 3) = 0.3; // ice + // note cd_v(0, 4) is 0.0 + cd_v(0, 5) = 0.2; // liq + cd_v(0, 6) = 0.5; // liq ------> not max! + cd_v(0, 7) = 0.1; // liq + // note cd_v(0, 8) is 0.0 + qi.deep_copy(0.0); + qi_v(0, 1) = 100; + qi_v(0, 2) = 200; + qi_v(0, 3) = 50; + // note qi_v(0, 4) = 0.0 + qc.deep_copy(0.0); + // note qc_v(0, 4) = 0.0 + qc_v(0, 5) = 20; + qc_v(0, 6) = 50; + qc_v(0, 7) = 10; + + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) > 0.7); // unaffected (see test case 4) + REQUIRE(diag_v(0, 3) < 0.5); // not max! + REQUIRE(diag_v(0, 2) == 0.7); // max! } } From 18140a702b1852e68e4afe9415dce04453f247e0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 15:42:57 -0700 Subject: [PATCH 101/476] some fixes + trying to isolate a segfault on cuda --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 11 +- .../eamxx/src/diagnostics/aerocom_cld.hpp | 2 + .../diagnostics/tests/aerocom_cld_test.cpp | 279 +++++++++--------- 3 files changed, 154 insertions(+), 138 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 04b853633c72..54a489f2238d 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -53,6 +53,11 @@ void AeroComCld::set_grids( add_field("cldfrac_tot", scalar2d_layout, nondim, grid_name); add_field("nc", scalar2d_layout, kg / kg, grid_name); + // Phony field to store dz + FieldIdentifier m_dz_fid("dz", scalar2d_layout, m, grid_name); + m_dz = Field(m_dz_fid); + m_dz.allocate_view(); + // Construct and allocate the aodvis field FieldIdentifier fid(name(), vector1d_layout, nondim, grid_name); m_diagnostic_output = Field(fid); @@ -99,9 +104,7 @@ void AeroComCld::compute_diagnostic_impl() { const auto cld = get_field_in("cldfrac_tot").get_view(); const auto nc = get_field_in("nc").get_view(); - // Hack: phony field to store dz - auto dz_f = get_field_in("pseudo_density").clone("dz"); - auto dz = dz_f.get_view(); + auto dz = m_dz.get_view(); // Get gravity acceleration constant from constants using physconst = scream::physics::Constants; @@ -110,7 +113,7 @@ void AeroComCld::compute_diagnostic_impl() { // TODO: move tunable constant to namelist constexpr auto cldfrac_tot_threshold = 0.001; - const auto num_levs = m_nlevs; + // const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); std::vector level_vector(m_nlevs); diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index 3c5a9e67038c..6cb1328db009 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -33,6 +33,8 @@ class AeroComCld : public AtmosphereDiagnostic { // Bot or Top std::string m_topbot; + + Field m_dz; }; } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 998e332436a7..222f59ff5c3d 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -104,6 +104,7 @@ TEST_CASE("aerocom_cld") { std::map> diags; auto &diag_factory = AtmosphereDiagnosticFactory::instance(); register_diagnostics(); + ekat::ParameterList params; REQUIRE_THROWS( @@ -143,145 +144,155 @@ TEST_CASE("aerocom_cld") { diag->set_required_field(cd); diag->set_required_field(nc); + tm.sync_to_dev(); + pd.sync_to_dev(); + pm.sync_to_dev(); + qv.sync_to_dev(); + qc.sync_to_dev(); + qi.sync_to_dev(); + ec.sync_to_dev(); + ei.sync_to_dev(); + cd.sync_to_dev(); + nc.sync_to_dev(); + diag->initialize(t0, RunType::Initial); // Case 1: if the cloud fraction is zero, everything is zero cd.deep_copy(0.0); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - Field diag_f = diag->get_diagnostic(); - auto diag_v = diag_f.get_view(); - for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int idiag = 0; idiag < m_ndiag; ++idiag) { - REQUIRE(diag_v(icol, idiag) == 0.0); - } - } - - // Case 2: if the cloud fraction is one, everything takes 1 * its value - // Take a moment to set "sensible" for other things... - cd.deep_copy(1.0); - tm.deep_copy(300.0); - pd.deep_copy(10.0); - pm.deep_copy(100.0); - qv.deep_copy(1.0); - qc.deep_copy(1.0); - qi.deep_copy(1.0); - ec.deep_copy(10.0); - ei.deep_copy(10.0); - nc.deep_copy(5.0); - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - REQUIRE(diag_v(icol, 0) == 300.0); - REQUIRE(diag_v(icol, 1) == 100.0); - REQUIRE(diag_v(icol, 2) == 0.5); - REQUIRE(diag_v(icol, 3) == 0.5); - REQUIRE(diag_v(icol, 4) > 0.0); - REQUIRE(diag_v(icol, 5) > 0.0); - REQUIRE(diag_v(icol, 6) > 0.0); - REQUIRE(diag_v(icol, 7) == 1.0); - } - - // Case 3: test the max overlap (if contiguous cloudy layers, then max) - cd.deep_copy(0.0); - auto cd_v = cd.get_view(); - cd_v(0, 1) = 0.5; - cd_v(0, 2) = 0.7; // ------> max! - cd_v(0, 3) = 0.3; - cd_v(0, 4) = 0.2; - cd.sync_to_dev(); - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) == 0.7); - - // Case 3xtra: test max overlap again - // This case should produce >0.7 due to slight enhancement in the presence - // of a local minimum (0.1 is the local minimum between 0.2 and 0.4) - cd_v(0, 5) = 0.1; - cd_v(0, 6) = 0.4; - cd_v(0, 7) = 0.2; - cd.sync_to_dev(); - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) > 0.7); - - // Case 4: test random overlap - // If non-contiguous cloudy layers, then random - cd_v(0, 6) = 0.0; - cd_v(0, 7) = 0.1; - cd.sync_to_dev(); - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) > 0.7); // must be larger than the max! - - // Case 5a: test independence of ice and liq fractions - cd_v(0, 3) = 1.0; - cd_v(0, 8) = 1.0; - cd_v(0, 9) = 0.2; - qc.deep_copy(1.0); - qi.deep_copy(0.0); // zero ice! - cd.sync_to_dev(); - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) == 1.0); - REQUIRE(diag_v(0, 3) == 1.0); - REQUIRE(diag_v(0, 2) == 0.0); // zero ice! - - // Case 5b: test independence of ice and liq fractions - auto qc_v = qc.get_view(); - auto qi_v = qi.get_view(); - qc.deep_copy(0.0); // zero liq! - qi.deep_copy(1.0); - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) == 1.0); - REQUIRE(diag_v(0, 3) == 0.0); // zero liq! - REQUIRE(diag_v(0, 2) == 1.0); - - // Case 6: test independence of ice and liquid fractions - // There is NOT complete independence... - // Essentially, higher ice clouds mask lower liquid clouds - // This can be problematic if the ice clouds are thin... - // We will revisit and validate this assumption later - cd.deep_copy(0.0); - cd_v(0, 1) = 0.5; // ice - cd_v(0, 2) = 0.7; // ice ------> max! - cd_v(0, 3) = 0.3; // ice - // note cd_v(0, 4) is 0.0 - cd_v(0, 5) = 0.2; // liq - cd_v(0, 6) = 0.5; // liq ------> not max! - cd_v(0, 7) = 0.1; // liq - // note cd_v(0, 8) is 0.0 - qi.deep_copy(0.0); - qi_v(0, 1) = 100; - qi_v(0, 2) = 200; - qi_v(0, 3) = 50; - // note qi_v(0, 4) = 0.0 - qc.deep_copy(0.0); - // note qc_v(0, 4) = 0.0 - qc_v(0, 5) = 20; - qc_v(0, 6) = 50; - qc_v(0, 7) = 10; - - diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); - diag_f = diag->get_diagnostic(); - diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) > 0.7); // unaffected (see test case 4) - REQUIRE(diag_v(0, 3) < 0.5); // not max! - REQUIRE(diag_v(0, 2) == 0.7); // max! + // Field diag_f = diag->get_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // // diag_f.sync_to_host(); + // auto diag_v = diag_f.get_view(); + // for(int idiag = 0; idiag < m_ndiag; ++idiag) { + // REQUIRE(diag_v(0, idiag) == 0.0); + // } + + // // Case 2: if the cloud fraction is one, everything takes 1 * its value + // // Take a moment to set "sensible" for other things... + // cd.deep_copy(1.0); + // tm.deep_copy(300.0); + // pd.deep_copy(10.0); + // pm.deep_copy(100.0); + // qv.deep_copy(1.0); + // qc.deep_copy(1.0); + // qi.deep_copy(1.0); + // ec.deep_copy(10.0); + // ei.deep_copy(10.0); + // nc.deep_copy(5.0); + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // REQUIRE(diag_v(icol, 0) == 300.0); + // REQUIRE(diag_v(icol, 1) == 100.0); + // REQUIRE(diag_v(icol, 2) == 0.5); + // REQUIRE(diag_v(icol, 3) == 0.5); + // REQUIRE(diag_v(icol, 4) > 0.0); + // REQUIRE(diag_v(icol, 5) > 0.0); + // REQUIRE(diag_v(icol, 6) > 0.0); + // REQUIRE(diag_v(icol, 7) == 1.0); + // } + + // // Case 3: test the max overlap (if contiguous cloudy layers, then max) + // cd.deep_copy(0.0); + // auto cd_v = cd.get_view(); + // cd_v(0, 1) = 0.5; + // cd_v(0, 2) = 0.7; // ------> max! + // cd_v(0, 3) = 0.3; + // cd_v(0, 4) = 0.2; + // cd.sync_to_dev(); + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // REQUIRE(diag_v(0, 7) == 0.7); + + // // Case 3xtra: test max overlap again + // // This case should produce >0.7 due to slight enhancement in the presence + // // of a local minimum (0.1 is the local minimum between 0.2 and 0.4) + // cd_v(0, 5) = 0.1; + // cd_v(0, 6) = 0.4; + // cd_v(0, 7) = 0.2; + // cd.sync_to_dev(); + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // REQUIRE(diag_v(0, 7) > 0.7); + + // // Case 4: test random overlap + // // If non-contiguous cloudy layers, then random + // cd_v(0, 6) = 0.0; + // cd_v(0, 7) = 0.1; + // cd.sync_to_dev(); + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // REQUIRE(diag_v(0, 7) > 0.7); // must be larger than the max! + + // // Case 5a: test independence of ice and liq fractions + // cd_v(0, 3) = 1.0; + // cd_v(0, 8) = 1.0; + // cd_v(0, 9) = 0.2; + // qc.deep_copy(1.0); + // qi.deep_copy(0.0); // zero ice! + // cd.sync_to_dev(); + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // REQUIRE(diag_v(0, 7) == 1.0); + // REQUIRE(diag_v(0, 3) == 1.0); + // REQUIRE(diag_v(0, 2) == 0.0); // zero ice! + + // // Case 5b: test independence of ice and liq fractions + // auto qc_v = qc.get_view(); + // auto qi_v = qi.get_view(); + // qc.deep_copy(0.0); // zero liq! + // qi.deep_copy(1.0); + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // REQUIRE(diag_v(0, 7) == 1.0); + // REQUIRE(diag_v(0, 3) == 0.0); // zero liq! + // REQUIRE(diag_v(0, 2) == 1.0); + + // // Case 6: test independence of ice and liquid fractions + // // There is NOT complete independence... + // // Essentially, higher ice clouds mask lower liquid clouds + // // This can be problematic if the ice clouds are thin... + // // We will revisit and validate this assumption later + // cd.deep_copy(0.0); + // cd_v(0, 1) = 0.5; // ice + // cd_v(0, 2) = 0.7; // ice ------> max! + // cd_v(0, 3) = 0.3; // ice + // // note cd_v(0, 4) is 0.0 + // cd_v(0, 5) = 0.2; // liq + // cd_v(0, 6) = 0.5; // liq ------> not max! + // cd_v(0, 7) = 0.1; // liq + // // note cd_v(0, 8) is 0.0 + // qi.deep_copy(0.0); + // qi_v(0, 1) = 100; + // qi_v(0, 2) = 200; + // qi_v(0, 3) = 50; + // // note qi_v(0, 4) = 0.0 + // qc.deep_copy(0.0); + // // note qc_v(0, 4) = 0.0 + // qc_v(0, 5) = 20; + // qc_v(0, 6) = 50; + // qc_v(0, 7) = 10; + + // diag->compute_diagnostic(); + // diag->get_diagnostic().sync_to_host(); + // diag_f = diag->get_diagnostic(); + // diag_v = diag_f.get_view(); + // REQUIRE(diag_v(0, 7) > 0.7); // unaffected (see test case 4) + // REQUIRE(diag_v(0, 3) < 0.5); // not max! + // REQUIRE(diag_v(0, 2) == 0.7); // max! } } From a8f251d9a22bc302141dd152cd1d05508a605887 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 17:00:36 -0700 Subject: [PATCH 102/476] fixes to cuda segfault, reorg --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 50 +-- .../diagnostics/tests/aerocom_cld_test.cpp | 300 +++++++++--------- 2 files changed, 178 insertions(+), 172 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 54a489f2238d..7e1b5a1efdec 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -53,12 +53,12 @@ void AeroComCld::set_grids( add_field("cldfrac_tot", scalar2d_layout, nondim, grid_name); add_field("nc", scalar2d_layout, kg / kg, grid_name); - // Phony field to store dz + // A field to store dz FieldIdentifier m_dz_fid("dz", scalar2d_layout, m, grid_name); m_dz = Field(m_dz_fid); m_dz.allocate_view(); - // Construct and allocate the aodvis field + // Construct and allocate the output field FieldIdentifier fid(name(), vector1d_layout, nondim, grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); @@ -83,7 +83,7 @@ void AeroComCld::compute_diagnostic_impl() { * based on the AeroCom recommendation. See reference for routine * get_subcolumn_mask in rrtmpg, where equation 14 is used for the * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. The logic is reversed to calculate + * equation 13, the column counterpart. The order is reversed to calculate * cloud-bottom properties as well. */ using KT = KokkosTypes; @@ -104,7 +104,7 @@ void AeroComCld::compute_diagnostic_impl() { const auto cld = get_field_in("cldfrac_tot").get_view(); const auto nc = get_field_in("nc").get_view(); - auto dz = m_dz.get_view(); + auto dz = m_dz.get_view(); // Get gravity acceleration constant from constants using physconst = scream::physics::Constants; @@ -113,19 +113,16 @@ void AeroComCld::compute_diagnostic_impl() { // TODO: move tunable constant to namelist constexpr auto cldfrac_tot_threshold = 0.001; - // const auto num_levs = m_nlevs; - const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); - - std::vector level_vector(m_nlevs); - for(int i = 1; i < m_nlevs; ++i) { - level_vector[i] = m_topbot == "Top" ? i : m_nlevs - i; - } + const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); const auto out = m_diagnostic_output.get_view(); // zero out the outputs Kokkos::deep_copy(out, 0.0); + const int nlevs = m_nlevs; + bool is_top = (m_topbot == "Top"); + Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); @@ -155,13 +152,13 @@ void AeroComCld::compute_diagnostic_impl() { // highest is assumed to hav no clouds for cldtop, but starting // at m_nlevs-1 for cldbot - for(int ilay : level_vector) { + auto topbot_calcs = [&](int ilay) { // Only do the calculation if certain conditions are met if((qc_icol(ilay) + qi_icol(ilay)) > q_threshold && (cld_icol(ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ + /* PART I: Probabilistically determining cloud top/bot */ // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of + // probability of this level, where clr_icol is that of // the previous level auto aerocom_tmp = clr_icol * @@ -176,34 +173,39 @@ void AeroComCld::compute_diagnostic_impl() { /* In general, converting a 3D property X to a 2D cloud-top * counterpart x follows: x(i) += X(i,k) * weights * Phase * but X and Phase are not always needed */ - // T_mid_at_cldtop out(icol, /* T_mid */ 0) += tmid_icol(ilay) * aerocom_wts; - // p_mid_at_cldtop out(icol, /* p_mid */ 1) += pmid_icol(ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop out(icol, /* cldfrac_ice */ 2) += (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop out(icol, /* cldfrac_liq */ 3) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop + // cdnc /* We need to convert nc from 1/mass to 1/volume first, and * from grid-mean to in-cloud, but after that, the * calculation follows the general logic */ auto cdnc = nc_icol(ilay) * pden_icol(ilay) / dz_icol(ilay) / physconst::gravit / cld_icol(ilay); out(icol, /* cdnc */ 4) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtopbot out(icol, /* eff_radius_qc */ 5) += rel_icol(ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtopbot out(icol, /* eff_radius_qi */ 6) += rei_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate + // Reset clr_icol to aerocom_tmp to accumulate clr_icol = aerocom_tmp; } + }; + + if(is_top) { + for(int ilay = 1; ilay < nlevs; ++ilay) { + topbot_calcs(ilay); + } + } else { + for(int ilay = nlevs - 1; ilay > 0; --ilay) { + topbot_calcs(ilay); + } } + // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities + // defined as (1 - clr_icol). This is true because + // clr_icol is the result of accumulative probabilities // (their products) out(icol, /* cldfrac_tot */ 7) = 1.0 - clr_icol; }); diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 222f59ff5c3d..39276496ecc0 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -33,7 +33,7 @@ TEST_CASE("aerocom_cld") { ekat::Comm comm(MPI_COMM_WORLD); // A time stamp - util::TimeStamp t0({2022, 1, 1}, {0, 0, 0}); + util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); const auto nondim = Units::nondimensional(); const auto micron = m / 1000000; @@ -47,7 +47,7 @@ TEST_CASE("aerocom_cld") { auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); - // Input (randomized) qc, nc + // Input FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; // Create fields @@ -113,9 +113,9 @@ TEST_CASE("aerocom_cld") { REQUIRE_THROWS(diag_factory.create("AeroComCld", comm, params)); // Invalid 'AeroComCld Kind' - constexpr int ntests = 2; + constexpr int ntests = 3; for(int itest = 0; itest < ntests; ++itest) { - // Randomize everything + // Randomize everything to add ensure resiliency randomize(tm, engine, pdf); randomize(pd, engine, pdf); randomize(pm, engine, pdf); @@ -144,155 +144,159 @@ TEST_CASE("aerocom_cld") { diag->set_required_field(cd); diag->set_required_field(nc); - tm.sync_to_dev(); - pd.sync_to_dev(); - pm.sync_to_dev(); - qv.sync_to_dev(); - qc.sync_to_dev(); - qi.sync_to_dev(); - ec.sync_to_dev(); - ei.sync_to_dev(); - cd.sync_to_dev(); - nc.sync_to_dev(); - + // tm.sync_to_dev(); + // pd.sync_to_dev(); + // pm.sync_to_dev(); + // qv.sync_to_dev(); + // qc.sync_to_dev(); + // qi.sync_to_dev(); + // ec.sync_to_dev(); + // ei.sync_to_dev(); + // cd.sync_to_dev(); + // nc.sync_to_dev(); + diag->initialize(t0, RunType::Initial); // Case 1: if the cloud fraction is zero, everything is zero cd.deep_copy(0.0); diag->compute_diagnostic(); - // Field diag_f = diag->get_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // // diag_f.sync_to_host(); - // auto diag_v = diag_f.get_view(); - // for(int idiag = 0; idiag < m_ndiag; ++idiag) { - // REQUIRE(diag_v(0, idiag) == 0.0); - // } - - // // Case 2: if the cloud fraction is one, everything takes 1 * its value - // // Take a moment to set "sensible" for other things... - // cd.deep_copy(1.0); - // tm.deep_copy(300.0); - // pd.deep_copy(10.0); - // pm.deep_copy(100.0); - // qv.deep_copy(1.0); - // qc.deep_copy(1.0); - // qi.deep_copy(1.0); - // ec.deep_copy(10.0); - // ei.deep_copy(10.0); - // nc.deep_copy(5.0); - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - // REQUIRE(diag_v(icol, 0) == 300.0); - // REQUIRE(diag_v(icol, 1) == 100.0); - // REQUIRE(diag_v(icol, 2) == 0.5); - // REQUIRE(diag_v(icol, 3) == 0.5); - // REQUIRE(diag_v(icol, 4) > 0.0); - // REQUIRE(diag_v(icol, 5) > 0.0); - // REQUIRE(diag_v(icol, 6) > 0.0); - // REQUIRE(diag_v(icol, 7) == 1.0); - // } - - // // Case 3: test the max overlap (if contiguous cloudy layers, then max) - // cd.deep_copy(0.0); - // auto cd_v = cd.get_view(); - // cd_v(0, 1) = 0.5; - // cd_v(0, 2) = 0.7; // ------> max! - // cd_v(0, 3) = 0.3; - // cd_v(0, 4) = 0.2; - // cd.sync_to_dev(); - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // REQUIRE(diag_v(0, 7) == 0.7); - - // // Case 3xtra: test max overlap again - // // This case should produce >0.7 due to slight enhancement in the presence - // // of a local minimum (0.1 is the local minimum between 0.2 and 0.4) - // cd_v(0, 5) = 0.1; - // cd_v(0, 6) = 0.4; - // cd_v(0, 7) = 0.2; - // cd.sync_to_dev(); - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // REQUIRE(diag_v(0, 7) > 0.7); - - // // Case 4: test random overlap - // // If non-contiguous cloudy layers, then random - // cd_v(0, 6) = 0.0; - // cd_v(0, 7) = 0.1; - // cd.sync_to_dev(); - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // REQUIRE(diag_v(0, 7) > 0.7); // must be larger than the max! - - // // Case 5a: test independence of ice and liq fractions - // cd_v(0, 3) = 1.0; - // cd_v(0, 8) = 1.0; - // cd_v(0, 9) = 0.2; - // qc.deep_copy(1.0); - // qi.deep_copy(0.0); // zero ice! - // cd.sync_to_dev(); - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // REQUIRE(diag_v(0, 7) == 1.0); - // REQUIRE(diag_v(0, 3) == 1.0); - // REQUIRE(diag_v(0, 2) == 0.0); // zero ice! - - // // Case 5b: test independence of ice and liq fractions - // auto qc_v = qc.get_view(); - // auto qi_v = qi.get_view(); - // qc.deep_copy(0.0); // zero liq! - // qi.deep_copy(1.0); - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // REQUIRE(diag_v(0, 7) == 1.0); - // REQUIRE(diag_v(0, 3) == 0.0); // zero liq! - // REQUIRE(diag_v(0, 2) == 1.0); - - // // Case 6: test independence of ice and liquid fractions - // // There is NOT complete independence... - // // Essentially, higher ice clouds mask lower liquid clouds - // // This can be problematic if the ice clouds are thin... - // // We will revisit and validate this assumption later - // cd.deep_copy(0.0); - // cd_v(0, 1) = 0.5; // ice - // cd_v(0, 2) = 0.7; // ice ------> max! - // cd_v(0, 3) = 0.3; // ice - // // note cd_v(0, 4) is 0.0 - // cd_v(0, 5) = 0.2; // liq - // cd_v(0, 6) = 0.5; // liq ------> not max! - // cd_v(0, 7) = 0.1; // liq - // // note cd_v(0, 8) is 0.0 - // qi.deep_copy(0.0); - // qi_v(0, 1) = 100; - // qi_v(0, 2) = 200; - // qi_v(0, 3) = 50; - // // note qi_v(0, 4) = 0.0 - // qc.deep_copy(0.0); - // // note qc_v(0, 4) = 0.0 - // qc_v(0, 5) = 20; - // qc_v(0, 6) = 50; - // qc_v(0, 7) = 10; - - // diag->compute_diagnostic(); - // diag->get_diagnostic().sync_to_host(); - // diag_f = diag->get_diagnostic(); - // diag_v = diag_f.get_view(); - // REQUIRE(diag_v(0, 7) > 0.7); // unaffected (see test case 4) - // REQUIRE(diag_v(0, 3) < 0.5); // not max! - // REQUIRE(diag_v(0, 2) == 0.7); // max! + Field diag_f = diag->get_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f.sync_to_host(); + auto diag_v = diag_f.get_view(); + for(int idiag = 0; idiag < m_ndiag; ++idiag) { + REQUIRE(diag_v(0, idiag) == 0.0); + } + + // Case 2: if the cloud fraction is one, everything takes 1 * its value + // Take a moment to set "sensible" for other things... + cd.deep_copy(1.0); + tm.deep_copy(300.0); + pd.deep_copy(10.0); + pm.deep_copy(100.0); + qv.deep_copy(1.0); + qc.deep_copy(1.0); + qi.deep_copy(1.0); + ec.deep_copy(10.0); + ei.deep_copy(10.0); + nc.deep_copy(5.0); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + REQUIRE(diag_v(icol, 0) == 300.0); + REQUIRE(diag_v(icol, 1) == 100.0); + REQUIRE(diag_v(icol, 2) == 0.5); + REQUIRE(diag_v(icol, 3) == 0.5); + REQUIRE(diag_v(icol, 4) > 0.0); + REQUIRE(diag_v(icol, 5) > 0.0); + REQUIRE(diag_v(icol, 6) > 0.0); + REQUIRE(diag_v(icol, 7) == 1.0); + } + + // Case 3: test the max overlap (if contiguous cloudy layers, then max) + cd.deep_copy(0.0); + auto cd_v = cd.get_view(); + cd_v(0, 1) = 0.5; + cd_v(0, 2) = 0.7; // ------> max! + cd_v(0, 3) = 0.3; + cd_v(0, 4) = 0.2; + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) == 0.7); + + // Case 3xtra: test max overlap again + // This case should produce >0.7 due to slight enhancement in the presence + // of a local minimum (0.1 is the local minimum between 0.2 and 0.4) + cd_v(0, 5) = 0.1; + cd_v(0, 6) = 0.4; + cd_v(0, 7) = 0.2; + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) > 0.7); + + // Case 4: test random overlap + // If non-contiguous cloudy layers, then random + cd_v(0, 6) = 0.0; + cd_v(0, 7) = 0.1; + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) > 0.7); // must be larger than the max! + + // Case 5a: test independence of ice and liq fractions + cd_v(0, 3) = 1.0; + cd_v(0, 8) = 1.0; + cd_v(0, 9) = 0.2; + qc.deep_copy(1.0); + qi.deep_copy(0.0); // zero ice! + cd.sync_to_dev(); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) == 1.0); + REQUIRE(diag_v(0, 3) == 1.0); + REQUIRE(diag_v(0, 2) == 0.0); // zero ice! + + // Case 5b: test independence of ice and liq fractions + qc.deep_copy(0.0); // zero liq! + qi.deep_copy(1.0); + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) == 1.0); + REQUIRE(diag_v(0, 3) == 0.0); // zero liq! + REQUIRE(diag_v(0, 2) == 1.0); + + // Case 6: test independence of ice and liquid fractions + // There is NOT complete independence... + // Essentially, higher ice clouds mask lower liquid clouds + // This can be problematic if the ice clouds are thin... + // We will revisit and validate this assumption later + auto qc_v = qc.get_view(); + auto qi_v = qi.get_view(); + cd.deep_copy(0.0); + cd_v(0, 1) = 0.5; // ice + cd_v(0, 2) = 0.7; // ice ------> max! + cd_v(0, 3) = 0.3; // ice + // note cd_v(0, 4) is 0.0 + cd_v(0, 5) = 0.2; // liq + cd_v(0, 6) = 0.5; // liq ------> not max! + cd_v(0, 7) = 0.1; // liq + // note cd_v(0, 8) is 0.0 + qi.deep_copy(0.0); + qi_v(0, 1) = 100; + qi_v(0, 2) = 200; + qi_v(0, 3) = 50; + // note qi_v(0, 4) = 0.0 + qc.deep_copy(0.0); + // note qc_v(0, 4) = 0.0 + qc_v(0, 5) = 20; + qc_v(0, 6) = 50; + qc_v(0, 7) = 10; + + cd.sync_to_dev(); + qi.sync_to_dev(); + qc.sync_to_dev(); + + diag->compute_diagnostic(); + diag->get_diagnostic().sync_to_host(); + diag_f = diag->get_diagnostic(); + diag_v = diag_f.get_view(); + REQUIRE(diag_v(0, 7) > 0.7); // unaffected (see test case 4) + REQUIRE(diag_v(0, 3) < 0.5); // not max! + REQUIRE(diag_v(0, 2) == 0.7); // max! } } From 7360e9e6bee5918755434760e159c27b853c3531 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 30 Apr 2024 17:27:53 -0700 Subject: [PATCH 103/476] fix double/float issues in sp --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 2 +- .../diagnostics/tests/aerocom_cld_test.cpp | 60 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 7e1b5a1efdec..19c08a5070c3 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -164,7 +164,7 @@ void AeroComCld::compute_diagnostic_impl() { clr_icol * (1.0 - ekat::impl::max(cld_icol(ilay - 1), cld_icol(ilay))) / (1.0 - ekat::impl::min(cld_icol(ilay - 1), - 1.0 - cldfrac_tot_threshold)); + Real(1.0 - cldfrac_tot_threshold))); // Temporary variable for probability "weights" auto aerocom_wts = clr_icol - aerocom_tmp; // Temporary variable for liquid "phase" diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 39276496ecc0..ae4d2e1d3389 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -144,16 +144,16 @@ TEST_CASE("aerocom_cld") { diag->set_required_field(cd); diag->set_required_field(nc); - // tm.sync_to_dev(); - // pd.sync_to_dev(); - // pm.sync_to_dev(); - // qv.sync_to_dev(); - // qc.sync_to_dev(); - // qi.sync_to_dev(); - // ec.sync_to_dev(); - // ei.sync_to_dev(); - // cd.sync_to_dev(); - // nc.sync_to_dev(); + tm.sync_to_dev(); + pd.sync_to_dev(); + pm.sync_to_dev(); + qv.sync_to_dev(); + qc.sync_to_dev(); + qi.sync_to_dev(); + ec.sync_to_dev(); + ei.sync_to_dev(); + cd.sync_to_dev(); + nc.sync_to_dev(); diag->initialize(t0, RunType::Initial); @@ -185,14 +185,14 @@ TEST_CASE("aerocom_cld") { diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - REQUIRE(diag_v(icol, 0) == 300.0); - REQUIRE(diag_v(icol, 1) == 100.0); - REQUIRE(diag_v(icol, 2) == 0.5); - REQUIRE(diag_v(icol, 3) == 0.5); - REQUIRE(diag_v(icol, 4) > 0.0); - REQUIRE(diag_v(icol, 5) > 0.0); - REQUIRE(diag_v(icol, 6) > 0.0); - REQUIRE(diag_v(icol, 7) == 1.0); + REQUIRE(diag_v(icol, 0) == Real(300.0)); + REQUIRE(diag_v(icol, 1) == Real(100.0)); + REQUIRE(diag_v(icol, 2) == Real(0.5)); + REQUIRE(diag_v(icol, 3) == Real(0.5)); + REQUIRE(diag_v(icol, 4) > Real(0.0)); + REQUIRE(diag_v(icol, 5) > Real(0.0)); + REQUIRE(diag_v(icol, 6) > Real(0.0)); + REQUIRE(diag_v(icol, 7) == Real(1.0)); } // Case 3: test the max overlap (if contiguous cloudy layers, then max) @@ -207,7 +207,7 @@ TEST_CASE("aerocom_cld") { diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) == 0.7); + REQUIRE(diag_v(0, 7) == Real(0.7)); // Case 3xtra: test max overlap again // This case should produce >0.7 due to slight enhancement in the presence @@ -220,7 +220,7 @@ TEST_CASE("aerocom_cld") { diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) > 0.7); + REQUIRE(diag_v(0, 7) > Real(0.7)); // Case 4: test random overlap // If non-contiguous cloudy layers, then random @@ -231,7 +231,7 @@ TEST_CASE("aerocom_cld") { diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) > 0.7); // must be larger than the max! + REQUIRE(diag_v(0, 7) > Real(0.7)); // must be larger than the max! // Case 5a: test independence of ice and liq fractions cd_v(0, 3) = 1.0; @@ -244,9 +244,9 @@ TEST_CASE("aerocom_cld") { diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) == 1.0); - REQUIRE(diag_v(0, 3) == 1.0); - REQUIRE(diag_v(0, 2) == 0.0); // zero ice! + REQUIRE(diag_v(0, 7) == Real(1.0)); + REQUIRE(diag_v(0, 3) == Real(1.0)); + REQUIRE(diag_v(0, 2) == Real(0.0)); // zero ice! // Case 5b: test independence of ice and liq fractions qc.deep_copy(0.0); // zero liq! @@ -255,9 +255,9 @@ TEST_CASE("aerocom_cld") { diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) == 1.0); - REQUIRE(diag_v(0, 3) == 0.0); // zero liq! - REQUIRE(diag_v(0, 2) == 1.0); + REQUIRE(diag_v(0, 7) == Real(1.0)); + REQUIRE(diag_v(0, 3) == Real(0.0)); // zero liq! + REQUIRE(diag_v(0, 2) == Real(1.0)); // Case 6: test independence of ice and liquid fractions // There is NOT complete independence... @@ -294,9 +294,9 @@ TEST_CASE("aerocom_cld") { diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); diag_v = diag_f.get_view(); - REQUIRE(diag_v(0, 7) > 0.7); // unaffected (see test case 4) - REQUIRE(diag_v(0, 3) < 0.5); // not max! - REQUIRE(diag_v(0, 2) == 0.7); // max! + REQUIRE(diag_v(0, 7) > Real(0.7)); // unaffected (see test case 4) + REQUIRE(diag_v(0, 3) < Real(0.5)); // not max! + REQUIRE(diag_v(0, 2) == Real(0.7)); // max! } } From a3d251d76176e471bf9d1cd61ce4a9a3498a56c9 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 1 May 2024 01:29:30 -0400 Subject: [PATCH 104/476] update nudging docs + fix bibtex --- .github/workflows/eamxx-gh-pages.yml | 2 +- components/eamxx/docs/refs/aerocom_cldtop.bib | 24 ++++++++++++++ .../eamxx/docs/{refs.bib => refs/general.bib} | 24 -------------- components/eamxx/docs/user/nudging.md | 32 +++++++++++++++---- components/eamxx/mkdocs.yml | 4 +++ 5 files changed, 54 insertions(+), 32 deletions(-) create mode 100644 components/eamxx/docs/refs/aerocom_cldtop.bib rename components/eamxx/docs/{refs.bib => refs/general.bib} (87%) diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index 91f591a32ba6..a78f76c9c071 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -60,7 +60,7 @@ jobs: - name: Install Python deps run: | - pip install mkdocs pymdown-extensions mkdocs-material mdutils + pip install mkdocs pymdown-extensions mkdocs-material mdutils mkdocs-bibtex - name: Generate EAMxx params docs working-directory: components/eamxx/scripts diff --git a/components/eamxx/docs/refs/aerocom_cldtop.bib b/components/eamxx/docs/refs/aerocom_cldtop.bib new file mode 100644 index 000000000000..45ed58088600 --- /dev/null +++ b/components/eamxx/docs/refs/aerocom_cldtop.bib @@ -0,0 +1,24 @@ + +@techreport{tiedtke_ecmwf_1979, + address = {Shinfield Park, Reading}, + type = {Technical {Report}}, + title = {{ECMWF} model parameterisation of sub-grid scale processes}, + language = {en}, + institution = {ECMWF}, + author = {Tiedtke, M. and Geleyn, J.-F. and Hollingsworth, A. and Louis, J.-F.}, + month = jan, + year = {1979}, + note = {10}, + pages = {146}, +} + +@article{raisanen2004stochastic, + title={Stochastic generation of subgrid-scale cloudy columns for large-scale models}, + author={R{\"a}is{\"a}nen, Petri and Barker, Howard W and Khairoutdinov, Marat F and Li, Jiangnan and Randall, David A}, + journal={Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography}, + volume={130}, + number={601}, + pages={2047--2067}, + year={2004}, + publisher={Wiley Online Library} +} diff --git a/components/eamxx/docs/refs.bib b/components/eamxx/docs/refs/general.bib similarity index 87% rename from components/eamxx/docs/refs.bib rename to components/eamxx/docs/refs/general.bib index 7a8745e07096..6ab4e21c9173 100644 --- a/components/eamxx/docs/refs.bib +++ b/components/eamxx/docs/refs/general.bib @@ -123,27 +123,3 @@ @article{Taylor_et20 note = {e2019MS001783 10.1029/2019MS001783}, year = {2020} } - -@techreport{tiedtke_ecmwf_1979, - address = {Shinfield Park, Reading}, - type = {Technical {Report}}, - title = {{ECMWF} model parameterisation of sub-grid scale processes}, - language = {en}, - institution = {ECMWF}, - author = {Tiedtke, M. and Geleyn, J.-F. and Hollingsworth, A. and Louis, J.-F.}, - month = jan, - year = {1979}, - note = {10}, - pages = {146}, -} - -@article{raisanen2004stochastic, - title={Stochastic generation of subgrid-scale cloudy columns for large-scale models}, - author={R{\"a}is{\"a}nen, Petri and Barker, Howard W and Khairoutdinov, Marat F and Li, Jiangnan and Randall, David A}, - journal={Quarterly Journal of the Royal Meteorological Society: A journal of the atmospheric sciences, applied meteorology and physical oceanography}, - volume={130}, - number={601}, - pages={2047--2067}, - year={2004}, - publisher={Wiley Online Library} -} diff --git a/components/eamxx/docs/user/nudging.md b/components/eamxx/docs/user/nudging.md index acb72468d97c..e6137e666bea 100644 --- a/components/eamxx/docs/user/nudging.md +++ b/components/eamxx/docs/user/nudging.md @@ -1,24 +1,42 @@ # Nudging in EAMxx Nudging is supported in EAMxx. -Currently, it is possible to nudge EAMxx to the output from a different EAMxx run or to reanalysis. Nudging data can be on your model grid or an arbitrary coarser grid. Inline interpolating of finer-grid nudging data to a coarser model resolution isn't implemented yet but may be in the future. +Currently, it is possible to nudge EAMxx to the output from a different EAMxx run or to reanalysis. Nudging data can be on your model grid or an arbitrary coarser grid. Inline interpolating of finer-grid nudging data to a coarser model resolution isn't implemented yet but may be in the future. + +## Data to nudge towards + +The user is expected to prepapre (and then use `atmchange` to point to) nudging data files that are compliant with EAMxx specification. +In practice, this means that the data files must contain variable names known to EAMxx (only U, V, T_mid, and qv are supported now). +The files can be specified with an explicit list or via pattern matching. +The files must contain an arbitary global attribute `case_t0`, and it is recommended to be the same as the time dimension unit (the files must be time-dimensioned). +Finally, the dimension order must be such that `lev` is the last dimension, so most likely, the user must transpose the dimensions. + +## Pressure in the nudging data + +Pressure can be explicitly provided in the nudging data as time-varying `p_mid` corresponding to the option `TIME_DEPENDENT_3D_PROFILE` for `source_pressure_type`. +Alternatively, the data can contain a time-invariant pressure variable `p_lev` corresponding to the option `TIME_DEPENDENT_3D_PROFILE` for `source_pressure_type`. + +## Weighted nudging for RRM applications + +In regionally refined model applications, it is possible to use weighted nudging, for example, to avoid nudging the refined region. +To achieve that, the user can use `atmchange` to set `use_nudging_weights` (boolean) and provide `nudging_weights_file` that has the weight to apply for nudging (for example, zeros in the refined region). +Currently, weighted nudging is only supported if the user provides the nudging data at the target grid. ## Example setup (current as of April 2024) To enable nudging as a process, one must declare it in the `atm_procs_list` runtime parameter. ```shell - ./atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp,nudging" +./atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp,nudging" ``` The following options are needed to specify the nudging. ```shell -./atmchange nudging::nudging_filenames_patterns="/pathto/nudging_files/*.nc" -./atmchange nudging::source_pressure_type=TIME_DEPENDENT_3D_PROFILE -./atmchange nudging::nudging_fields=U,V -./atmchange nudging::nudging_timescale=21600 +./atmchange nudging::nudging_filenames_patterns="/pathto/nudging_files/*.nc" # can provide file name explicitly here instead (or multiple patterns) +./atmchange nudging::source_pressure_type=TIME_DEPENDENT_3D_PROFILE # see section on pressure +./atmchange nudging::nudging_fields=U,V # can include qv, T_mid as well +./atmchange nudging::nudging_timescale=21600 # in seconds ``` To gain a deeper understanding of these parameters and options, please refer to code implementation of the nudging process. -In particular, depending on the specific needs, one may utilize weighted nudging (for regionally refined model runs) or nudging from coarse data via specifying a map in the `atmchange` calls above. diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index 770362671d34..609bd346bf4b 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -66,3 +66,7 @@ extra_javascript: - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js repo_url: https://github.com/E3SM-Project/scream + +plugins: + - bibtex: + bib_dir: refs \ No newline at end of file From 7a53ecc42ab48c38f75e62c13316ef0a80b6e36b Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 01:43:03 -0400 Subject: [PATCH 105/476] updates to get bibtex working correctly --- .github/workflows/eamxx-gh-pages.yml | 4 ++-- components/eamxx/mkdocs.yml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index a78f76c9c071..abd8b92e4f6c 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -53,10 +53,10 @@ jobs: run: | echo "= The job was automatically triggered by a ${{github.event_name}} event." - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v5.1.0 with: - python-version: "3.10" + python-version: "3.11" - name: Install Python deps run: | diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index 609bd346bf4b..ccf72c080392 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -59,6 +59,7 @@ markdown_extensions: alternate_style: true - pymdownx.arithmatex: generic: true + - footnotes extra_javascript: # - javascript/mathjax.js @@ -68,5 +69,6 @@ extra_javascript: repo_url: https://github.com/E3SM-Project/scream plugins: + - search - bibtex: - bib_dir: refs \ No newline at end of file + bib_dir: docs/refs From 2e7b80ee8fc9abdfcbc1578520abba29d600853e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 29 Apr 2024 16:45:02 -0600 Subject: [PATCH 106/476] EAMxx: add documentation for model inputs --- components/eamxx/docs/user/model_input.md | 267 +++++++++++++++++++++- 1 file changed, 264 insertions(+), 3 deletions(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index 38486c77a21e..702bf7c89b72 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -1,8 +1,269 @@ -Model input +Model inputs ===================================== -TODO: explain how defaults XML, atmchange/atmquery, buildml, and input.yaml work. +This section explains how input parameters are passed to EAMxx, and how the user can +change their value. The full list of the currently configuraable runtime parameters for +EAMxx can be found [here](../common/eamxx_params.md). -[Here](../common/eamxx_params.md) is a list of the currently configurable runtime parameters for EAMxx. +The main files/scripts involved in the EAMxx runtime configuration are the following: +1. `buildnml`: this script, located in `components/eamxx/cime_config`, is called by CIME's case management + scripts (`case.setup`, `case.build, `case.submit`), and is responsible for creating EAMxx input files, + including handling of user-specific modifications for the runtime parameters. Users should not have to + modify this script, nor should they have to manually call it. +2. `scream_input.yaml`: EAMxx will parse this [YAML](https://yaml.org/spec/1.2.2) file at runtime, + to detect all of its configuration parameters. It is generated automatically by `buildnml` inside the + RUNDIR/data folder, where `RUNDIR` is the run directory generated by CIME. Since this file is automatically + generated when `buildnml` runs, users should not manually modify it. Any manual modification will be + lost the next time `buildnml` runs (e.g., at `case.submit` time). +3. 'namelist_scream.xml`: this XML file is generated by `buildnml`, and contains all the runtime parameters that EAMxx + will read in at runtime. `buildnml` uses this XML file as an intermediate file during the generation of + `scream_input.yaml`. In particular, `buildnml` generates this file using case information to select the + proper configuration from the file `namelist_defaults_scream.xml`, located in `components/eamxx/cime_config`. + As it was the case for `scream_input.yaml`, since this XML file is automatically generated by `buildnml`, + the user should not manually modify this file. Any manual modification will be lost the next time `buildnml` + runs (e.g., at `case.submit` time). + Since the main product of `buildnml` is `scream_input.yaml`, one may wonder why we keep + this XML file around at all. The main reason is to make it easier for the `atmquery` script (see below) to + retrieve settings. +4. `namelist.nl`: EAMxx will parse this [namelist](https://docs.oracle.com/cd/E19957-01/805-4939/6j4m0vnad/index.html) + file at runtime, to retrieve all the parameters specific to the [Homme](some_homme_ref) dycore. + It is generated automatically by `buildnml` inside the RUNDIR/data folder, where `RUNDIR` is the run + directory generated by CIME. Since this file is automatically generated when `buildnml` runs, + users should not manually modify it. Any manual modification will be lost the next time `buildnml` runs + (e.g., at `case.submit` time). +5. `atmchange` and `atmquery`: these script, located in `components/eamxx/scripts` and linked in the case folder, + can be used to query and change any of the runtime configuration parameters of EAMxx. +Among the files listed above, `atmquery` and `atmchange` are the only ones that the user should interact with, +and therefore we give here a brief overview of how they can be used, and how their output can be interpreted. +For both these scripts, a short help can also be obtained using the `-h` flag, highlighting their use cases and options. + +# Querying parameters with `atmquery` + +This script is the simplest way for the user to check the value and properties of EAMxx input parameters. +A basic usage of the script is + +```bash +$ ./atmquery my_param +``` +which will retrieve the value of the parameter called `my_param`, by locating the XML node "my_param" in the +file `namelist_scream.xml` in the RUNDIR folder. Obviously, an XML file can have multiple nodes with the same tag, +and the script is implemented to error out if multiple matches are found. In such a scenario, the user needs +to provide also the parents nodes names, using enough parents to uniquely identify the node (in most cases, +one parent is enough). To specify a parent, the user can prepend the parent name and `::` to the node name: + +```bash +$ ./atmquery foo::my_param +``` + +The output will contain the fully scoped parameter name, along with the value. E.g., +```bash +$ ./atmquery foo::my_param + namelist_defaults::node1::node2::foo::my_param: 10 +``` + +It is sometimes desirable to query _all_ the nodes that have a particular name, or that contain a particular +string. We can do that by using the `--grep` flag: +```blah +$ ./atmquery --grep sub + iop_options::iop_dosubsidence: false + ctl_nl::hypervis_subcycle: 1 + ctl_nl::hypervis_subcycle_tom: 1 + ctl_nl::hypervis_subcycle_q: 6 + atmosphere_processes::number_of_subcycles: 1 + sc_import::number_of_subcycles: 1 + homme::number_of_subcycles: 1 + physics::number_of_subcycles: 1 +``` +TODO: This difference between basica and `--grep` is not really intuitive (see [this +issue](https://github.com/E3SM-Project/scream/issues/2413)). +Using the `--grep` option has another effect: if the match is not a leaf of the XML tree, all its subelements +are printed: +```bash +$ ./atmquery --grep homme + homme + Moisture: moist + BfbHash: 18 + number_of_subcycles: 1 + enable_precondition_checks: true + enable_postcondition_checks: true + repair_log_level: trace + internal_diagnostics_level: 0 + compute_tendencies: None +``` + +Similarly to the CIME utility `xmlchange`, the options `--value`, `--type`, `--valid-values`, and `--full` can be +used to respectively retrieve just the parameter value (useful for bash scripting), the parameter's type, +a list of valid values for parameter (when applicable), or all of the above: +```bash +$ ./atmquery atm_log_level --value + info +$ ./atmquery atm_log_level --type + namelist_defaults::driver_options::atm_log_level: string +$ ./atmquery atm_log_level --valid-values + namelist_defaults::driver_options::atm_log_level: ['trace', 'debug', 'info', 'warn', 'error'] +$ ./atmquery atm_log_level --full + namelist_defaults::driver_options::atm_log_level + value: info + type: string + valid values: ['trace', 'debug', 'info', 'warn', 'error'] +``` + +Finally, the option `--listall` can be used to list the whole content of the XML file, which will be displayed +with each node indented in its parent scope: +```bash +$ ./atmquery --listall + namelist_defaults + grids_manager + Type: Homme + physics_grid_type: PG2 + physics_grid_rebalance: None + dynamics_namelist_file_name: ./data/namelist.nl + vertical_coordinate_filename: /some/path/to/coords/file.nc + initial_conditions + Filename: /some/path/to/ic/file.nc + topography_filename: /some/path/to/topo/file.nc + ... +``` + +# Changing model inputs via `atmchange` + +When `buildnml` runs, the model inputs are deduced from the case configuration settings (e.g., the grid, +the compset, etc.) and the `namelist_scream_defaults.xml` file, located in the eamxx source tree. +The user can change any of these parameters using the `atmchange` script. +A basic usage of the script is + +```bash +$ ./atmchange my_param=10 +``` +As for `atmquery`, if there are multiple matches for a given parameter name, the user must specify a unique +scoped name, which allows `atmchange` to uniquely identify the XML node to modify: +```bash +$ ./atmquery homme::number_of_subcycles + namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 1 +$ ./atmchange number_of_subcycles=10 +ERROR: number_of_subcycles is ambiguous (use --all to change all matches), matches: + namelist_defaults::atmosphere_processes::number_of_subcycles + namelist_defaults::atmosphere_processes::sc_import::number_of_subcycles + namelist_defaults::atmosphere_processes::homme::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::tms::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::shoc::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::cldFraction::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::p3::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::rrtmgp::number_of_subcycles + namelist_defaults::atmosphere_processes::sc_export::number_of_subcycles +$ ./atmchange homme::number_of_subcycles=10 +Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. +$ ./atmquery homme::number_of_subcycles + namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 10 +``` +In some cases, the user may be interested in changing _all_ nodes with a given name. In that case, +the `--all` flag can be used: +```bash +$ ./atmquery --grep number_of_subcycles + atmosphere_processes::number_of_subcycles: 1 + sc_import::number_of_subcycles: 1 + homme::number_of_subcycles: 1 + physics::number_of_subcycles: 1 + mac_aero_mic::number_of_subcycles: 24 + tms::number_of_subcycles: 1 + shoc::number_of_subcycles: 1 + cldFraction::number_of_subcycles: 1 + spa::number_of_subcycles: 1 + p3::number_of_subcycles: 1 + rrtmgp::number_of_subcycles: 1 + sc_export::number_of_subcycles: 1 +$ ./atmchange --all number_of_subcycles=3 +Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. +$ ./atmquery --grep number_of_subcycles + atmosphere_processes::number_of_subcycles: 3 + sc_import::number_of_subcycles: 3 + homme::number_of_subcycles: 3 + physics::number_of_subcycles: 3 + mac_aero_mic::number_of_subcycles: 3 + tms::number_of_subcycles: 3 + shoc::number_of_subcycles: 3 + cldFraction::number_of_subcycles: 3 + spa::number_of_subcycles: 3 + p3::number_of_subcycles: 3 + rrtmgp::number_of_subcycles: 3 + sc_export::number_of_subcycles: 3 +``` +Since the XML file stores constraints on the parameter value (like its type or valid values), attempting to use the +wrong type will cause an error: +```bash +$ ./atmquery --type se_ne + namelist_defaults::ctl_nl::se_ne: integer +$ ./atmchange se_ne=hello +ERROR: Could not refine 'hello' as type 'integer': +``` +There are three main types supported: integer, float, string, logical. When passing a string to `atmchange`, +the script will try to interpret it acoording to the parameter type, and throw an error if that's not possible: +for "string", anything works; for "integer", only digits are allowed, possibly with a negative sign in front; +for "float", only digits are allowed, possibly with a negative sign in front and a decimal point; for "logical", +only the strings "true" and "false" are allowed (case insensitive). There are two additional types supported: +"file" and "array(T)", where "T" is any of the other supported types (but not another array): + - "file" is used to inform CIME of the input files that have to be download from E3SM data servers, like initial conditions files, +or certain lookup tables. + - "array(T)" allows to specify a list of items (of the same type), which will be parsed inside EAMxx as + a `std::vector`. + +For type "string" and "array(T)", it is also possible to _append_ to the currently stored value +```bash +$ ./atmquery homme::compute_tendencies + namelist_defaults::atmosphere_processes::homme::compute_tendencies: + value: a, b + type: array(string) + valid values: [] +$ ./atmchange homme::compute_tendencies+=c +$ ./atmquery homme::compute_tendencies --full + namelist_defaults::atmosphere_processes::homme::compute_tendencies + value: a, b, c + type: array(string) + valid values: [] +``` + +# Modifying the list of atmosphere processes + +The `atmchange` script can be used to change any of the runtime parameters of EAMxx. In particular, it can +be used to add, remove, or reorder atmosphere processes. When adding an atmosphere process, we must first +make sure that the defaults for that process are present in `namelist_defaults_scream.xml`. For instance, +the default settings for the "physics" atmosphere process group include the following: +```bash +$ ./atmquery physics::atm_procs_list + namelist_defaults::atmosphere_processes::physics::atm_procs_list: mac_aero_mic,rrtmgp +``` +where "mac_aero_mic" is itself an atmosphere process group, consisting of macrophysics, aerosols, and microphysics +processes. If one wanted to add the "cosp" atmosphere process to this list, and change the number of its +subcycles, it could do so via +```bash +$ ./atmchange physics::atm_procs_list+=cosp +$ ./atmchange cosp::number_of_subcycles=3 +``` +Notice that if we swapped the two commands, we would get an error, since the node "cosp" is not present in +the XML generated from the defaults until we decide to add it. + +It is also possible to declare a new (empty) atmosphere process group, which can then be filled with valid +atmosphere processes via subsequent calls to `atmchange`. The syntax to trigger this behavior consists +in specifying a process name that begins and ends with an underscore: +```bash +$ ./atmchange physics::atm_procs_list+=_my_group_ +``` +This adds a new process to the list of processes in "physics", called "\_my_group\_", and which is itself +an atmosphere process group. Hence, we can then do +```bash +$ ./atmchange _my_group_::atm_procs_list+=A,B +``` +where A and B must be valid atmosphere process names (i.e., present in `namelist_defaults_scream.xml`) +or be themselves new atmosphere process groups (i.e., beginning/ending with an underscore) + +`atmchange` can also be used to completely change a list of atmosphere processes: +```bash +$ ./atmchange physics::atm_procs_list=A,B,C +``` +Notice that we used "=" instead of "+=", which means we will be overwriting the value, rather than appending. +Any atmosphere process that was previously in the list but is no longer in it will be removed from +the generated `namelist_defaults.xml` (and `scream_input.yaml`) files, along with all their nested parameters. From def068302ef2f3b132d7b3cc7642d3e1cafeda22 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Apr 2024 17:04:37 -0600 Subject: [PATCH 107/476] EAMxx: minor reworks of the user guide layout No need for the 'installation' section, which is only for standalone builds (and we never install scream anyways). Also, add a small md file where we can put some info about what compsets/grids we currently support for EAMxx --- components/eamxx/docs/user/eamxx_cases.md | 5 +++++ components/eamxx/docs/user/index.md | 6 ++++++ components/eamxx/mkdocs.yml | 6 ++---- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 components/eamxx/docs/user/eamxx_cases.md diff --git a/components/eamxx/docs/user/eamxx_cases.md b/components/eamxx/docs/user/eamxx_cases.md new file mode 100644 index 000000000000..2655d7782a78 --- /dev/null +++ b/components/eamxx/docs/user/eamxx_cases.md @@ -0,0 +1,5 @@ +Basics of EAMxx cases +===================================== + +This section explains how to create a case which uses EAMxx as the atmosphere model, +as well as what are the currently supported compsets and grids for EAMxx. diff --git a/components/eamxx/docs/user/index.md b/components/eamxx/docs/user/index.md index ba53083fc75c..557c37c7aef0 100644 --- a/components/eamxx/docs/user/index.md +++ b/components/eamxx/docs/user/index.md @@ -1,3 +1,9 @@ # SCREAM User Guide +This section contains documentation on how to create, setup, and run CIME cases with EAMxx as the atmosphere component. +It is assumed that the reader has a familiarity with [CIME case +control system](https://esmci.github.io/cime/versions/master/html/users_guide/index.html). In particular, we assume +that the user knows how to create a case, and what the `case.setup`, `case.build`, and `case.submit` commands do. + + For the time being, see our [public confluence EAMxx user guide](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3858890786/EAMxx+User+s+Guide) diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index ccf72c080392..a88b97ff0b91 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -4,10 +4,9 @@ nav: - 'Home': 'index.md' - 'User Guide': - 'Overview': 'user/index.md' - - 'Installation': 'common/installation.md' - - 'Model output': 'user/model_output.md' + - 'EAMxx case basics': 'user/eamxx_cases.md' - 'Model input': 'user/model_input.md' - - 'Runtime parameters': 'common/eamxx_params.md' + - 'Model output': 'user/model_output.md' - 'Nudging': 'user/nudging.md' - 'Extra radiation calls': 'user/clean_clear_sky.md' - 'Developer Guide': @@ -48,7 +47,6 @@ theme: features: - navigation.indices - navigation.instant - - navigation.sections - navigation.top # - navigation.tabs From 5561c5c38669a54f0e4581e8c024fbb9f20cbfb8 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Apr 2024 17:06:35 -0600 Subject: [PATCH 108/476] EAMxx: avoid bad link error from mkdocs --- components/eamxx/docs/user/model_input.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index 702bf7c89b72..5deb7c6dd3f2 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -27,7 +27,7 @@ The main files/scripts involved in the EAMxx runtime configuration are the follo this XML file around at all. The main reason is to make it easier for the `atmquery` script (see below) to retrieve settings. 4. `namelist.nl`: EAMxx will parse this [namelist](https://docs.oracle.com/cd/E19957-01/805-4939/6j4m0vnad/index.html) - file at runtime, to retrieve all the parameters specific to the [Homme](some_homme_ref) dycore. + file at runtime, to retrieve all the parameters specific to the Homme (ADD REF) dycore. It is generated automatically by `buildnml` inside the RUNDIR/data folder, where `RUNDIR` is the run directory generated by CIME. Since this file is automatically generated when `buildnml` runs, users should not manually modify it. Any manual modification will be lost the next time `buildnml` runs From c5eaa89b178bb4f35c5d2221bf3dc3defbbc0c17 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 1 May 2024 11:59:15 -0600 Subject: [PATCH 109/476] EAMxx: improvements to the model input user guide docs --- components/eamxx/docs/user/model_input.md | 128 +++++++++++++--------- 1 file changed, 75 insertions(+), 53 deletions(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index 5deb7c6dd3f2..b0d662e5328a 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -5,67 +5,66 @@ This section explains how input parameters are passed to EAMxx, and how the user change their value. The full list of the currently configuraable runtime parameters for EAMxx can be found [here](../common/eamxx_params.md). -The main files/scripts involved in the EAMxx runtime configuration are the following: - -1. `buildnml`: this script, located in `components/eamxx/cime_config`, is called by CIME's case management - scripts (`case.setup`, `case.build, `case.submit`), and is responsible for creating EAMxx input files, - including handling of user-specific modifications for the runtime parameters. Users should not have to - modify this script, nor should they have to manually call it. -2. `scream_input.yaml`: EAMxx will parse this [YAML](https://yaml.org/spec/1.2.2) file at runtime, - to detect all of its configuration parameters. It is generated automatically by `buildnml` inside the - RUNDIR/data folder, where `RUNDIR` is the run directory generated by CIME. Since this file is automatically - generated when `buildnml` runs, users should not manually modify it. Any manual modification will be - lost the next time `buildnml` runs (e.g., at `case.submit` time). -3. 'namelist_scream.xml`: this XML file is generated by `buildnml`, and contains all the runtime parameters that EAMxx - will read in at runtime. `buildnml` uses this XML file as an intermediate file during the generation of - `scream_input.yaml`. In particular, `buildnml` generates this file using case information to select the - proper configuration from the file `namelist_defaults_scream.xml`, located in `components/eamxx/cime_config`. - As it was the case for `scream_input.yaml`, since this XML file is automatically generated by `buildnml`, - the user should not manually modify this file. Any manual modification will be lost the next time `buildnml` - runs (e.g., at `case.submit` time). - Since the main product of `buildnml` is `scream_input.yaml`, one may wonder why we keep - this XML file around at all. The main reason is to make it easier for the `atmquery` script (see below) to - retrieve settings. -4. `namelist.nl`: EAMxx will parse this [namelist](https://docs.oracle.com/cd/E19957-01/805-4939/6j4m0vnad/index.html) - file at runtime, to retrieve all the parameters specific to the Homme (ADD REF) dycore. - It is generated automatically by `buildnml` inside the RUNDIR/data folder, where `RUNDIR` is the run - directory generated by CIME. Since this file is automatically generated when `buildnml` runs, - users should not manually modify it. Any manual modification will be lost the next time `buildnml` runs - (e.g., at `case.submit` time). -5. `atmchange` and `atmquery`: these script, located in `components/eamxx/scripts` and linked in the case folder, - can be used to query and change any of the runtime configuration parameters of EAMxx. - -Among the files listed above, `atmquery` and `atmchange` are the only ones that the user should interact with, -and therefore we give here a brief overview of how they can be used, and how their output can be interpreted. -For both these scripts, a short help can also be obtained using the `-h` flag, highlighting their use cases and options. +The infrastructure for reading inputs into EAMxx involves a few scripts/files: + +1. `atmchange` and `atmquery`: these scripts are located in `SRCDIR/components/eamxx/scripts`, + and are soft-linked in the case folder. As their names suggest, they can be used to query + and change the runtime configuration parameter of EAMxx. Since these two scripts are the + only scripts thta the average user needs to know and interact with, in the next sections + we give a brief overview of how they can be used and how their output can be interpreted. + Additionally, for both of the scripts a short help can also be obtained using the `-h` flag. +2. `buildnml`: this script, located in `SRCDIR/components/eamxx/cime_config`, is called by CIME's + case management scripts (`case.setup`, `case.build`, and `case.submit`), and is responsible for creating + the input files that EAMxx will read to load its runtime parameters. Users should not have to modify this script, + nor should they have to manually call it, but it is useful to know what it does. + When `buildnml` runs, it creates a few files, containing EAMxx input parameters: + - `scream_input.yaml`: this [YAML](https://yaml.org/spec/1.2.2) file is located in the `RUNDIR/data` folder, + and will be read by EAMxx at runtime to load all of its configuration parameters. More precisely, + this file contains parameters that need to be used inside the EAMxx interfaces. + - `namelist.nl`: this [namelist](https://docs.oracle.com/cd/E19957-01/805-4939/6j4m0vnad/index.html) file is + located in the `RUNDIR/data` folder, and will be parsed at runtime to get all the parameters for the + HOMME dycore (ADD REF). This file only contains dycore-specific parameters that are only recognized + inside HOMME, and does not contain any parameter pertaining EAMxx infrastructure. + - `namelist_scream.xml`: this XML file is located in the case directory, and contains all the runtime parameters that EAMxx + will read in at runtime. `buildnml` uses this XML file as an intermediate file during the generation of + `scream_input.yaml` and `namelist.nl`. More specifically, `buildnml` generates this file using case information to select the + proper configurations from the file `namelist_defaults_scream.xml`, located in `SRCDIR/components/eamxx/cime_config`. + Despite the fact that the only files that are needed at runtime are `scream_input.yaml` and `namelist.nl`, + we generate and keep this XML file around to make the implementation of `atmquery` easier. + + Since these files are automatically generated when `buildnml` runs, users should not manually modify them. + Any manual modification will be lost the next time `buildnml` runs (e.g., at `case.submit` time). # Querying parameters with `atmquery` This script is the simplest way for the user to check the value and properties of EAMxx input parameters. A basic usage of the script is -```bash +```shell $ ./atmquery my_param ``` + which will retrieve the value of the parameter called `my_param`, by locating the XML node "my_param" in the file `namelist_scream.xml` in the RUNDIR folder. Obviously, an XML file can have multiple nodes with the same tag, and the script is implemented to error out if multiple matches are found. In such a scenario, the user needs to provide also the parents nodes names, using enough parents to uniquely identify the node (in most cases, one parent is enough). To specify a parent, the user can prepend the parent name and `::` to the node name: -```bash +```shell $ ./atmquery foo::my_param ``` The output will contain the fully scoped parameter name, along with the value. E.g., -```bash + +```shell $ ./atmquery foo::my_param namelist_defaults::node1::node2::foo::my_param: 10 ``` It is sometimes desirable to query _all_ the nodes that have a particular name, or that contain a particular string. We can do that by using the `--grep` flag: -```blah + +```shell $ ./atmquery --grep sub iop_options::iop_dosubsidence: false ctl_nl::hypervis_subcycle: 1 @@ -76,11 +75,14 @@ $ ./atmquery --grep sub homme::number_of_subcycles: 1 physics::number_of_subcycles: 1 ``` -TODO: This difference between basica and `--grep` is not really intuitive (see [this -issue](https://github.com/E3SM-Project/scream/issues/2413)). + +TODO: This difference between basic and `--grep` is not really intuitive: as pointer out in [this +issue](https://github.com/E3SM-Project/scream/issues/2413), we should change this. If we do, don't forget +to update this following part of the docs. Using the `--grep` option has another effect: if the match is not a leaf of the XML tree, all its subelements are printed: -```bash + +```shell $ ./atmquery --grep homme homme Moisture: moist @@ -94,9 +96,10 @@ $ ./atmquery --grep homme ``` Similarly to the CIME utility `xmlchange`, the options `--value`, `--type`, `--valid-values`, and `--full` can be -used to respectively retrieve just the parameter value (useful for bash scripting), the parameter's type, +used to respectively retrieve just the parameter value (useful for shell scripting), the parameter's type, a list of valid values for parameter (when applicable), or all of the above: -```bash + +```shell $ ./atmquery atm_log_level --value info $ ./atmquery atm_log_level --type @@ -112,7 +115,8 @@ $ ./atmquery atm_log_level --full Finally, the option `--listall` can be used to list the whole content of the XML file, which will be displayed with each node indented in its parent scope: -```bash + +```shell $ ./atmquery --listall namelist_defaults grids_manager @@ -134,12 +138,14 @@ the compset, etc.) and the `namelist_scream_defaults.xml` file, located in the e The user can change any of these parameters using the `atmchange` script. A basic usage of the script is -```bash +```shell $ ./atmchange my_param=10 ``` + As for `atmquery`, if there are multiple matches for a given parameter name, the user must specify a unique scoped name, which allows `atmchange` to uniquely identify the XML node to modify: -```bash + +```shell $ ./atmquery homme::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 1 $ ./atmchange number_of_subcycles=10 @@ -160,9 +166,11 @@ Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. $ ./atmquery homme::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 10 ``` + In some cases, the user may be interested in changing _all_ nodes with a given name. In that case, the `--all` flag can be used: -```bash + +```shell $ ./atmquery --grep number_of_subcycles atmosphere_processes::number_of_subcycles: 1 sc_import::number_of_subcycles: 1 @@ -192,14 +200,17 @@ $ ./atmquery --grep number_of_subcycles rrtmgp::number_of_subcycles: 3 sc_export::number_of_subcycles: 3 ``` + Since the XML file stores constraints on the parameter value (like its type or valid values), attempting to use the wrong type will cause an error: -```bash + +```shell $ ./atmquery --type se_ne namelist_defaults::ctl_nl::se_ne: integer $ ./atmchange se_ne=hello ERROR: Could not refine 'hello' as type 'integer': ``` + There are three main types supported: integer, float, string, logical. When passing a string to `atmchange`, the script will try to interpret it acoording to the parameter type, and throw an error if that's not possible: for "string", anything works; for "integer", only digits are allowed, possibly with a negative sign in front; @@ -212,7 +223,8 @@ or certain lookup tables. a `std::vector`. For type "string" and "array(T)", it is also possible to _append_ to the currently stored value -```bash + +```shell $ ./atmquery homme::compute_tendencies namelist_defaults::atmosphere_processes::homme::compute_tendencies: value: a, b @@ -232,38 +244,48 @@ The `atmchange` script can be used to change any of the runtime parameters of EA be used to add, remove, or reorder atmosphere processes. When adding an atmosphere process, we must first make sure that the defaults for that process are present in `namelist_defaults_scream.xml`. For instance, the default settings for the "physics" atmosphere process group include the following: -```bash + +```shell $ ./atmquery physics::atm_procs_list namelist_defaults::atmosphere_processes::physics::atm_procs_list: mac_aero_mic,rrtmgp ``` + where "mac_aero_mic" is itself an atmosphere process group, consisting of macrophysics, aerosols, and microphysics processes. If one wanted to add the "cosp" atmosphere process to this list, and change the number of its subcycles, it could do so via -```bash + +```shell $ ./atmchange physics::atm_procs_list+=cosp $ ./atmchange cosp::number_of_subcycles=3 ``` + Notice that if we swapped the two commands, we would get an error, since the node "cosp" is not present in the XML generated from the defaults until we decide to add it. It is also possible to declare a new (empty) atmosphere process group, which can then be filled with valid atmosphere processes via subsequent calls to `atmchange`. The syntax to trigger this behavior consists in specifying a process name that begins and ends with an underscore: -```bash + +```shell $ ./atmchange physics::atm_procs_list+=_my_group_ ``` + This adds a new process to the list of processes in "physics", called "\_my_group\_", and which is itself an atmosphere process group. Hence, we can then do -```bash + +```shell $ ./atmchange _my_group_::atm_procs_list+=A,B ``` + where A and B must be valid atmosphere process names (i.e., present in `namelist_defaults_scream.xml`) or be themselves new atmosphere process groups (i.e., beginning/ending with an underscore) `atmchange` can also be used to completely change a list of atmosphere processes: -```bash + +```shell $ ./atmchange physics::atm_procs_list=A,B,C ``` + Notice that we used "=" instead of "+=", which means we will be overwriting the value, rather than appending. Any atmosphere process that was previously in the list but is no longer in it will be removed from the generated `namelist_defaults.xml` (and `scream_input.yaml`) files, along with all their nested parameters. From 79569bc5b87adc363f6344beeb5957988e31d785 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 13:15:34 -0500 Subject: [PATCH 110/476] no need to sync results from randomize --- .../eamxx/src/diagnostics/tests/aerocom_cld_test.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index ae4d2e1d3389..c2818727c904 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -144,17 +144,6 @@ TEST_CASE("aerocom_cld") { diag->set_required_field(cd); diag->set_required_field(nc); - tm.sync_to_dev(); - pd.sync_to_dev(); - pm.sync_to_dev(); - qv.sync_to_dev(); - qc.sync_to_dev(); - qi.sync_to_dev(); - ec.sync_to_dev(); - ei.sync_to_dev(); - cd.sync_to_dev(); - nc.sync_to_dev(); - diag->initialize(t0, RunType::Initial); // Case 1: if the cloud fraction is zero, everything is zero From f1dfe549e87bf843ba59825c62b0d94df2a27205 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 13:33:00 -0500 Subject: [PATCH 111/476] better self-documenting through maps --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 37 +++++++++++-------- .../eamxx/src/diagnostics/aerocom_cld.hpp | 3 ++ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 19c08a5070c3..5c6775634c44 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -63,19 +63,24 @@ void AeroComCld::set_grids( m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); + // Set the attrs maps + m_index_map["T_mid"] = 0; m_units_map["T_mid"] = "K"; + m_index_map["p_mid"] = 1; m_units_map["p_mid"] = "Pa"; + m_index_map["cldfrac_ice"] = 2; m_units_map["cldfrac_ice"] = "nondim"; + m_index_map["cldfrac_liq"] = 3; m_units_map["cldfrac_liq"] = "nondim"; + m_index_map["cdnc"] = 4; m_units_map["cdnc"] = "#/m3"; + m_index_map["eff_radius_qc"] = 5; m_units_map["eff_radius_qc"] = "micron"; + m_index_map["eff_radius_qi"] = 6; m_units_map["eff_radius_qi"] = "micron"; + m_index_map["cldfrac_tot"] = 7; m_units_map["cldfrac_tot"] = "nondim"; + // Self-document the outputs to parse in post-processing using stratt_t = std::map; auto d = get_diagnostic(); auto &metadata = d.get_header().get_extra_data("io: string attributes"); - metadata["0"] = "T_mid"; - metadata["1"] = "p_mid"; - metadata["2"] = "cldfrac_ice"; - metadata["3"] = "cldfrac_liq"; - metadata["4"] = "cdnc"; - metadata["5"] = "eff_radius_qc"; - metadata["6"] = "eff_radius_qi"; - metadata["7"] = "cldfrac_tot"; + for (std::map::iterator it = m_index_map.begin(); it != m_index_map.end(); ++it) { + metadata[it->first] = m_index_map[it->first] + " (" + m_units_map[it->first] + ")"; + } } void AeroComCld::compute_diagnostic_impl() { @@ -173,20 +178,20 @@ void AeroComCld::compute_diagnostic_impl() { /* In general, converting a 3D property X to a 2D cloud-top * counterpart x follows: x(i) += X(i,k) * weights * Phase * but X and Phase are not always needed */ - out(icol, /* T_mid */ 0) += tmid_icol(ilay) * aerocom_wts; - out(icol, /* p_mid */ 1) += pmid_icol(ilay) * aerocom_wts; - out(icol, /* cldfrac_ice */ 2) += (1.0 - aerocom_phi) * aerocom_wts; - out(icol, /* cldfrac_liq */ 3) += aerocom_phi * aerocom_wts; + out(icol, m_index_map["T_mid"]) += tmid_icol(ilay) * aerocom_wts; + out(icol, m_index_map["p_mid"]) += pmid_icol(ilay) * aerocom_wts; + out(icol, m_index_map["cldfrac_ice"]) += (1.0 - aerocom_phi) * aerocom_wts; + out(icol, m_index_map["cldfrac_liq"]) += aerocom_phi * aerocom_wts; // cdnc /* We need to convert nc from 1/mass to 1/volume first, and * from grid-mean to in-cloud, but after that, the * calculation follows the general logic */ auto cdnc = nc_icol(ilay) * pden_icol(ilay) / dz_icol(ilay) / physconst::gravit / cld_icol(ilay); - out(icol, /* cdnc */ 4) += cdnc * aerocom_phi * aerocom_wts; - out(icol, /* eff_radius_qc */ 5) += + out(icol, m_index_map["cdnc"]) += cdnc * aerocom_phi * aerocom_wts; + out(icol, m_index_map["eff_radius_qc"]) += rel_icol(ilay) * aerocom_phi * aerocom_wts; - out(icol, /* eff_radius_qi */ 6) += + out(icol, m_index_map["eff_radius_qi"]) += rei_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; // Reset clr_icol to aerocom_tmp to accumulate clr_icol = aerocom_tmp; @@ -207,7 +212,7 @@ void AeroComCld::compute_diagnostic_impl() { // defined as (1 - clr_icol). This is true because // clr_icol is the result of accumulative probabilities // (their products) - out(icol, /* cldfrac_tot */ 7) = 1.0 - clr_icol; + out(icol, m_index_map["cldfrac_tot"]) = 1.0 - clr_icol; }); } diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index 6cb1328db009..2d67aeecc046 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -35,6 +35,9 @@ class AeroComCld : public AtmosphereDiagnostic { std::string m_topbot; Field m_dz; + + std::map m_index_map; + std::map m_units_map; }; } // namespace scream From bfc354454f4d36d8bce1d32ba4dd3f7d81d886e2 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Wed, 1 May 2024 11:50:23 -0700 Subject: [PATCH 112/476] A recent change to get_X_layout didn't make it to ML --- .../ml_correction/eamxx_ml_correction_process_interface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index e04f7a0ca3dc..42a54250fe55 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -36,9 +36,9 @@ void MLCorrection::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); - FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout_mid(); - FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout_int(); - FieldLayout vector3d_mid = m_grid->get_3d_vector_layout_mid(2); + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); + FieldLayout vector3d_mid = m_grid->get_3d_vector_layout(true,2); if (not m_ML_correction_unit_test) { const auto m2 = m*m; const auto s2 = s*s; From 7836f6c08eeedc34ee957c3d6d77b016cec22cc9 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 1 May 2024 14:28:58 -0500 Subject: [PATCH 113/476] Updating to SCORPIO v1.6.3 SCORPIO v1.6.3 includes the following fixes, * Fix for build issues with the Intel compiler on Perlmutter * Fix for issues writing/reading ADIOS string attributes * Misc bug fixes --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index dc6f1eb24f3d..cdd541e0cd70 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit dc6f1eb24f3dae1dcb0ebb13130c4941dc9bd445 +Subproject commit cdd541e0cd708bece13b1f3ee42f66dfd6440aa7 From 2d584f231c359296cb3a70acfbbe0aa26a6e1835 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 15:14:35 -0500 Subject: [PATCH 114/476] properly add maps and add some checks --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 52 ++++++++++++++----- .../diagnostics/tests/aerocom_cld_test.cpp | 2 +- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 5c6775634c44..472bdd94ad8a 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -36,6 +36,39 @@ void AeroComCld::set_grids( m_ncols = grid->get_num_local_dofs(); m_nlevs = grid->get_num_vertical_levels(); m_ndiag = 8; + // Set the attrs maps + // T_mid + m_index_map["T_mid"] = 0; + m_units_map["T_mid"] = "K"; + // p_mid + m_index_map["p_mid"] = 1; + m_units_map["p_mid"] = "Pa"; + // cldfrac_ice + m_index_map["cldfrac_ice"] = 2; + m_units_map["cldfrac_ice"] = "nondim"; + // cldfrac_liq + m_index_map["cldfrac_liq"] = 3; + m_units_map["cldfrac_liq"] = "nondim"; + // cdnc + m_index_map["cdnc"] = 4; + m_units_map["cdnc"] = "#/m3"; + // eff_radius_qc + m_index_map["eff_radius_qc"] = 5; + m_units_map["eff_radius_qc"] = "micron"; + // eff_radius_qi + m_index_map["eff_radius_qi"] = 6; + m_units_map["eff_radius_qi"] = "micron"; + // cldfrac_tot + m_index_map["cldfrac_tot"] = 7; + m_units_map["cldfrac_tot"] = "nondim"; + + // Ensure m_index_map.size() is the same as m_ndiag + EKAT_REQUIRE_MSG(m_index_map.size() == m_units_map.size(), + "Error! Some inconsistency in AeroComCld: index and units " + "maps do not match!\n"); + EKAT_REQUIRE_MSG(m_index_map.size() == m_ndiag, + "Error! Some inconsistency in AeroComCld: index map doesn't " + "match m_ndiag!\n"); // Define layouts we need (both inputs and outputs) FieldLayout scalar2d_layout{{COL, LEV}, {m_ncols, m_nlevs}}; @@ -63,23 +96,15 @@ void AeroComCld::set_grids( m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); - // Set the attrs maps - m_index_map["T_mid"] = 0; m_units_map["T_mid"] = "K"; - m_index_map["p_mid"] = 1; m_units_map["p_mid"] = "Pa"; - m_index_map["cldfrac_ice"] = 2; m_units_map["cldfrac_ice"] = "nondim"; - m_index_map["cldfrac_liq"] = 3; m_units_map["cldfrac_liq"] = "nondim"; - m_index_map["cdnc"] = 4; m_units_map["cdnc"] = "#/m3"; - m_index_map["eff_radius_qc"] = 5; m_units_map["eff_radius_qc"] = "micron"; - m_index_map["eff_radius_qi"] = 6; m_units_map["eff_radius_qi"] = "micron"; - m_index_map["cldfrac_tot"] = 7; m_units_map["cldfrac_tot"] = "nondim"; - // Self-document the outputs to parse in post-processing using stratt_t = std::map; auto d = get_diagnostic(); auto &metadata = d.get_header().get_extra_data("io: string attributes"); - for (std::map::iterator it = m_index_map.begin(); it != m_index_map.end(); ++it) { - metadata[it->first] = m_index_map[it->first] + " (" + m_units_map[it->first] + ")"; + for(std::map::iterator it = m_index_map.begin(); + it != m_index_map.end(); ++it) { + metadata[it->first] = + m_index_map[it->first] + " (" + m_units_map[it->first] + ")"; } } @@ -180,7 +205,8 @@ void AeroComCld::compute_diagnostic_impl() { * but X and Phase are not always needed */ out(icol, m_index_map["T_mid"]) += tmid_icol(ilay) * aerocom_wts; out(icol, m_index_map["p_mid"]) += pmid_icol(ilay) * aerocom_wts; - out(icol, m_index_map["cldfrac_ice"]) += (1.0 - aerocom_phi) * aerocom_wts; + out(icol, m_index_map["cldfrac_ice"]) += + (1.0 - aerocom_phi) * aerocom_wts; out(icol, m_index_map["cldfrac_liq"]) += aerocom_phi * aerocom_wts; // cdnc /* We need to convert nc from 1/mass to 1/volume first, and diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index c2818727c904..912e5900c0f1 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -154,7 +154,7 @@ TEST_CASE("aerocom_cld") { diag_f.sync_to_host(); auto diag_v = diag_f.get_view(); for(int idiag = 0; idiag < m_ndiag; ++idiag) { - REQUIRE(diag_v(0, idiag) == 0.0); + REQUIRE(diag_v(0, idiag) == Real(0.0)); } // Case 2: if the cloud fraction is one, everything takes 1 * its value From 3d9d224e727bc41796958800b7d2a7df00420957 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 15:20:51 -0500 Subject: [PATCH 115/476] reorder the sync to host calls --- .../src/diagnostics/tests/aerocom_cld_test.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 912e5900c0f1..0e317d7c9067 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -150,7 +150,6 @@ TEST_CASE("aerocom_cld") { cd.deep_copy(0.0); diag->compute_diagnostic(); Field diag_f = diag->get_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f.sync_to_host(); auto diag_v = diag_f.get_view(); for(int idiag = 0; idiag < m_ndiag; ++idiag) { @@ -193,8 +192,8 @@ TEST_CASE("aerocom_cld") { cd_v(0, 4) = 0.2; cd.sync_to_dev(); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); + diag_f.sync_to_host(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) == Real(0.7)); @@ -206,8 +205,8 @@ TEST_CASE("aerocom_cld") { cd_v(0, 7) = 0.2; cd.sync_to_dev(); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); + diag_f.sync_to_host(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) > Real(0.7)); @@ -217,8 +216,8 @@ TEST_CASE("aerocom_cld") { cd_v(0, 7) = 0.1; cd.sync_to_dev(); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); + diag_f.sync_to_host(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) > Real(0.7)); // must be larger than the max! @@ -230,8 +229,8 @@ TEST_CASE("aerocom_cld") { qi.deep_copy(0.0); // zero ice! cd.sync_to_dev(); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); + diag_f.sync_to_host(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) == Real(1.0)); REQUIRE(diag_v(0, 3) == Real(1.0)); @@ -241,8 +240,8 @@ TEST_CASE("aerocom_cld") { qc.deep_copy(0.0); // zero liq! qi.deep_copy(1.0); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); + diag_f.sync_to_host(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) == Real(1.0)); REQUIRE(diag_v(0, 3) == Real(0.0)); // zero liq! @@ -280,8 +279,8 @@ TEST_CASE("aerocom_cld") { qc.sync_to_dev(); diag->compute_diagnostic(); - diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); + diag_f.sync_to_host(); diag_v = diag_f.get_view(); REQUIRE(diag_v(0, 7) > Real(0.7)); // unaffected (see test case 4) REQUIRE(diag_v(0, 3) < Real(0.5)); // not max! From 83795dcb82685230b83eba1e8f1432d03feeed76 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 15:59:59 -0500 Subject: [PATCH 116/476] properly handling adding an int with a string --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 472bdd94ad8a..dd3a0bdd71ed 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -1,6 +1,7 @@ #include "diagnostics/aerocom_cld.hpp" #include +#include #include "share/util/scream_common_physics_functions.hpp" @@ -38,20 +39,20 @@ void AeroComCld::set_grids( m_ndiag = 8; // Set the attrs maps // T_mid - m_index_map["T_mid"] = 0; - m_units_map["T_mid"] = "K"; + m_index_map["T_mid"] = 0; + m_units_map["T_mid"] = "K"; // p_mid - m_index_map["p_mid"] = 1; - m_units_map["p_mid"] = "Pa"; + m_index_map["p_mid"] = 1; + m_units_map["p_mid"] = "Pa"; // cldfrac_ice - m_index_map["cldfrac_ice"] = 2; - m_units_map["cldfrac_ice"] = "nondim"; + m_index_map["cldfrac_ice"] = 2; + m_units_map["cldfrac_ice"] = "nondim"; // cldfrac_liq - m_index_map["cldfrac_liq"] = 3; - m_units_map["cldfrac_liq"] = "nondim"; + m_index_map["cldfrac_liq"] = 3; + m_units_map["cldfrac_liq"] = "nondim"; // cdnc - m_index_map["cdnc"] = 4; - m_units_map["cdnc"] = "#/m3"; + m_index_map["cdnc"] = 4; + m_units_map["cdnc"] = "#/m3"; // eff_radius_qc m_index_map["eff_radius_qc"] = 5; m_units_map["eff_radius_qc"] = "micron"; @@ -59,8 +60,8 @@ void AeroComCld::set_grids( m_index_map["eff_radius_qi"] = 6; m_units_map["eff_radius_qi"] = "micron"; // cldfrac_tot - m_index_map["cldfrac_tot"] = 7; - m_units_map["cldfrac_tot"] = "nondim"; + m_index_map["cldfrac_tot"] = 7; + m_units_map["cldfrac_tot"] = "nondim"; // Ensure m_index_map.size() is the same as m_ndiag EKAT_REQUIRE_MSG(m_index_map.size() == m_units_map.size(), @@ -101,10 +102,9 @@ void AeroComCld::set_grids( auto d = get_diagnostic(); auto &metadata = d.get_header().get_extra_data("io: string attributes"); - for(std::map::iterator it = m_index_map.begin(); - it != m_index_map.end(); ++it) { - metadata[it->first] = - m_index_map[it->first] + " (" + m_units_map[it->first] + ")"; + for(const auto &it : m_index_map) { + metadata[it.first] = + std::to_string(it.second) + " (" + m_units_map[it.first] + ")"; } } From 09ce61e0c5ba87d328fde8a27fcf43087341f33d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 1 May 2024 15:00:42 -0600 Subject: [PATCH 117/476] EAMxx: fix typos in docs --- components/eamxx/docs/user/model_input.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index b0d662e5328a..fb25422846ec 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -10,7 +10,7 @@ The infrastructure for reading inputs into EAMxx involves a few scripts/files: 1. `atmchange` and `atmquery`: these scripts are located in `SRCDIR/components/eamxx/scripts`, and are soft-linked in the case folder. As their names suggest, they can be used to query and change the runtime configuration parameter of EAMxx. Since these two scripts are the - only scripts thta the average user needs to know and interact with, in the next sections + only scripts that the average user needs to know and interact with, in the next sections we give a brief overview of how they can be used and how their output can be interpreted. Additionally, for both of the scripts a short help can also be obtained using the `-h` flag. 2. `buildnml`: this script, located in `SRCDIR/components/eamxx/cime_config`, is called by CIME's @@ -76,7 +76,7 @@ $ ./atmquery --grep sub physics::number_of_subcycles: 1 ``` -TODO: This difference between basic and `--grep` is not really intuitive: as pointer out in [this +TODO: This difference between basic and `--grep` is not really intuitive: as pointed out in [this issue](https://github.com/E3SM-Project/scream/issues/2413), we should change this. If we do, don't forget to update this following part of the docs. Using the `--grep` option has another effect: if the match is not a leaf of the XML tree, all its subelements From 4c14955e9603d1b982c9af21150e06236b608c8f Mon Sep 17 00:00:00 2001 From: mahf708 Date: Wed, 1 May 2024 22:44:22 -0500 Subject: [PATCH 118/476] create a util to save info and add nc, ni --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 59 ++++++---------- .../eamxx/src/diagnostics/aerocom_cld.hpp | 6 +- .../src/diagnostics/aerocom_cld_util.hpp | 67 +++++++++++++++++++ .../diagnostics/tests/aerocom_cld_test.cpp | 18 +++-- 4 files changed, 106 insertions(+), 44 deletions(-) create mode 100644 components/eamxx/src/diagnostics/aerocom_cld_util.hpp diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index dd3a0bdd71ed..0c66deab32c9 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -3,6 +3,7 @@ #include #include +#include "diagnostics/aerocom_cld_util.hpp" #include "share/util/scream_common_physics_functions.hpp" namespace scream { @@ -34,42 +35,20 @@ void AeroComCld::set_grids( const auto nondim = Units::nondimensional(); const auto micron = m / 1000000; - m_ncols = grid->get_num_local_dofs(); - m_nlevs = grid->get_num_vertical_levels(); - m_ndiag = 8; - // Set the attrs maps - // T_mid - m_index_map["T_mid"] = 0; - m_units_map["T_mid"] = "K"; - // p_mid - m_index_map["p_mid"] = 1; - m_units_map["p_mid"] = "Pa"; - // cldfrac_ice - m_index_map["cldfrac_ice"] = 2; - m_units_map["cldfrac_ice"] = "nondim"; - // cldfrac_liq - m_index_map["cldfrac_liq"] = 3; - m_units_map["cldfrac_liq"] = "nondim"; - // cdnc - m_index_map["cdnc"] = 4; - m_units_map["cdnc"] = "#/m3"; - // eff_radius_qc - m_index_map["eff_radius_qc"] = 5; - m_units_map["eff_radius_qc"] = "micron"; - // eff_radius_qi - m_index_map["eff_radius_qi"] = 6; - m_units_map["eff_radius_qi"] = "micron"; - // cldfrac_tot - m_index_map["cldfrac_tot"] = 7; - m_units_map["cldfrac_tot"] = "nondim"; + // Set the index map and units map + AeroComCldDiagUtil aercom_util; + m_index_map = aercom_util.index_map; + m_units_map = aercom_util.units_map; + m_ndiag = aercom_util.size; // Ensure m_index_map.size() is the same as m_ndiag EKAT_REQUIRE_MSG(m_index_map.size() == m_units_map.size(), "Error! Some inconsistency in AeroComCld: index and units " "maps do not match!\n"); - EKAT_REQUIRE_MSG(m_index_map.size() == m_ndiag, - "Error! Some inconsistency in AeroComCld: index map doesn't " - "match m_ndiag!\n"); + + m_ncols = grid->get_num_local_dofs(); + m_nlevs = grid->get_num_vertical_levels(); + // m_ndiag = m_index_map.size(); // Define layouts we need (both inputs and outputs) FieldLayout scalar2d_layout{{COL, LEV}, {m_ncols, m_nlevs}}; @@ -85,7 +64,8 @@ void AeroComCld::set_grids( add_field("eff_radius_qc", scalar2d_layout, micron, grid_name); add_field("eff_radius_qi", scalar2d_layout, micron, grid_name); add_field("cldfrac_tot", scalar2d_layout, nondim, grid_name); - add_field("nc", scalar2d_layout, kg / kg, grid_name); + add_field("nc", scalar2d_layout, 1 / kg, grid_name); + add_field("ni", scalar2d_layout, 1 / kg, grid_name); // A field to store dz FieldIdentifier m_dz_fid("dz", scalar2d_layout, m, grid_name); @@ -133,15 +113,16 @@ void AeroComCld::compute_diagnostic_impl() { const auto rei = get_field_in("eff_radius_qi").get_view(); const auto cld = get_field_in("cldfrac_tot").get_view(); const auto nc = get_field_in("nc").get_view(); + const auto ni = get_field_in("ni").get_view(); auto dz = m_dz.get_view(); // Get gravity acceleration constant from constants using physconst = scream::physics::Constants; - // TODO: move tunable constant to namelist - constexpr auto q_threshold = 0.0; - // TODO: move tunable constant to namelist - constexpr auto cldfrac_tot_threshold = 0.001; + + // Set two potentially tunable parameters + auto q_threshold = q_thresh_set(); + auto cldfrac_tot_threshold = cldfrac_tot_thresh_set(); const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); @@ -167,9 +148,9 @@ void AeroComCld::compute_diagnostic_impl() { auto rei_icol = ekat::subview(rei, icol); auto cld_icol = ekat::subview(cld, icol); auto nc_icol = ekat::subview(nc, icol); + auto ni_icol = ekat::subview(ni, icol); // We need dz too - // TODO: deduce dz_icol type, etc. on the spot here w/o field cloning? auto dz_icol = ekat::subview(dz, icol); PF::calculate_dz(team, pden_icol, pmid_icol, tmid_icol, qv_icol, dz_icol); @@ -215,6 +196,10 @@ void AeroComCld::compute_diagnostic_impl() { auto cdnc = nc_icol(ilay) * pden_icol(ilay) / dz_icol(ilay) / physconst::gravit / cld_icol(ilay); out(icol, m_index_map["cdnc"]) += cdnc * aerocom_phi * aerocom_wts; + out(icol, m_index_map["nc"]) += + nc_icol(ilay) * aerocom_phi * aerocom_wts; + out(icol, m_index_map["ni"]) += + ni_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; out(icol, m_index_map["eff_radius_qc"]) += rel_icol(ilay) * aerocom_phi * aerocom_wts; out(icol, m_index_map["eff_radius_qi"]) += diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index 2d67aeecc046..bee3ffb15132 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -34,10 +34,12 @@ class AeroComCld : public AtmosphereDiagnostic { // Bot or Top std::string m_topbot; - Field m_dz; - + // Attribute maps for self-documentation std::map m_index_map; std::map m_units_map; + + // Set a field for dz + Field m_dz; }; } // namespace scream diff --git a/components/eamxx/src/diagnostics/aerocom_cld_util.hpp b/components/eamxx/src/diagnostics/aerocom_cld_util.hpp new file mode 100644 index 000000000000..1f8543ed9ef8 --- /dev/null +++ b/components/eamxx/src/diagnostics/aerocom_cld_util.hpp @@ -0,0 +1,67 @@ +#ifndef EAMXX_AEROCOMCLD_DIAG_UTIL +#define EAMXX_AEROCOMCLD_DIAG_UTIL + +#include + +namespace scream { + +class AeroComCldDiagUtil { + public: + std::map index_map; + std::map units_map; + int size; + + AeroComCldDiagUtil() { + // T_mid + index_map["T_mid"] = 0; + units_map["T_mid"] = "K"; + // p_mid + index_map["p_mid"] = 1; + units_map["p_mid"] = "Pa"; + // cldfrac_ice + index_map["cldfrac_ice"] = 2; + units_map["cldfrac_ice"] = "nondim"; + // cldfrac_liq + index_map["cldfrac_liq"] = 3; + units_map["cldfrac_liq"] = "nondim"; + // cdnc + index_map["cdnc"] = 4; + units_map["cdnc"] = "#/m3"; + // eff_radius_qc + index_map["eff_radius_qc"] = 5; + units_map["eff_radius_qc"] = "micron"; + // eff_radius_qi + index_map["eff_radius_qi"] = 6; + units_map["eff_radius_qi"] = "micron"; + // cldfrac_tot + index_map["cldfrac_tot"] = 7; + units_map["cldfrac_tot"] = "nondim"; + // nc + index_map["nc"] = 8; + units_map["nc"] = "1/kg"; + // ni + index_map["ni"] = 9; + units_map["ni"] = "1/kg"; + + size = index_map.size(); + } +}; + +constexpr double q_thresh_set() { + // The total q is checked at the outset of the aerocom_cld routine + // to determine whether or not there is any cloud. This function sets + // the arbitrary q threshold to be used. + return 0.0; +} + +constexpr double cldfrac_tot_thresh_set() { + // The total cldfrac is checked at the outset of the aerocom_cld routine + // to determine whether or not there is any cloud. This function sets + // the arbitrary cldfrac threshold to be used, which is slightly more than + // 0.0 to guard against potential issues near zero. + return 0.001; +} + +} // namespace scream + +#endif // EAMXX_AEROCOMCLD_DIAG_UTIL diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 0e317d7c9067..5ffaf5498e60 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -1,4 +1,5 @@ #include "catch2/catch.hpp" +#include "diagnostics/aerocom_cld_util.hpp" #include "diagnostics/register_diagnostics.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" @@ -42,7 +43,9 @@ TEST_CASE("aerocom_cld") { constexpr int nlevs = 9; const int ngcols = 1 * comm.size(); - int m_ndiag = 8; + // See how many diags we are calculating + AeroComCldDiagUtil aercom_util; + int ndiags = aercom_util.size; auto gm = create_gm(comm, ngcols, nlevs); auto grid = gm->get_grid("Physics"); @@ -62,7 +65,8 @@ TEST_CASE("aerocom_cld") { FieldIdentifier ei_fid("eff_radius_qi", scalar2d_layout, micron, grid->name()); FieldIdentifier cd_fid("cldfrac_tot", scalar2d_layout, nondim, grid->name()); - FieldIdentifier nc_fid("nc", scalar2d_layout, kg / kg, grid->name()); + FieldIdentifier nc_fid("nc", scalar2d_layout, 1 / kg, grid->name()); + FieldIdentifier ni_fid("ni", scalar2d_layout, 1 / kg, grid->name()); Field tm(tm_fid); tm.allocate_view(); @@ -94,6 +98,9 @@ TEST_CASE("aerocom_cld") { Field nc(nc_fid); nc.allocate_view(); nc.get_header().get_tracking().update_time_stamp(t0); + Field ni(ni_fid); + ni.allocate_view(); + ni.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; @@ -126,6 +133,7 @@ TEST_CASE("aerocom_cld") { randomize(ei, engine, pdf); randomize(cd, engine, pdf); randomize(nc, engine, pdf); + randomize(ni, engine, pdf); // Create and set up the diagnostic params.set("AeroComCld Kind", "Top"); @@ -143,6 +151,7 @@ TEST_CASE("aerocom_cld") { diag->set_required_field(ei); diag->set_required_field(cd); diag->set_required_field(nc); + diag->set_required_field(ni); diag->initialize(t0, RunType::Initial); @@ -152,7 +161,7 @@ TEST_CASE("aerocom_cld") { Field diag_f = diag->get_diagnostic(); diag_f.sync_to_host(); auto diag_v = diag_f.get_view(); - for(int idiag = 0; idiag < m_ndiag; ++idiag) { + for(int idiag = 0; idiag < ndiags; ++idiag) { REQUIRE(diag_v(0, idiag) == Real(0.0)); } @@ -168,6 +177,7 @@ TEST_CASE("aerocom_cld") { ec.deep_copy(10.0); ei.deep_copy(10.0); nc.deep_copy(5.0); + ni.deep_copy(1.0); diag->compute_diagnostic(); diag->get_diagnostic().sync_to_host(); diag_f = diag->get_diagnostic(); @@ -273,11 +283,9 @@ TEST_CASE("aerocom_cld") { qc_v(0, 5) = 20; qc_v(0, 6) = 50; qc_v(0, 7) = 10; - cd.sync_to_dev(); qi.sync_to_dev(); qc.sync_to_dev(); - diag->compute_diagnostic(); diag_f = diag->get_diagnostic(); diag_f.sync_to_host(); From 4600445a8045a10c2cc7dbe31162bc203bbc818d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 2 May 2024 09:54:29 -0600 Subject: [PATCH 119/476] EAMxx: docs rephrase pointer to confluence docs --- components/eamxx/docs/user/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/docs/user/index.md b/components/eamxx/docs/user/index.md index 557c37c7aef0..2e24e9fa736a 100644 --- a/components/eamxx/docs/user/index.md +++ b/components/eamxx/docs/user/index.md @@ -6,4 +6,5 @@ control system](https://esmci.github.io/cime/versions/master/html/users_guide/in that the user knows how to create a case, and what the `case.setup`, `case.build`, and `case.submit` commands do. -For the time being, see our [public confluence EAMxx user guide](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3858890786/EAMxx+User+s+Guide) +This user guide is still under construction. In the meantime, in case you can't find the information you need, +you may visit our public confluence [EAMxx user guide](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3858890786/EAMxx+User+s+Guide). From cbcb50d21274cf74f9db0ec5e52dd7297c4e4ca0 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 2 May 2024 09:10:43 -0700 Subject: [PATCH 120/476] make sure kernels are clean coal for ops --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 37 ++++++++++++------- .../eamxx/src/diagnostics/aerocom_cld.hpp | 3 ++ .../src/diagnostics/aerocom_cld_util.hpp | 27 +++++++------- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 0c66deab32c9..50eea7995c04 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -41,14 +41,13 @@ void AeroComCld::set_grids( m_units_map = aercom_util.units_map; m_ndiag = aercom_util.size; - // Ensure m_index_map.size() is the same as m_ndiag + // Ensure m_index_map and m_units_map match EKAT_REQUIRE_MSG(m_index_map.size() == m_units_map.size(), "Error! Some inconsistency in AeroComCld: index and units " "maps do not match!\n"); m_ncols = grid->get_num_local_dofs(); m_nlevs = grid->get_num_vertical_levels(); - // m_ndiag = m_index_map.size(); // Define layouts we need (both inputs and outputs) FieldLayout scalar2d_layout{{COL, LEV}, {m_ncols, m_nlevs}}; @@ -134,6 +133,18 @@ void AeroComCld::compute_diagnostic_impl() { const int nlevs = m_nlevs; bool is_top = (m_topbot == "Top"); + // Get indices of each call we care about + const int T_mid_idx = m_index_map["T_mid"]; + const int p_mid_idx = m_index_map["p_mid"]; + const int cldfrac_ice_idx = m_index_map["cldfrac_ice"]; + const int cldfrac_liq_idx = m_index_map["cldfrac_liq"]; + const int cdnc_idx = m_index_map["cdnc"]; + const int eff_radius_qc_idx = m_index_map["eff_radius_qc"]; + const int eff_radius_qi_idx = m_index_map["eff_radius_qi"]; + const int nc_idx = m_index_map["nc"]; + const int ni_idx = m_index_map["ni"]; + const int cldfrac_tot_idx = m_index_map["cldfrac_tot"]; + Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); @@ -184,25 +195,23 @@ void AeroComCld::compute_diagnostic_impl() { /* In general, converting a 3D property X to a 2D cloud-top * counterpart x follows: x(i) += X(i,k) * weights * Phase * but X and Phase are not always needed */ - out(icol, m_index_map["T_mid"]) += tmid_icol(ilay) * aerocom_wts; - out(icol, m_index_map["p_mid"]) += pmid_icol(ilay) * aerocom_wts; - out(icol, m_index_map["cldfrac_ice"]) += - (1.0 - aerocom_phi) * aerocom_wts; - out(icol, m_index_map["cldfrac_liq"]) += aerocom_phi * aerocom_wts; + out(icol, T_mid_idx) += tmid_icol(ilay) * aerocom_wts; + out(icol, p_mid_idx) += pmid_icol(ilay) * aerocom_wts; + out(icol, cldfrac_ice_idx) += (1.0 - aerocom_phi) * aerocom_wts; + out(icol, cldfrac_liq_idx) += aerocom_phi * aerocom_wts; // cdnc /* We need to convert nc from 1/mass to 1/volume first, and * from grid-mean to in-cloud, but after that, the * calculation follows the general logic */ auto cdnc = nc_icol(ilay) * pden_icol(ilay) / dz_icol(ilay) / physconst::gravit / cld_icol(ilay); - out(icol, m_index_map["cdnc"]) += cdnc * aerocom_phi * aerocom_wts; - out(icol, m_index_map["nc"]) += - nc_icol(ilay) * aerocom_phi * aerocom_wts; - out(icol, m_index_map["ni"]) += + out(icol, cdnc_idx) += cdnc * aerocom_phi * aerocom_wts; + out(icol, nc_idx) += nc_icol(ilay) * aerocom_phi * aerocom_wts; + out(icol, ni_idx) += ni_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; - out(icol, m_index_map["eff_radius_qc"]) += + out(icol, eff_radius_qc_idx) += rel_icol(ilay) * aerocom_phi * aerocom_wts; - out(icol, m_index_map["eff_radius_qi"]) += + out(icol, eff_radius_qi_idx) += rei_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; // Reset clr_icol to aerocom_tmp to accumulate clr_icol = aerocom_tmp; @@ -223,7 +232,7 @@ void AeroComCld::compute_diagnostic_impl() { // defined as (1 - clr_icol). This is true because // clr_icol is the result of accumulative probabilities // (their products) - out(icol, m_index_map["cldfrac_tot"]) = 1.0 - clr_icol; + out(icol, cldfrac_tot_idx) = 1.0 - clr_icol; }); } diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index bee3ffb15132..d4ead7848284 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -27,8 +27,11 @@ class AeroComCld : public AtmosphereDiagnostic { #endif void compute_diagnostic_impl(); + // Grid info int m_ncols; int m_nlevs; + + // How many diags we have int m_ndiag; // Bot or Top diff --git a/components/eamxx/src/diagnostics/aerocom_cld_util.hpp b/components/eamxx/src/diagnostics/aerocom_cld_util.hpp index 1f8543ed9ef8..dd2cf7323021 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld_util.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld_util.hpp @@ -9,41 +9,42 @@ class AeroComCldDiagUtil { public: std::map index_map; std::map units_map; - int size; + unsigned int size; AeroComCldDiagUtil() { + // Start (post)incrementing size from 0 + size = 0; + // T_mid - index_map["T_mid"] = 0; + index_map["T_mid"] = size++; units_map["T_mid"] = "K"; // p_mid - index_map["p_mid"] = 1; + index_map["p_mid"] = size++; units_map["p_mid"] = "Pa"; // cldfrac_ice - index_map["cldfrac_ice"] = 2; + index_map["cldfrac_ice"] = size++; units_map["cldfrac_ice"] = "nondim"; // cldfrac_liq - index_map["cldfrac_liq"] = 3; + index_map["cldfrac_liq"] = size++; units_map["cldfrac_liq"] = "nondim"; // cdnc - index_map["cdnc"] = 4; + index_map["cdnc"] = size++; units_map["cdnc"] = "#/m3"; // eff_radius_qc - index_map["eff_radius_qc"] = 5; + index_map["eff_radius_qc"] = size++; units_map["eff_radius_qc"] = "micron"; // eff_radius_qi - index_map["eff_radius_qi"] = 6; + index_map["eff_radius_qi"] = size++; units_map["eff_radius_qi"] = "micron"; // cldfrac_tot - index_map["cldfrac_tot"] = 7; + index_map["cldfrac_tot"] = size++; units_map["cldfrac_tot"] = "nondim"; // nc - index_map["nc"] = 8; + index_map["nc"] = size++; units_map["nc"] = "1/kg"; // ni - index_map["ni"] = 9; + index_map["ni"] = size++; units_map["ni"] = "1/kg"; - - size = index_map.size(); } }; From bda8f70fcbaf57ae13fec4c6f734040eab1c3291 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Thu, 2 May 2024 12:45:50 -0400 Subject: [PATCH 121/476] add a banner to scream --- components/eamxx/docs/overrides/main.html | 8 ++++++++ components/eamxx/mkdocs.yml | 1 + 2 files changed, 9 insertions(+) create mode 100644 components/eamxx/docs/overrides/main.html diff --git a/components/eamxx/docs/overrides/main.html b/components/eamxx/docs/overrides/main.html new file mode 100644 index 000000000000..17340d121d44 --- /dev/null +++ b/components/eamxx/docs/overrides/main.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block announce %} +You're not viewing the latest version. + + Go to docs.e3sm.org instead! + +{% endblock %} diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index ccf72c080392..04a7d598fd7d 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -34,6 +34,7 @@ edit_uri: "" theme: name: material + custom_dir: docs/overrides palette: - media: "(prefers-color-scheme: light)" scheme: default From fbf640fa8694057f2ec0b0716dd7ccdf2480b8c6 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 2 May 2024 12:50:18 -0400 Subject: [PATCH 122/476] fixes, etc --- components/eamxx/docs/overrides/main.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/docs/overrides/main.html b/components/eamxx/docs/overrides/main.html index 17340d121d44..7c535238088d 100644 --- a/components/eamxx/docs/overrides/main.html +++ b/components/eamxx/docs/overrides/main.html @@ -1,8 +1,8 @@ {% extends "base.html" %} {% block announce %} -You're not viewing the latest version. - - Go to docs.e3sm.org instead! +EAMxx SCREAM is not supported yet. Read more about this here. + + In the meanwhile, go to docs.e3sm.org instead! {% endblock %} From 67ac2c812622deb07cf2ae34e2199723f9708e6a Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 25 Apr 2024 16:30:35 -0700 Subject: [PATCH 123/476] Add an option to specify a 'longname' for output variables. This commit adds a structure to io_utils that allows the developer to specify a longname to go into the metadata of an output variable. --- .../eamxx/src/share/io/scorpio_output.cpp | 5 ++++- .../eamxx/src/share/io/scorpio_output.hpp | 1 + .../eamxx/src/share/io/scream_io_utils.hpp | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 2762c7fcf8aa..0a6a24999b7f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -964,11 +964,14 @@ register_variables(const std::string& filename, auto vec_of_dims = set_vec_of_dims(layout); std::string units = fid.get_units().get_string(); + // Gather longname + auto longname = m_longnames.get_longname(name); + // TODO Need to change dtype to allow for other variables. // Currently the field_manager only stores Real variables so it is not an issue, // but in the future if non-Real variables are added we will want to accomodate that. - register_variable(filename, name, name, units, vec_of_dims, + register_variable(filename, name, longname, units, vec_of_dims, "real",fp_precision, io_decomp_tag); // Add any extra attributes for this variable diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 13877425f614..a07dd4678dba 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -204,6 +204,7 @@ class AtmosphereOutput std::map> m_diagnostics; std::map> m_diag_depends_on_diags; std::map m_diag_computed; + LongNames m_longnames; // Use float, so that if output fp_precision=float, this is a representable value. // Otherwise, you would get an error from Netcdf, like diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 8cbcd39a1a2c..6b50fbaba576 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -65,5 +65,27 @@ std::string find_filename_in_rpointer ( const ekat::Comm& comm, const util::TimeStamp& run_t0); +struct LongNames { + + std::string get_longname (const std::string& name) { + if (name_2_longname.count(name)>0) { + return name_2_longname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_longname = { + {"lev","hybrid level at midpoints (1000*(A+B))"}, + {"hyai","hybrid A coefficient at layer interfaces"}, + {"hybi","hybrid B coefficient at layer interfaces"}, + {"hyam","hybrid A coefficient at layer midpoints"}, + {"hybm","hybrid B coefficient at layer midpoints"} + }; + +}; + } // namespace scream #endif // SCREAM_IO_UTILS_HPP From 546bc662e0c25564ad4dca61387cb2089c5566db Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 25 Apr 2024 16:34:59 -0700 Subject: [PATCH 124/476] Add the variable 'lev' to the grid data so that it can be output. This commit adds a new grid variable 'lev' which is a representative pressure at each level built from hyam and hybm. It is necessary for some post-processing. Adding this variable to the grid data also makes it available as a default variable in EAMxx output. --- .../share/grid/mesh_free_grids_manager.cpp | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index ed8e5bb2c254..6f08ddb9e19c 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -144,19 +144,22 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); const auto units = ekat::units::Units::nondimensional(); - auto lat = grid->create_geometry_data("lat" , grid->get_2d_scalar_layout(), units); - auto lon = grid->create_geometry_data("lon" , grid->get_2d_scalar_layout(), units); - auto hyam = grid->create_geometry_data("hyam" , layout_mid, units); - auto hybm = grid->create_geometry_data("hybm" , layout_mid, units); + auto lat = grid->create_geometry_data("lat" , grid->get_2d_scalar_layout(), units); + auto lon = grid->create_geometry_data("lon" , grid->get_2d_scalar_layout(), units); + auto hyam = grid->create_geometry_data("hyam" , layout_mid, units); + auto hybm = grid->create_geometry_data("hybm" , layout_mid, units); + auto lev = grid->create_geometry_data("lev" , layout_mid, units); lat.deep_copy(ekat::ScalarTraits::invalid()); lon.deep_copy(ekat::ScalarTraits::invalid()); hyam.deep_copy(ekat::ScalarTraits::invalid()); hybm.deep_copy(ekat::ScalarTraits::invalid()); + lev.deep_copy(ekat::ScalarTraits::invalid()); lat.sync_to_dev(); lon.sync_to_dev(); hyam.sync_to_dev(); hybm.sync_to_dev(); + lev.sync_to_dev(); } else if (geo_data_source=="IC_FILE"){ const auto& filename = m_params.get("ic_filename"); if (scorpio::has_variable(filename,"lat") && @@ -225,9 +228,12 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using namespace ShortFieldTagsNames; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); const auto units = ekat::units::Units::nondimensional(); + auto lev_unit = ekat::units::Units::nondimensional();; + lev_unit.set_string("mb"); auto hyam = grid->create_geometry_data("hyam", layout_mid, units); auto hybm = grid->create_geometry_data("hybm", layout_mid, units); + auto lev = grid->create_geometry_data("lev", layout_mid, lev_unit); // Create host mirrors for reading in data std::map host_views = { @@ -251,9 +257,21 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& vcoord_reader.read_variables(); vcoord_reader.finalize(); + // Build lev from hyam and hybm + using PC = scream::physics::Constants; + const Real ps0 = PC::P0; + + auto hya_v = hyam.get_view(); + auto hyb_v = hybm.get_view(); + auto lev_v = lev.get_view(); + for (int ii=0;iiget_num_vertical_levels();ii++) { + lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + } + // Sync to dev hyam.sync_to_dev(); hybm.sync_to_dev(); + lev.sync_to_dev(); #ifndef NDEBUG for (auto f : {hyam, hybm}) { From ca9c02350a8bdb665bde86495e294f433762fcce Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 26 Apr 2024 09:21:52 -0700 Subject: [PATCH 125/476] fix issue on gpu where a view was not grabbed on Host --- components/eamxx/src/share/grid/mesh_free_grids_manager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index 6f08ddb9e19c..d1525bfec020 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -261,9 +261,9 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using PC = scream::physics::Constants; const Real ps0 = PC::P0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); + auto hya_v = hyam.get_view(); + auto hyb_v = hybm.get_view(); + auto lev_v = lev.get_view(); for (int ii=0;iiget_num_vertical_levels();ii++) { lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); } From f9be31b30a914d9989096e0ba16f8aa1c0fdf363 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 2 May 2024 11:22:53 -0700 Subject: [PATCH 126/476] take subview_1 and improve readability greatly --- .../eamxx/src/diagnostics/aerocom_cld.cpp | 61 ++++++++++--------- .../src/diagnostics/aerocom_cld_util.hpp | 2 +- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.cpp b/components/eamxx/src/diagnostics/aerocom_cld.cpp index 50eea7995c04..8606e065b9e3 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.cpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.cpp @@ -45,6 +45,10 @@ void AeroComCld::set_grids( EKAT_REQUIRE_MSG(m_index_map.size() == m_units_map.size(), "Error! Some inconsistency in AeroComCld: index and units " "maps do not match!\n"); + // Ensure m_index_map and m_ndiag match + EKAT_REQUIRE_MSG(static_cast(m_index_map.size()) == m_ndiag, + "Error! Some inconsistency in AeroComCld: index and units " + "maps do not match!\n"); m_ncols = grid->get_num_local_dofs(); m_nlevs = grid->get_num_vertical_levels(); @@ -133,17 +137,17 @@ void AeroComCld::compute_diagnostic_impl() { const int nlevs = m_nlevs; bool is_top = (m_topbot == "Top"); - // Get indices of each call we care about - const int T_mid_idx = m_index_map["T_mid"]; - const int p_mid_idx = m_index_map["p_mid"]; - const int cldfrac_ice_idx = m_index_map["cldfrac_ice"]; - const int cldfrac_liq_idx = m_index_map["cldfrac_liq"]; - const int cdnc_idx = m_index_map["cdnc"]; - const int eff_radius_qc_idx = m_index_map["eff_radius_qc"]; - const int eff_radius_qi_idx = m_index_map["eff_radius_qi"]; - const int nc_idx = m_index_map["nc"]; - const int ni_idx = m_index_map["ni"]; - const int cldfrac_tot_idx = m_index_map["cldfrac_tot"]; + // subview the out field for each variable + auto o_tmid = ekat::subview_1(out, m_index_map["T_mid"]); + auto o_pmid = ekat::subview_1(out, m_index_map["p_mid"]); + auto o_cldfrac_ice = ekat::subview_1(out, m_index_map["cldfrac_ice"]); + auto o_cldfrac_liq = ekat::subview_1(out, m_index_map["cldfrac_liq"]); + auto o_cdnc = ekat::subview_1(out, m_index_map["cdnc"]); + auto o_eff_radius_qc = ekat::subview_1(out, m_index_map["eff_radius_qc"]); + auto o_eff_radius_qi = ekat::subview_1(out, m_index_map["eff_radius_qi"]); + auto o_nc = ekat::subview_1(out, m_index_map["nc"]); + auto o_ni = ekat::subview_1(out, m_index_map["ni"]); + auto o_cldfrac_tot = ekat::subview_1(out, m_index_map["cldfrac_tot"]); Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { @@ -179,42 +183,39 @@ void AeroComCld::compute_diagnostic_impl() { if((qc_icol(ilay) + qi_icol(ilay)) > q_threshold && (cld_icol(ilay) > cldfrac_tot_threshold)) { /* PART I: Probabilistically determining cloud top/bot */ - // Populate aerocom_tmp as the clear-sky fraction + // Populate clr_tmp as the clear-sky fraction // probability of this level, where clr_icol is that of // the previous level - auto aerocom_tmp = + auto clr_tmp = clr_icol * (1.0 - ekat::impl::max(cld_icol(ilay - 1), cld_icol(ilay))) / (1.0 - ekat::impl::min(cld_icol(ilay - 1), Real(1.0 - cldfrac_tot_threshold))); // Temporary variable for probability "weights" - auto aerocom_wts = clr_icol - aerocom_tmp; + auto wts = clr_icol - clr_tmp; // Temporary variable for liquid "phase" - auto aerocom_phi = qc_icol(ilay) / (qc_icol(ilay) + qi_icol(ilay)); + auto phi = qc_icol(ilay) / (qc_icol(ilay) + qi_icol(ilay)); /* PART II: The inferred properties */ /* In general, converting a 3D property X to a 2D cloud-top * counterpart x follows: x(i) += X(i,k) * weights * Phase * but X and Phase are not always needed */ - out(icol, T_mid_idx) += tmid_icol(ilay) * aerocom_wts; - out(icol, p_mid_idx) += pmid_icol(ilay) * aerocom_wts; - out(icol, cldfrac_ice_idx) += (1.0 - aerocom_phi) * aerocom_wts; - out(icol, cldfrac_liq_idx) += aerocom_phi * aerocom_wts; + o_tmid(icol) += tmid_icol(ilay) * wts; + o_pmid(icol) += pmid_icol(ilay) * wts; + o_cldfrac_ice(icol) += (1.0 - phi) * wts; + o_cldfrac_liq(icol) += phi * wts; // cdnc /* We need to convert nc from 1/mass to 1/volume first, and * from grid-mean to in-cloud, but after that, the * calculation follows the general logic */ auto cdnc = nc_icol(ilay) * pden_icol(ilay) / dz_icol(ilay) / physconst::gravit / cld_icol(ilay); - out(icol, cdnc_idx) += cdnc * aerocom_phi * aerocom_wts; - out(icol, nc_idx) += nc_icol(ilay) * aerocom_phi * aerocom_wts; - out(icol, ni_idx) += - ni_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; - out(icol, eff_radius_qc_idx) += - rel_icol(ilay) * aerocom_phi * aerocom_wts; - out(icol, eff_radius_qi_idx) += - rei_icol(ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset clr_icol to aerocom_tmp to accumulate - clr_icol = aerocom_tmp; + o_cdnc(icol) += cdnc * phi * wts; + o_nc(icol) += nc_icol(ilay) * phi * wts; + o_ni(icol) += ni_icol(ilay) * (1.0 - phi) * wts; + o_eff_radius_qc(icol) += rel_icol(ilay) * phi * wts; + o_eff_radius_qi(icol) += rei_icol(ilay) * (1.0 - phi) * wts; + // Reset clr_icol to clr_tmp to accumulate + clr_icol = clr_tmp; } }; @@ -232,7 +233,7 @@ void AeroComCld::compute_diagnostic_impl() { // defined as (1 - clr_icol). This is true because // clr_icol is the result of accumulative probabilities // (their products) - out(icol, cldfrac_tot_idx) = 1.0 - clr_icol; + o_cldfrac_tot(icol) = 1.0 - clr_icol; }); } diff --git a/components/eamxx/src/diagnostics/aerocom_cld_util.hpp b/components/eamxx/src/diagnostics/aerocom_cld_util.hpp index dd2cf7323021..483c625cd977 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld_util.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld_util.hpp @@ -12,7 +12,7 @@ class AeroComCldDiagUtil { unsigned int size; AeroComCldDiagUtil() { - // Start (post)incrementing size from 0 + // Start (post) incrementing size from 0 size = 0; // T_mid From ca283ab42d5ef43524d3992030618bed7114d880 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Thu, 2 May 2024 13:23:50 -0700 Subject: [PATCH 127/476] Missed one of the locations where grids are constructed --- .../src/dynamics/homme/homme_grids_manager.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 23b47a59aa20..55b236adee04 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -258,17 +258,31 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { auto layout_mid = phys_grid->get_vertical_layout(true); auto layout_int = phys_grid->get_vertical_layout(false); const auto nondim = ekat::units::Units::nondimensional(); + auto lev_unit = ekat::units::Units::nondimensional();; + lev_unit.set_string("mb"); auto hyai = phys_grid->create_geometry_data("hyai",layout_int,nondim); auto hybi = phys_grid->create_geometry_data("hybi",layout_int,nondim); auto hyam = phys_grid->create_geometry_data("hyam",layout_mid,nondim); auto hybm = phys_grid->create_geometry_data("hybm",layout_mid,nondim); + auto lev = phys_grid->create_geometry_data("lev", layout_mid, lev_unit); for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); f.deep_copy(f_d); f.sync_to_host(); } + + // Build lev from hyam and hybm + const Real ps0 = 100000.0; + + auto hya_v = hyam.get_view(); + auto hyb_v = hybm.get_view(); + auto lev_v = lev.get_view(); + for (int ii=0;iiget_num_vertical_levels();ii++) { + lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + } + lev.sync_to_dev(); } if (is_planar_geometry_f90()) { From 97df5871561a49440c13a34da6c1544f9618f6af Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 2 May 2024 14:39:32 -0700 Subject: [PATCH 128/476] add optional pressure layers to nudge DP t and q to --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 7f25fc55fc82..61d17e2ead66 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -554,6 +554,8 @@ be lost if SCREAM_HACK_XML is not enabled. true false false + 1100 + 0 10800 false From 9a1e3ef50408f275cbbdc661009e73096c0c5329 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 2 May 2024 14:42:11 -0700 Subject: [PATCH 129/476] modification to doc comment --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 61d17e2ead66..9a33879b01d8 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -555,7 +555,7 @@ be lost if SCREAM_HACK_XML is not enabled. false false 1100 - 0 + 0 10800 false From ee97964c64296f759700819b023b59ccfb254e02 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 4 May 2024 14:54:14 -0500 Subject: [PATCH 130/476] generalize potential temperature calc Including the LiqPotentialTemperature --- .../src/diagnostics/potential_temperature.cpp | 34 +++++++++++++++++-- .../src/diagnostics/potential_temperature.hpp | 6 +++- .../tests/potential_temperature_test.cpp | 20 ++++++++--- .../eamxx/src/share/io/scorpio_output.cpp | 8 +++++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index ff197eaedea1..d4b9316f6837 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -7,7 +7,26 @@ namespace scream PotentialTemperatureDiagnostic::PotentialTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereDiagnostic(comm,params) { - // Nothing to do here + EKAT_REQUIRE_MSG(params.isParameter("Temperature Kind"), + "Error! PotentialTemperatureDiagnostic requires 'Temperature Kind' in its input parameters.\n"); + + auto pt_type = params.get("Temperature Kind"); + + if (pt_type=="Base"){ + m_ptype = "PotentialTemperature"; + } else if (pt_type=="Liq") { + m_ptype = "LiqPotentialTemperature"; + } else { + EKAT_ERROR_MSG ( + "Error! Invalid choice for 'TemperatureKind' in PotentialTemperatureDiagnostic.\n" + " - input value: " + pt_type + "\n" + " - valid values: Base, Liq\n"); + } +} + +std::string PotentialTemperatureDiagnostic::name() const +{ + return m_ptype; } // ========================================================================================= @@ -30,6 +49,8 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, ps); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); + // Only needed for LiqPotentialTemperature, but put it here for ease + add_field("qc", scalar3d_layout_mid, Q, grid_name, ps); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); @@ -46,12 +67,21 @@ void PotentialTemperatureDiagnostic::compute_diagnostic_impl() auto theta = m_diagnostic_output.get_view(); auto T_mid = get_field_in("T_mid").get_view(); auto p_mid = get_field_in("p_mid").get_view(); + auto q_mid = get_field_in("qc").get_view(); + Kokkos::parallel_for("PotentialTemperatureDiagnostic", Kokkos::RangePolicy<>(0,m_num_cols*npacks), KOKKOS_LAMBDA (const int& idx) { const int icol = idx / npacks; const int jpack = idx % npacks; - theta(icol,jpack) = PF::calculate_theta_from_T(T_mid(icol,jpack),p_mid(icol,jpack)); + auto temp = PF::calculate_theta_from_T(T_mid(icol,jpack),p_mid(icol,jpack)); + if (m_ptype=="LiqPotentialTemperature") { + // Liquid potential temperature (consistent with how it is calculated in SHOC) + theta(icol,jpack) = temp - (temp/T_mid(icol,jpack))*(C::LatVap/C::Cpair)*q_mid(icol,jpack); + } else { + // The regular potential temperature + theta(icol,jpack) = temp; + } }); Kokkos::fence(); } diff --git a/components/eamxx/src/diagnostics/potential_temperature.hpp b/components/eamxx/src/diagnostics/potential_temperature.hpp index 933a6935005d..37fd1a30806c 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.hpp +++ b/components/eamxx/src/diagnostics/potential_temperature.hpp @@ -16,12 +16,13 @@ class PotentialTemperatureDiagnostic : public AtmosphereDiagnostic public: using Pack = ekat::Pack; using PF = scream::PhysicsFunctions; + using C = physics::Constants; // Constructors PotentialTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); // The name of the diagnostic - std::string name () const { return "PotentialTemperature"; } + std::string name () const; // Set the grid void set_grids (const std::shared_ptr grids_manager); @@ -37,6 +38,9 @@ class PotentialTemperatureDiagnostic : public AtmosphereDiagnostic Int m_num_cols; Int m_num_levs; + // What type of potential temperature to compute + std::string m_ptype; + }; // class PotentialTemperatureDiagnostic } //namespace scream diff --git a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp index b11d8aa72ef6..110a064c3988 100644 --- a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp +++ b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp @@ -40,7 +40,7 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { //-----------------------------------------------------------------------------------------------// template -void run(std::mt19937_64& engine) +void run(std::mt19937_64& engine, std::string ptype) { using PF = scream::PhysicsFunctions; using PC = scream::physics::Constants; @@ -67,7 +67,8 @@ void run(std::mt19937_64& engine) // Input (randomized) views view_1d temperature("temperature",num_mid_packs), - pressure("pressure",num_mid_packs); + pressure("pressure",num_mid_packs), + condensate("condensate",num_mid_packs); auto dview_as_real = [&] (const view_1d& v) -> rview_1d { return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); @@ -85,6 +86,7 @@ void run(std::mt19937_64& engine) ekat::ParameterList params; register_diagnostics(); auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + params.set("Temperature Kind", ptype); auto diag = diag_factory.create("PotentialTemperature",comm,params); diag->set_grids(gm); @@ -114,13 +116,18 @@ void run(std::mt19937_64& engine) const auto& T_mid_v = T_mid_f.get_view(); const auto& p_mid_f = input_fields["p_mid"]; const auto& p_mid_v = p_mid_f.get_view(); + const auto& q_mid_f = input_fields["qc"]; + const auto& q_mid_v = q_mid_f.get_view(); for (int icol=0;icol Testing Pack scalar type...",SCREAM_PACK_SIZE); for (int irun=0; irun(engine); + for (const auto& ptype : {"Base","Liq"}) { + run(engine, ptype); + } } printf("ok!\n"); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 3943e59d5da1..4a90998cce49 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1324,6 +1324,14 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "VaporFlux"; // split will return the list [X, ''], with X being whatever is before 'VapFlux' params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); + } else if (diag_field_name=="PotentialTemperature" or + diag_field_name=="LiqPotentialTemperature") { + diag_name = "PotentialTemperature"; + if (diag_field_name == "LiqPotentialTemperature") { + params.set("Temperature Kind", "Liq"); + } else { + params.set("Temperature Kind", "Base"); + } } else { diag_name = diag_field_name; } From 5775ff7078a615e0fe3c67c55fa1e85ff4a5e495 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 5 May 2024 10:43:12 -0500 Subject: [PATCH 131/476] extra string op out of the kernel --- .../src/diagnostics/potential_temperature.cpp | 5 ++-- .../util/scream_common_physics_functions.hpp | 20 ++++++++++++++ .../scream_common_physics_functions_impl.hpp | 27 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index d4b9316f6837..5ab0cb7f176f 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -62,6 +62,7 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr(m_num_levs); auto theta = m_diagnostic_output.get_view(); @@ -75,9 +76,9 @@ void PotentialTemperatureDiagnostic::compute_diagnostic_impl() const int icol = idx / npacks; const int jpack = idx % npacks; auto temp = PF::calculate_theta_from_T(T_mid(icol,jpack),p_mid(icol,jpack)); - if (m_ptype=="LiqPotentialTemperature") { + if (is_liq) { // Liquid potential temperature (consistent with how it is calculated in SHOC) - theta(icol,jpack) = temp - (temp/T_mid(icol,jpack))*(C::LatVap/C::Cpair)*q_mid(icol,jpack); + theta(icol,jpack) = PF::calculate_thetal_from_theta(temp,T_mid(icol,jpack),q_mid(icol,jpack)); } else { // The regular potential temperature theta(icol,jpack) = temp; diff --git a/components/eamxx/src/share/util/scream_common_physics_functions.hpp b/components/eamxx/src/share/util/scream_common_physics_functions.hpp index 92050bccee8a..cda07e55ba65 100644 --- a/components/eamxx/src/share/util/scream_common_physics_functions.hpp +++ b/components/eamxx/src/share/util/scream_common_physics_functions.hpp @@ -94,6 +94,19 @@ struct PhysicsFunctions KOKKOS_INLINE_FUNCTION static ScalarT calculate_theta_from_T(const ScalarT& temperature, const ScalarT& pressure); + //-----------------------------------------------------------------------------------------------// + // Converts potential temperature to liquid potental temperature: + // theta_l = theta - (theta / temperature) * (LatVap/Cpair) * qc, + // where + // theta is the potential temperature, [K] + // temperature is the temperature, [K] + // qc is the cloud liquid mixing ratio, [kg/kg] + // and the others are constants + //-----------------------------------------------------------------------------------------------// + template + KOKKOS_INLINE_FUNCTION + static ScalarT calculate_thetal_from_theta(const ScalarT& theta, const ScalarT& temperature, const ScalarT& qc); + //-----------------------------------------------------------------------------------------------// // Converts potential temperature to temperature using Exners function: // temperature = theta*exner(pressure), @@ -394,6 +407,13 @@ struct PhysicsFunctions template KOKKOS_INLINE_FUNCTION + static void calculate_thetal_from_theta(const MemberType& team, + const InputProviderT& theta, + const InputProviderT& temperature, + const InputProviderP& qc, + const view_1d& thetal); + template + KOKKOS_INLINE_FUNCTION static void calculate_T_from_theta (const MemberType& team, const InputProviderT& theta, const InputProviderP& pressure, diff --git a/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp b/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp index 4a6a654056cb..769622a30adb 100644 --- a/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp +++ b/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp @@ -117,6 +117,18 @@ ScalarT PhysicsFunctions::calculate_theta_from_T(const ScalarT& tempera return temperature/exner_function(pressure); } +template +template +KOKKOS_INLINE_FUNCTION +ScalarT PhysicsFunctions::calculate_thetal_from_theta(const ScalarT& theta, const ScalarT& temperature, const ScalarT& qc) +{ + using C = scream::physics::Constants; + static constexpr auto cpair = C::Cpair; + static constexpr auto latvap = C::LatVap; + + return theta - (theta / temperature) * (latvap/cpair) * qc; +} + template template KOKKOS_INLINE_FUNCTION @@ -131,6 +143,21 @@ void PhysicsFunctions::calculate_theta_from_T(const MemberType& team, }); } +template +template +KOKKOS_INLINE_FUNCTION +void PhysicsFunctions::calculate_thetal_from_theta(const MemberType& team, + const InputProviderT& theta, + const InputProviderT& temperature, + const InputProviderP& qc, + const view_1d& thetal) +{ + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,thetal.extent(0)), + [&] (const int k) { + thetal(k) = calculate_thetal_from_theta(theta(k),temperature(k),qc(k)); + }); +} + template template KOKKOS_INLINE_FUNCTION From 3f16c3ebd0b5be46146deab08b2173170ae053e7 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 5 May 2024 11:26:12 -0700 Subject: [PATCH 132/476] fix the tests --- .../src/diagnostics/tests/potential_temperature_test.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp index 110a064c3988..b342b9372caf 100644 --- a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp +++ b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp @@ -40,7 +40,7 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { //-----------------------------------------------------------------------------------------------// template -void run(std::mt19937_64& engine, std::string ptype) +void run(std::mt19937_64& engine, int int_ptype) { using PF = scream::PhysicsFunctions; using PC = scream::physics::Constants; @@ -86,6 +86,7 @@ void run(std::mt19937_64& engine, std::string ptype) ekat::ParameterList params; register_diagnostics(); auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + std::string ptype = int_ptype == 0 ? "Base" : "Liq"; params.set("Temperature Kind", ptype); auto diag = diag_factory.create("PotentialTemperature",comm,params); diag->set_grids(gm); @@ -141,7 +142,7 @@ void run(std::mt19937_64& engine, std::string ptype) const int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { auto theta = PF::calculate_theta_from_T(T_mid_v(icol,jpack),p_mid_v(icol,jpack)); - if (ptype=="Liq") { + if (int_ptype==1) { theta_v(icol,jpack) = theta - (theta/T_mid_v(icol,jpack)) * (PC::LatVap/PC::Cpair) * q_mid_v(icol,jpack); } else { theta_v(icol,jpack) = theta; } }); @@ -169,8 +170,8 @@ TEST_CASE("potential_temp_test", "potential_temp_test]"){ printf(" -> Testing Pack scalar type...",SCREAM_PACK_SIZE); for (int irun=0; irun(engine, ptype); + for (const int int_ptype : {0,1}) { + run(engine, int_ptype); } } printf("ok!\n"); From aac2fff8b6297a6597d76ce94898f802813d08b6 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sun, 5 May 2024 12:03:14 -0700 Subject: [PATCH 133/476] change name to Tot, add code comments --- components/eamxx/src/diagnostics/potential_temperature.cpp | 6 +++--- .../src/diagnostics/tests/potential_temperature_test.cpp | 2 +- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp | 4 ++++ components/eamxx/src/share/io/scorpio_output.cpp | 2 +- .../src/share/util/scream_common_physics_functions.hpp | 6 ++++++ 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index 5ab0cb7f176f..602f3ffc218a 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -12,7 +12,7 @@ PotentialTemperatureDiagnostic::PotentialTemperatureDiagnostic (const ekat::Comm auto pt_type = params.get("Temperature Kind"); - if (pt_type=="Base"){ + if (pt_type=="Tot"){ m_ptype = "PotentialTemperature"; } else if (pt_type=="Liq") { m_ptype = "LiqPotentialTemperature"; @@ -20,7 +20,7 @@ PotentialTemperatureDiagnostic::PotentialTemperatureDiagnostic (const ekat::Comm EKAT_ERROR_MSG ( "Error! Invalid choice for 'TemperatureKind' in PotentialTemperatureDiagnostic.\n" " - input value: " + pt_type + "\n" - " - valid values: Base, Liq\n"); + " - valid values: Tot, Liq\n"); } } @@ -80,7 +80,7 @@ void PotentialTemperatureDiagnostic::compute_diagnostic_impl() // Liquid potential temperature (consistent with how it is calculated in SHOC) theta(icol,jpack) = PF::calculate_thetal_from_theta(temp,T_mid(icol,jpack),q_mid(icol,jpack)); } else { - // The regular potential temperature + // The total potential temperature theta(icol,jpack) = temp; } }); diff --git a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp index b342b9372caf..d82f85fa8ce6 100644 --- a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp +++ b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp @@ -86,7 +86,7 @@ void run(std::mt19937_64& engine, int int_ptype) ekat::ParameterList params; register_diagnostics(); auto& diag_factory = AtmosphereDiagnosticFactory::instance(); - std::string ptype = int_ptype == 0 ? "Base" : "Liq"; + std::string ptype = int_ptype == 0 ? "Tot" : "Liq"; params.set("Temperature Kind", ptype); auto diag = diag_factory.create("PotentialTemperature",comm,params); diag->set_grids(gm); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 099794adafce..8d9feb496b82 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -107,8 +107,12 @@ class SHOCMacrophysics : public scream::AtmosphereProcess qw(i,k) = qv(i,k) + qc(i,k); // Temperature + // NOTE: vtheta (thv) is inconsistent with one in HOMME + // SEE gh issue #2813 and gh pr #2812 const auto theta_zt = PF::calculate_theta_from_T(T_mid(i,k),p_mid(i,k)); + // TODO? Can use PF::calculate_thetal_from_theta instead? (added in gh pr #2812) thlm(i,k) = theta_zt-(theta_zt/T_mid(i,k))*(latvap/cpair)*qc(i,k); + // TODO? Can use PF::calculate_virtual_temperature instead? (also be consistent with HOMME) thv(i,k) = theta_zt*(1 + zvir*qv(i,k) - qc(i,k)); // Vertical layer thickness diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 4a90998cce49..17c330fb88a1 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1330,7 +1330,7 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { if (diag_field_name == "LiqPotentialTemperature") { params.set("Temperature Kind", "Liq"); } else { - params.set("Temperature Kind", "Base"); + params.set("Temperature Kind", "Tot"); } } else { diag_name = diag_field_name; diff --git a/components/eamxx/src/share/util/scream_common_physics_functions.hpp b/components/eamxx/src/share/util/scream_common_physics_functions.hpp index cda07e55ba65..b926a7b7bc8e 100644 --- a/components/eamxx/src/share/util/scream_common_physics_functions.hpp +++ b/components/eamxx/src/share/util/scream_common_physics_functions.hpp @@ -133,6 +133,12 @@ struct PhysicsFunctions static ScalarT calculate_temperature_from_virtual_temperature(const ScalarT& T_virtual, const ScalarT& qv); //-----------------------------------------------------------------------------------------------// + // FIXME: this comment is different from what's actually happening in the code: + // FIXME: T_virtual = temperature * ( one + c1*qv ) where c1 = - one + one/ep_2 + // FIXME: The code seems to be correct, and the comment is wrong. + // FIXME: ep_2 = 0.622, so the comment produces vpt = pt * (0.75) for qv = 0.5 + // FIXME: but the code produces vpt = pt * (1.30) for qv = 0.5 + // XREFS: pr #1059, issue #2812 // Compute virtual temperature // T_virtual = temperature * (qv+ep_2)/(qv+1) // where From 96cb17563b10ec604b605cb24b63910a45d04ceb Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 15 Jan 2024 17:40:20 -0700 Subject: [PATCH 134/476] EAMxx: minor mod in SE grid CG dofs field is allocated at construction time, so checking whether it is allocated is pointless --- components/eamxx/src/share/grid/se_grid.cpp | 14 -------------- components/eamxx/src/share/grid/se_grid.hpp | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index 741bdb4e640d..dbf950f56475 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -127,20 +127,6 @@ SEGrid::get_3d_tensor_layout (const bool midpoints, return fl.rename_dims(m_special_tag_names); } -Field SEGrid::get_cg_dofs_gids () -{ - EKAT_REQUIRE_MSG (m_cg_dofs_gids.is_allocated(), - "Error! CG dofs have not been created yet.\n"); - return m_cg_dofs_gids; -} - -Field SEGrid::get_cg_dofs_gids () const -{ - EKAT_REQUIRE_MSG (m_cg_dofs_gids.is_allocated(), - "Error! CG dofs have not been created yet.\n"); - return m_cg_dofs_gids.get_const(); -} - std::shared_ptr SEGrid::clone (const std::string& clone_name, const bool shallow) const { auto grid = std::make_shared(clone_name,m_num_local_elem,m_num_gp,get_num_vertical_levels(),get_comm()); diff --git a/components/eamxx/src/share/grid/se_grid.hpp b/components/eamxx/src/share/grid/se_grid.hpp index 79274e96bcac..4f172ed6c29f 100644 --- a/components/eamxx/src/share/grid/se_grid.hpp +++ b/components/eamxx/src/share/grid/se_grid.hpp @@ -43,8 +43,8 @@ class SEGrid : public AbstractGrid } // Retrieve list of the CG grid dofs. Const version returns a read-only field - Field get_cg_dofs_gids (); - Field get_cg_dofs_gids () const; + Field get_cg_dofs_gids () { return m_cg_dofs_gids; } + Field get_cg_dofs_gids () const { return m_cg_dofs_gids.get_const(); } std::shared_ptr clone (const std::string& clone_name, const bool shallow) const override; From 798b1ca0c2628dfe7e44623b48da453749b2ae06 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 15 Jan 2024 17:41:06 -0700 Subject: [PATCH 135/476] EAMxx: add partitioned dim gids to AbstractGrid These are not necessarily the dof GIDs. They are the GIDs of the entries along the partitioned dim that this rank owns. For PointGrid grids, they do coincide, since dofs are COL indices, and COL is also the partitioned dimension. For SEGrid, they are not the same: dofs are GLL points, while the partitioned dim is the ELEM dimension. --- .../dynamics/homme/homme_grids_manager.cpp | 5 ++- .../dynamics/homme/interface/dyn_grid_mod.F90 | 5 +-- .../homme/interface/homme_grid_mod.F90 | 9 +++--- .../interface/scream_homme_interface.hpp | 1 + .../eamxx/src/share/grid/abstract_grid.cpp | 32 +++++++++++++++++++ .../eamxx/src/share/grid/abstract_grid.hpp | 15 ++++++++- .../share/grid/mesh_free_grids_manager.cpp | 8 +++-- .../eamxx/src/share/grid/point_grid.cpp | 3 ++ components/eamxx/src/share/grid/se_grid.cpp | 6 +++- components/eamxx/src/share/grid/se_grid.hpp | 4 +++ 10 files changed, 77 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 55b236adee04..270f1ffbdda7 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -155,21 +155,24 @@ void HommeGridsManager::build_dynamics_grid () { auto dg_dofs = dyn_grid->get_dofs_gids(); auto cg_dofs = dyn_grid->get_cg_dofs_gids(); auto elgpgp = dyn_grid->get_lid_to_idx_map(); + auto elgids = dyn_grid->get_partitioned_dim_gids (); auto lat = dyn_grid->create_geometry_data("lat",layout2d,rad); auto lon = dyn_grid->create_geometry_data("lon",layout2d,rad); auto dg_dofs_h = dg_dofs.get_view(); auto cg_dofs_h = cg_dofs.get_view(); auto elgpgp_h = elgpgp.get_view(); + auto elgids_h = elgids.get_view(); auto lat_h = lat.get_view(); auto lon_h = lon.get_view(); // Get (ie,igp,jgp,gid) data for each dof - get_dyn_grid_data_f90 (dg_dofs_h.data(),cg_dofs_h.data(),elgpgp_h.data(), lat_h.data(), lon_h.data()); + get_dyn_grid_data_f90 (dg_dofs_h.data(),cg_dofs_h.data(),elgpgp_h.data(),elgids_h.data(), lat_h.data(), lon_h.data()); dg_dofs.sync_to_dev(); cg_dofs.sync_to_dev(); elgpgp.sync_to_dev(); + elgids.sync_to_dev(); lat.sync_to_dev(); lon.sync_to_dev(); diff --git a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 index 851659c7f12e..cf8d955fe902 100644 --- a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 @@ -42,7 +42,7 @@ subroutine dyn_grid_init () end subroutine dyn_grid_init - subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, lat, lon) + subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) use iso_c_binding, only: c_int, c_double use dimensions_mod, only: nelemd, np use homme_context_mod, only: elem, par @@ -55,7 +55,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, lat, lon) ! Inputs ! real(kind=c_double), intent(out) :: lat (:,:,:), lon(:,:,:) - integer(kind=c_int), intent(out) :: cg_gids (:), dg_gids(:), elgpgp(:,:) + integer(kind=c_int), intent(out) :: cg_gids (:), dg_gids(:), elgpgp(:,:), elgids(:) ! ! Local(s) ! @@ -80,6 +80,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, lat, lon) call bndry_exchangeV(par,edge) do ie=1,nelemd call edgeVunpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) + elgids(ie) = elem(ie)%GlobalId do ip=1,np do jp=1,np idof = (ie-1)*16+(jp-1)*4+ip diff --git a/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 index 6f11024627f6..28dfed502f8f 100644 --- a/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/homme_grid_mod.F90 @@ -105,18 +105,18 @@ end subroutine finalize_geometry_f90 ! Note: dg_grid=.true. is to request dofs for a Discontinuous Galerkin grid, ! that is, corresponding edge dofs on bordering elems have different gids. ! If dg_grid=.false., the shared dofs have the same gid. - subroutine get_dyn_grid_data_f90 (dg_gids_ptr, cg_gids_ptr, elgpgp_ptr, lat_ptr, lon_ptr) bind(c) + subroutine get_dyn_grid_data_f90 (dg_gids_ptr, cg_gids_ptr, elgpgp_ptr, elgid_ptr, lat_ptr, lon_ptr) bind(c) use dimensions_mod, only: nelemd, np use dyn_grid_mod, only: get_my_dyn_data ! ! Input(s) ! - type (c_ptr), intent(in) :: dg_gids_ptr, cg_gids_ptr, elgpgp_ptr, lat_ptr, lon_ptr + type (c_ptr), intent(in) :: dg_gids_ptr, cg_gids_ptr, elgpgp_ptr, elgid_ptr, lat_ptr, lon_ptr ! ! Local(s) ! real(kind=c_double), pointer :: lat (:,:,:), lon(:,:,:) - integer(kind=c_int), pointer :: cg_gids (:), dg_gids(:), elgpgp(:,:) + integer(kind=c_int), pointer :: cg_gids (:), dg_gids(:), elgpgp(:,:), elgid(:) ! Sanity check call check_grids_inited(.true.) @@ -124,10 +124,11 @@ subroutine get_dyn_grid_data_f90 (dg_gids_ptr, cg_gids_ptr, elgpgp_ptr, lat_ptr, call c_f_pointer (dg_gids_ptr, dg_gids, [nelemd*np*np]) call c_f_pointer (cg_gids_ptr, cg_gids, [nelemd*np*np]) call c_f_pointer (elgpgp_ptr, elgpgp, [3,nelemd*np*np]) + call c_f_pointer (elgid_ptr, elgid, [nelemd]) call c_f_pointer (lat_ptr, lat, [np,np,nelemd]) call c_f_pointer (lon_ptr, lon, [np,np,nelemd]) - call get_my_dyn_data (dg_gids, cg_gids, elgpgp, lat, lon) + call get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgid, lat, lon) end subroutine get_dyn_grid_data_f90 subroutine get_phys_grid_data_f90 (pg_type, gids_ptr, lat_ptr, lon_ptr, area_ptr) bind(c) diff --git a/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp b/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp index 8968fc49fb16..496472b54c0f 100644 --- a/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp +++ b/components/eamxx/src/dynamics/homme/interface/scream_homme_interface.hpp @@ -65,6 +65,7 @@ int get_num_global_elems_f90 (); void get_dyn_grid_data_f90 (AbstractGrid::gid_type* const& dg_gids, AbstractGrid::gid_type* const& cg_gids, int* const& elgp, + AbstractGrid::gid_type* const& elgids, double* const& lat, double* const& lon); void get_phys_grid_data_f90 (const int& pg_type, AbstractGrid::gid_type* const& gids, diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 3e6b9fd0a0c8..0cc83822c23c 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -223,6 +223,26 @@ get_global_max_dof_gid () const ->gid_type return m_global_max_dof_gid; } +auto AbstractGrid:: +get_global_min_partitioned_dim_gid () const ->gid_type +{ + // Lazy calculation + if (m_global_min_partitioned_dim_gid==std::numeric_limits::max()) { + m_global_min_partitioned_dim_gid = field_min(m_partitioned_dim_gids,&get_comm()); + } + return m_global_min_partitioned_dim_gid; +} + +auto AbstractGrid:: +get_global_max_partitioned_dim_gid () const ->gid_type +{ + // Lazy calculation + if (m_global_max_partitioned_dim_gid==-std::numeric_limits::max()) { + m_global_max_partitioned_dim_gid = field_max(m_partitioned_dim_gids,&get_comm()); + } + return m_global_max_partitioned_dim_gid; +} + Field AbstractGrid::get_dofs_gids () const { return m_dofs_gids.get_const(); @@ -233,6 +253,16 @@ AbstractGrid::get_dofs_gids () { return m_dofs_gids; } +Field +AbstractGrid::get_partitioned_dim_gids () { + return m_partitioned_dim_gids; +} + +Field +AbstractGrid::get_partitioned_dim_gids () const { + return m_partitioned_dim_gids.get_const(); +} + Field AbstractGrid::get_lid_to_idx_map () const { return m_lid_to_idx.get_const(); @@ -521,8 +551,10 @@ void AbstractGrid::copy_data (const AbstractGrid& src, const bool shallow) { if (shallow) { m_dofs_gids = src.m_dofs_gids; + m_partitioned_dim_gids = src.m_partitioned_dim_gids; } else { m_dofs_gids = src.m_dofs_gids.clone(); + m_partitioned_dim_gids = src.m_partitioned_dim_gids.clone(); } if (shallow) { diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 00abba1571a7..5da06c1edff9 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -112,11 +112,19 @@ class AbstractGrid : public ekat::enable_shared_from_this gid_type get_num_global_dofs () const { return m_num_global_dofs; } gid_type get_global_min_dof_gid () const; gid_type get_global_max_dof_gid () const; + gid_type get_global_min_partitioned_dim_gid () const; + gid_type get_global_max_partitioned_dim_gid () const; // Get a Field storing 1d data (the dof gids) Field get_dofs_gids () const; Field get_dofs_gids (); + // Get Field storing the gids that this process owns along the partitioned dim + // NOTE: for some grids, this is the same as get_dofs_gids. The SEGrid is a counterexample: + // the dofs are the GLL dofs, but the partitioned dim is the element dimension + Field get_partitioned_dim_gids (); + Field get_partitioned_dim_gids () const; + // Get a Field storing 2d data, where (i,j) entry contains the j-th coordinate of // the i-th dof in the native dof layout. Const verison returns a read-only field Field get_lid_to_idx_map () const; @@ -203,7 +211,6 @@ class AbstractGrid : public ekat::enable_shared_from_this // since it calls get_2d_scalar_layout. void create_dof_fields (const int scalar2d_layout_rank); - // The grid name and type GridType m_type; std::string m_name; @@ -222,9 +229,15 @@ class AbstractGrid : public ekat::enable_shared_from_this // The global ID of each dof Field m_dofs_gids; + // The global ID of the owned entries of the partitioned dimension (if any) + Field m_partitioned_dim_gids; + // The max/min dof GID across all ranks. Mutable, to allow for lazy calculation mutable gid_type m_global_min_dof_gid = std::numeric_limits::max(); mutable gid_type m_global_max_dof_gid = -std::numeric_limits::max(); + // Same as above, but for partitioned dim gids + mutable gid_type m_global_min_partitioned_dim_gid = std::numeric_limits::max(); + mutable gid_type m_global_max_partitioned_dim_gid = -std::numeric_limits::max(); // The fcn is_unique is expensive, so we lazy init this at the first call. mutable bool m_is_unique; diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index d1525bfec020..604ee6ba8cad 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -70,10 +70,12 @@ build_se_grid (const std::string& name, ekat::ParameterList& params) se_grid->setSelfPointer(se_grid); // Set up the degrees of freedom. - auto dof_gids = se_grid->get_dofs_gids(); - auto lid2idx = se_grid->get_lid_to_idx_map(); + auto dof_gids = se_grid->get_dofs_gids(); + auto elem_gids = se_grid->get_partitioned_dim_gids(); + auto lid2idx = se_grid->get_lid_to_idx_map(); auto host_dofs = dof_gids.template get_view(); + auto host_elems = elem_gids.template get_view(); auto host_lid2idx = lid2idx.template get_view(); // Count unique local dofs. On all elems except the very last one (on rank N), @@ -82,6 +84,7 @@ build_se_grid (const std::string& name, ekat::ParameterList& params) int offset = num_local_dofs*m_comm.rank(); for (int ie = 0; ie < num_local_elems; ++ie) { + host_elems[ie] = ie + num_local_elems*m_comm.rank(); for (int igp = 0; igp < num_gp; ++igp) { for (int jgp = 0; jgp < num_gp; ++jgp) { int idof = ie*num_gp*num_gp + igp*num_gp + jgp; @@ -96,6 +99,7 @@ build_se_grid (const std::string& name, ekat::ParameterList& params) // Sync to device dof_gids.sync_to_dev(); + elem_gids.sync_to_dev(); lid2idx.sync_to_dev(); se_grid->m_short_name = "se"; diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index 8cf73e21f2a0..7346020fa3e7 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -15,6 +15,9 @@ PointGrid (const std::string& grid_name, { create_dof_fields (get_2d_scalar_layout().rank()); + // The partitioned dim is the COL dim, which concide with the dofs + m_partitioned_dim_gids = m_dofs_gids; + // The lid->idx map is the identity map. auto lid2idx = get_lid_to_idx_map(); auto h_lid_to_idx = lid2idx.get_view(); diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index dbf950f56475..284191a69daa 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -1,6 +1,7 @@ #include "share/grid/se_grid.hpp" +#include "share/field/field_utils.hpp" -#include "ekat/kokkos//ekat_subview_utils.hpp" +#include namespace scream { @@ -29,6 +30,9 @@ SEGrid (const std::string& grid_name, const auto units = ekat::units::Units::nondimensional(); m_cg_dofs_gids = Field(FieldIdentifier("cg_gids",FieldLayout({CMP},{get_num_local_dofs()}),units,this->name(),DataType::IntType)); m_cg_dofs_gids.allocate_view(); + + m_partitioned_dim_gids = Field(FieldIdentifier("el_gids",FieldLayout({EL},{m_num_local_elem}),units,this->name(),DataType::IntType)); + m_partitioned_dim_gids.allocate_view(); } FieldLayout diff --git a/components/eamxx/src/share/grid/se_grid.hpp b/components/eamxx/src/share/grid/se_grid.hpp index 4f172ed6c29f..87c86887fe31 100644 --- a/components/eamxx/src/share/grid/se_grid.hpp +++ b/components/eamxx/src/share/grid/se_grid.hpp @@ -59,6 +59,10 @@ class SEGrid : public AbstractGrid int m_num_global_elem; int m_num_gp; + // The max/min elem GID across all ranks. Mutable, to allow for lazy calculation + mutable gid_type m_global_min_elem_gid = std::numeric_limits::max(); + mutable gid_type m_global_max_elem_gid = -std::numeric_limits::max(); + // The dofs gids for a CG version of this grid Field m_cg_dofs_gids; }; From 88c6b7a0f927213143fc48646ed62ac57b772577 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 25 Apr 2024 18:30:32 -0600 Subject: [PATCH 136/476] EAMxx: refactor io lib to use scorpio C library * Avoid completely piof lib * Create scream_scorpio_interface minilib, with just pio wrappers * Make IO layer call new scorpio interfaces --- components/eamxx/src/share/io/CMakeLists.txt | 65 +- .../eamxx/src/share/io/scorpio_input.cpp | 188 +- .../eamxx/src/share/io/scorpio_input.hpp | 5 +- .../eamxx/src/share/io/scorpio_output.cpp | 152 +- .../eamxx/src/share/io/scorpio_output.hpp | 4 +- .../src/share/io/scream_io_file_specs.hpp | 3 +- .../eamxx/src/share/io/scream_io_utils.cpp | 22 + .../eamxx/src/share/io/scream_io_utils.hpp | 7 + .../src/share/io/scream_output_manager.cpp | 194 +- .../src/share/io/scream_scorpio_interface.F90 | 1889 ----------------- .../src/share/io/scream_scorpio_interface.cpp | 1876 +++++++++++----- .../src/share/io/scream_scorpio_interface.hpp | 331 +-- .../io/scream_scorpio_interface_iso_c2f.F90 | 520 ----- .../src/share/io/scream_scorpio_types.cpp | 59 + .../src/share/io/scream_scorpio_types.hpp | 154 ++ .../src/share/io/scream_shr_interface_c2f.F90 | 53 + .../src/share/io/scream_shr_interface_c2f.hpp | 16 + .../eamxx/src/share/io/tests/CMakeLists.txt | 6 + .../eamxx/src/share/io/tests/io_basic.cpp | 14 +- .../eamxx/src/share/io/tests/io_diags.cpp | 4 +- .../eamxx/src/share/io/tests/io_filled.cpp | 10 +- .../eamxx/src/share/io/tests/io_monthly.cpp | 4 +- .../eamxx/src/share/io/tests/io_packed.cpp | 4 +- .../src/share/io/tests/io_remap_test.cpp | 58 +- .../eamxx/src/share/io/tests/io_se_grid.cpp | 5 +- .../src/share/io/tests/output_restart.cpp | 5 +- .../io/tests/scorpio_interface_tests.cpp | 217 ++ 27 files changed, 2373 insertions(+), 3492 deletions(-) delete mode 100644 components/eamxx/src/share/io/scream_scorpio_interface.F90 delete mode 100644 components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 create mode 100644 components/eamxx/src/share/io/scream_scorpio_types.cpp create mode 100644 components/eamxx/src/share/io/scream_scorpio_types.hpp create mode 100644 components/eamxx/src/share/io/scream_shr_interface_c2f.F90 create mode 100644 components/eamxx/src/share/io/scream_shr_interface_c2f.hpp create mode 100644 components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index 8866e2944956..afb0ef888db7 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -1,38 +1,65 @@ -set(SCREAM_SCORPIO_SRCS - scream_scorpio_interface.F90 +######################################### +# SCORPIO interface library # +######################################### + +# This small lib contains some interfaces to scorpio, which are formulated +# in terms of names rather than IDs. Internally, we store a list of +# small structs, holding handlers to PIO data and extra metadata, which +# allows to perform sanity checks and print more helpful messages + +add_library(scream_scorpio_interface + scream_scorpio_types.cpp scream_scorpio_interface.cpp - scream_scorpio_interface_iso_c2f.F90 - scream_output_manager.cpp - scorpio_input.cpp - scorpio_output.cpp - scream_io_utils.cpp ) - -# Create io lib -add_library(scream_io ${SCREAM_SCORPIO_SRCS}) -set_target_properties(scream_io PROPERTIES - Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules +target_link_libraries(scream_scorpio_interface PUBLIC ekat) +target_link_libraries(scream_scorpio_interface PRIVATE pioc) +target_include_directories(scream_scorpio_interface PUBLIC + ${SCREAM_BIN_DIR}/src # For scream_config.h ) -target_include_directories(scream_io PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/modules) if (DEFINED ENV{ADIOS2_ROOT}) - target_include_directories(scream_io PRIVATE $ENV{ADIOS2_ROOT}/include) + target_include_directories(scream_scorpio_interface PRIVATE $ENV{ADIOS2_ROOT}/include) endif () if (SCORPIO_Fortran_INCLUDE_DIRS) - target_include_directories(scream_io PUBLIC ${SCORPIO_Fortran_INCLUDE_DIRS}) + target_include_directories(scream_scorpio_interface PUBLIC ${SCORPIO_Fortran_INCLUDE_DIRS}) endif () if (SCORPIO_C_INCLUDE_DIRS) - target_include_directories(scream_io PUBLIC ${SCORPIO_C_INCLUDE_DIRS}) + target_include_directories(scream_scorpio_interface PUBLIC ${SCORPIO_C_INCLUDE_DIRS}) endif () -target_link_libraries(scream_io PUBLIC scream_share piof pioc) - if (SCREAM_CIME_BUILD) - target_link_libraries(scream_io PUBLIC csm_share) + # Add interface to E3SM shr lib (to retrieve PIO subsystem info) + target_sources (scream_scorpio_interface PRIVATE + scream_shr_interface_c2f.F90 + ) + target_compile_definitions (scream_scorpio_interface PRIVATE SCREAM_CIME_BUILD) + set_target_properties(scream_scorpio_interface PROPERTIES + Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules + ) + target_include_directories(scream_scorpio_interface PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/modules) + target_link_libraries(scream_scorpio_interface PUBLIC csm_share) endif() +################################## +# EAMxx I/O library # +################################## + +# This library allows for a simple(r) interaction between EAMxx +# data structures (such as grids, fields, remappers,...) and +# the scorpio interface library + +# Create io lib +add_library(scream_io + scream_output_manager.cpp + scorpio_input.cpp + scorpio_output.cpp + scream_io_utils.cpp +) + +target_link_libraries(scream_io PUBLIC scream_share scream_scorpio_interface) + if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 0c04a612a020..50e1badb00f5 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -220,7 +220,7 @@ void AtmosphereInput::read_variables (const int time_index) // Read the data auto v1d = m_host_views_1d.at(name); - scorpio::grid_read_data_array(m_filename,name,time_index,v1d.data(),v1d.size()); + scorpio::read_var(m_filename,name,v1d.data(),time_index); // If we have a field manager, make sure the data is correctly // synced to both host and device views of the field. @@ -335,7 +335,7 @@ void AtmosphereInput::read_variables (const int time_index) /* ---------------------------------------------------------- */ void AtmosphereInput::finalize() { - scorpio::eam_pio_closefile(m_filename); + scorpio::release_file(m_filename); m_field_mgr = nullptr; m_io_grid = nullptr; @@ -351,51 +351,37 @@ void AtmosphereInput::finalize() void AtmosphereInput::init_scorpio_structures() { std::string iotype_str = m_params.get("iotype", "default"); - int iotype = scorpio::str2iotype(iotype_str); + auto iotype = scorpio::str2iotype(iotype_str); scorpio::register_file(m_filename,scorpio::Read,iotype); - // Register variables with netCDF file. - register_variables(); - set_degrees_of_freedom(); + // scorpio interface. To avoid trouble, if a dim called 'time' is present we - // Finish the definition phase for this file. - scorpio::set_decomp (m_filename); -} - -/* ---------------------------------------------------------- */ -void AtmosphereInput::register_variables() -{ - // Register each variable in IO stream with the SCORPIO interface. - // This allows SCORPIO to lookup vars in the nc file with the correct - // dof decomposition across different ranks. - - // Cycle through all fields - const auto& fp_precision = "real"; + // Check variables are in the input file for (auto const& name : m_fields_names) { // Determine the IO-decomp and construct a vector of dimension ids for this variable: - const auto& layout = m_layouts.at(name); - auto vec_of_dims = get_vec_of_dims(layout); - auto io_decomp_tag = get_io_decomp(layout); - - for (size_t i=0; iget_partitioned_dim_tag()==layout.tags()[i]; - auto dimlen = partitioned ? m_io_grid->get_partitioned_dim_global_size() : layout.dims()[i]; - scorpio::register_dimension(m_filename, vec_of_dims[i], vec_of_dims[i], dimlen, partitioned); - } - - // TODO: Reverse order of dimensions to match flip between C++ -> F90 -> PIO, - // may need to delete this line when switching to full C++/C implementation. - std::reverse(vec_of_dims.begin(),vec_of_dims.end()); - - // Register the variable - // TODO Need to change dtype to allow for other variables. - // Currently the field_manager only stores Real variables so it is not an issue, - // but in the future if non-Real variables are added we will want to accomodate that. - //TODO: Should be able to simply inquire from the netCDF the dimensions for each variable. - scorpio::register_variable(m_filename, name, name, - vec_of_dims, fp_precision, io_decomp_tag); + auto vec_of_dims = get_vec_of_dims(m_layouts.at(name)); + + // Check that the variable is in the file. + EKAT_REQUIRE_MSG (scorpio::has_var(m_filename,name), + "Error! Input file does not store a required variable.\n" + " - filename: " + m_filename + "\n" + " - varname : " + name + "\n"); + + const auto& var = scorpio::get_var(m_filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Dimensions mismatch for input file variable.\n" + " - filename: " + m_filename + "\n" + " - varname : " + name + "\n" + " - expected dims : " + ekat::join(vec_of_dims,",") + "\n" + " - dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + + // Ensure that we can read the var using Real data type + scorpio::change_var_dtype (m_filename,name,"real"); } + + // Set decompositions for the variables + set_decompositions(); } /* ---------------------------------------------------------- */ @@ -418,111 +404,37 @@ AtmosphereInput::get_vec_of_dims(const FieldLayout& layout) } /* ---------------------------------------------------------- */ -std::string AtmosphereInput:: -get_io_decomp(const FieldLayout& layout) -{ - std::string decomp_tag = "dt=real,grid-idx=" + std::to_string(m_io_grid->get_unique_grid_id()) + ",layout="; - - std::vector range(layout.rank()); - std::iota(range.begin(),range.end(),0); - auto tag_and_dim = [&](int i) { - return m_io_grid->get_dim_name(layout,i) + - std::to_string(layout.dim(i)); - }; - - decomp_tag += ekat::join (range, tag_and_dim,"-"); - - return decomp_tag; -} - -/* ---------------------------------------------------------- */ -void AtmosphereInput::set_degrees_of_freedom() -{ - // For each field, tell PIO the offset of each DOF to be read. - // Here, offset is meant in the *global* array in the nc file. - for (auto const& name : m_fields_names) { - auto var_dof = get_var_dof_offsets(m_layouts.at(name)); - scorpio::set_dof(m_filename,name,var_dof.size(),var_dof.data()); - } -} // set_degrees_of_freedom - -/* ---------------------------------------------------------- */ -std::vector -AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) +void AtmosphereInput::set_decompositions() { using namespace ShortFieldTagsNames; - // Precompute this *before* the early return, since it involves collectives. - // If one rank owns zero cols, and returns prematurely, the others will be left waiting. - AbstractGrid::gid_type min_gid; - if (layout.has_tag(COL) or layout.has_tag(EL)) { - min_gid = m_io_grid->get_global_min_dof_gid(); - } - - // It may be that this MPI ranks owns no chunk of the field - if (layout.size()==0) { - return {}; - } + // First, check if any of the vars is indeed partitioned + const auto decomp_tag = m_io_grid->get_partitioned_dim_tag(); - std::vector var_dof(layout.size()); - - // Gather the offsets of the dofs of this variable w.r.t. the *global* array. - // Since we order the global array based on dof gid, and we *assume* (we actually - // check this during set_grid) that the grid global gids are in the interval - // [gid_0, gid_0+num_global_dofs), the offset is simply given by - // (dof_gid-gid_0)*column_size (for partitioned arrays). - // NOTE: a "dof" in the grid object is not the same as a "dof" in scorpio. - // For a SEGrid 3d vector field with (MPI local) layout (nelem,2,np,np,nlev), - // scorpio sees nelem*2*np*np*nlev dofs, while the SE grid sees nelem*np*np dofs. - // All we need to do in this routine is to compute the offset of all the entries - // of the MPI-local array w.r.t. the global array. So long as the offsets are in - // the same order as the corresponding entry in the data to be read/written, we're good. - auto dofs_h = m_io_grid->get_dofs_gids().get_view(); - if (layout.has_tag(COL)) { - const int num_cols = m_io_grid->get_num_local_dofs(); - - // Note: col_size might be *larger* than the number of vertical levels, or even smaller. - // E.g., (ncols,2,nlevs), or (ncols,2) respectively. - scorpio::offset_t col_size = layout.size() / num_cols; - - for (int icol=0; icolget_2d_scalar_layout(); - const int num_my_elems = layout2d.dim(0); - const int ngp = layout2d.dim(1); - const int num_cols = num_my_elems*ngp*ngp; - - // Note: col_size might be *larger* than the number of vertical levels, or even smaller. - // E.g., (ncols,2,nlevs), or (ncols,2) respectively. - scorpio::offset_t col_size = layout.size() / num_cols; - - for (int ie=0,icol=0; ieget_partitioned_dim_local_size(); + auto decomp_dim = m_io_grid->get_dim_name(decomp_tag); + auto gids_f = m_io_grid->get_partitioned_dim_gids(); + auto gids_h = gids_f.get_view(); + auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); + std::vector offsets(local_dim); + for (int idof=0; idof& layouts); void init_scorpio_structures (); - void register_variables(); - void set_degrees_of_freedom(); + void set_decompositions(); std::vector get_vec_of_dims (const FieldLayout& layout); - std::string get_io_decomp (const FieldLayout& layout); - std::vector get_var_dof_offsets (const FieldLayout& layout); // Internal variables ekat::ParameterList m_params; diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 27eff12a4191..e215d0bcc7aa 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -613,7 +613,7 @@ run (const std::string& filename, auto view_host = m_host_views_1d.at(name); Kokkos::deep_copy (view_host,view_dev); auto func_start = std::chrono::steady_clock::now(); - grid_write_data_array(filename,name,view_host.data(),view_host.size()); + scorpio::write_var(filename,name,view_host.data()); auto func_finish = std::chrono::steady_clock::now(); auto duration_loc = std::chrono::duration_cast(func_finish - func_start); duration_write += duration_loc.count(); @@ -627,7 +627,7 @@ run (const std::string& filename, auto view_host = m_host_views_1d.at(name); Kokkos::deep_copy (view_host,view_dev); auto func_start = std::chrono::steady_clock::now(); - grid_write_data_array(filename,name,view_host.data(),view_host.size()); + scorpio::write_var(filename,name,view_host.data()); auto func_finish = std::chrono::steady_clock::now(); auto duration_loc = std::chrono::duration_cast(func_finish - func_start); duration_write += duration_loc.count(); @@ -762,23 +762,17 @@ void AtmosphereOutput::register_dimensions(const std::string& name) if (tag_name=="dim") { tag_name += std::to_string(dims[i]); } - auto tag_loc = m_dims.find(tag_name); auto is_partitioned = m_io_grid->get_partitioned_dim_tag()==tags[i]; - if (tag_loc == m_dims.end()) { - int tag_len = 0; - if(is_partitioned) { - // This is the dimension that is partitioned across ranks. - tag_len = m_io_grid->get_partitioned_dim_global_size(); - } else { - tag_len = layout.dim(i); - } - m_dims[tag_name] = std::make_pair(tag_len,is_partitioned); - } else { - EKAT_REQUIRE_MSG(m_dims.at(tag_name).first==dims[i] or is_partitioned, - "Error! Dimension " + tag_name + " on field " + name + " has conflicting lengths. " - "If same name applies to different dims (e.g. PhysicsGLL and PhysicsPG2 define " - "\"ncol\" at different lengths), reset tag name for one of the grids.\n"); - } + int dim_len = is_partitioned + ? m_io_grid->get_partitioned_dim_global_size() + : layout.dim(i); + auto it_bool = m_dims.emplace(tag_name,dim_len); + EKAT_REQUIRE_MSG(it_bool.second or it_bool.first->second==dim_len, + "Error! Dimension " + tag_name + " on field " + name + " has conflicting lengths.\n" + " - old length: " + std::to_string(m_dims[tag_name]) + "\n" + " - new length: " + std::to_string(dim_len) + "\n" + "If same name applies to different dims (e.g. PhysicsGLL and PhysicsPG2 define " + "\"ncol\" at different lengths), reset tag name for one of the grids.\n"); } } // register_dimensions /* ---------------------------------------------------------- */ @@ -903,7 +897,6 @@ register_variables(const std::string& filename, const std::string& fp_precision, const scorpio::FileMode mode) { - using namespace scorpio; using namespace ShortFieldTagsNames; using strvec_t = std::vector; @@ -940,12 +933,6 @@ register_variables(const std::string& filename, } vec_of_dims.push_back(tag_name); // Add dimensions string to vector of dims. } - // TODO: Reverse order of dimensions to match flip between C++ -> F90 -> PIO, - // may need to delete this line when switching to fully C++/C implementation. - std::reverse(vec_of_dims.begin(),vec_of_dims.end()); - if (m_add_time_dim) { - vec_of_dims.push_back("time"); //TODO: See the above comment on time. - } return vec_of_dims; }; @@ -971,20 +958,46 @@ register_variables(const std::string& filename, // Currently the field_manager only stores Real variables so it is not an issue, // but in the future if non-Real variables are added we will want to accomodate that. - register_variable(filename, name, longname, units, vec_of_dims, - "real",fp_precision, io_decomp_tag); + if (mode==scorpio::FileMode::Append) { + // Simply check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==units, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + units + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + scorpio::define_var (filename, name, units, vec_of_dims, + "real",fp_precision, m_add_time_dim); + + scorpio::set_attribute(filename, name, "long_name", longname); - // Add any extra attributes for this variable - if (mode != FileMode::Append ) { // Add FillValue as an attribute of each variable // FillValue is a protected metadata, do not add it if it already existed if (fp_precision=="double" or (fp_precision=="real" and std::is_same::value)) { double fill_value = m_fill_value; - set_variable_metadata(filename, name, "_FillValue",fill_value); + scorpio::set_attribute(filename, name, "_FillValue",fill_value); } else { float fill_value = m_fill_value; - set_variable_metadata(filename, name, "_FillValue",fill_value); + scorpio::set_attribute(filename, name, "_FillValue",fill_value); } // If this is has subfields, add list of its children @@ -1001,20 +1014,20 @@ register_variables(const std::string& filename, children_list.pop_back(); children_list.pop_back(); children_list += " ]"; - set_variable_metadata(filename,name,"sub_fields",children_list); + scorpio::set_attribute(filename,name,"sub_fields",children_list); } // If tracking average count variables then add the name of the tracking variable for this variable if (m_track_avg_cnt) { const auto lookup = m_field_to_avg_cnt_map.at(name); - set_variable_metadata(filename,name,"averaging_count_tracker",lookup); + scorpio::set_attribute(filename,name,"averaging_count_tracker",lookup); } // Atm procs may have set some request for metadata. using stratts_t = std::map; const auto& str_atts = field.get_header().get_extra_data("io: string attributes"); for (const auto& [att_name,att_val] : str_atts) { - set_variable_metadata(filename,name,att_name,att_val); + scorpio::set_attribute(filename,name,att_name,att_val); } } } @@ -1024,8 +1037,8 @@ register_variables(const std::string& filename, const auto layout = m_layouts.at(name); auto io_decomp_tag = set_decomp_tag(layout); auto vec_of_dims = set_vec_of_dims(layout); - register_variable(filename, name, name, "unitless", vec_of_dims, - "real",fp_precision, io_decomp_tag); + scorpio::define_var(filename, name, "unitless", vec_of_dims, + "real",fp_precision, m_add_time_dim); } } } // register_variables @@ -1112,48 +1125,69 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) return var_dof; } -/* ---------------------------------------------------------- */ -void AtmosphereOutput::set_degrees_of_freedom(const std::string& filename) + +void AtmosphereOutput::set_decompositions(const std::string& filename) { - using namespace scorpio; using namespace ShortFieldTagsNames; - // Cycle through all fields and set dof. - for (auto const& name : m_fields_names) { - auto field = get_field(name,"io"); - const auto& fid = field.get_header().get_identifier(); - auto var_dof = get_var_dof_offsets(fid.get_layout()); - set_dof(filename,name,var_dof.size(),var_dof.data()); + // First, check if any of the vars is indeed partitioned + const auto decomp_tag = m_io_grid->get_partitioned_dim_tag(); + + bool has_decomposed_layouts = false; + for (const auto& it : m_layouts) { + if (it.second.has_tag(decomp_tag)) { + has_decomposed_layouts = true; + break; + } } - // Cycle through the average count fields and set degrees of freedom - for (auto const& name : m_avg_cnt_names) { - const auto layout = m_layouts.at(name); - auto var_dof = get_var_dof_offsets(layout); - set_dof(filename,name,var_dof.size(),var_dof.data()); + if (not has_decomposed_layouts) { + // If none of the vars are decomposed on this grid, + // then there's nothing to do here + return; + } + + // Set the decomposition for the partitioned dimension + const int local_dim = m_io_grid->get_partitioned_dim_local_size(); + auto decomp_dim = m_io_grid->get_dim_name(decomp_tag); + auto gids_f = m_io_grid->get_partitioned_dim_gids(); + auto gids_h = gids_f.get_view(); + auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); + std::vector offsets(local_dim); + for (int idof=0; idof get_var_dof_offsets (const FieldLayout& layout); void register_views(); Field get_field(const std::string& name, const std::string& mode) const; @@ -200,7 +200,7 @@ class AtmosphereOutput std::map m_field_to_avg_cnt_map; std::map m_field_to_avg_cnt_suffix; std::map m_layouts; - std::map> m_dims; + std::map m_dims; std::map> m_diagnostics; std::map> m_diag_depends_on_diags; std::map m_diag_computed; diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp index c80db36e094f..ec0e2f50d796 100644 --- a/components/eamxx/src/share/io/scream_io_file_specs.hpp +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -1,6 +1,7 @@ #ifndef SCREAM_IO_FILE_SPECS_HPP #define SCREAM_IO_FILE_SPECS_HPP +#include "share/io/scream_scorpio_types.hpp" #include "share/io/scream_io_utils.hpp" #include "share/util/scream_time_stamp.hpp" @@ -80,7 +81,7 @@ struct IOFileSpecs { bool is_open = false; std::string filename; - int iotype = 0; + scorpio::IOType iotype = scorpio::IOType::Invalid; // If positive, flush the output file every these many snapshots int flush_frequency = std::numeric_limits::max(); diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 65e5bf030bd5..4a0beb813b76 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -1,4 +1,6 @@ #include "share/io/scream_io_utils.hpp" + +#include "share/io/scream_scorpio_interface.hpp" #include "share/util/scream_utils.hpp" #include @@ -70,4 +72,24 @@ std::string find_filename_in_rpointer ( return filename; } +void write_timestamp (const std::string& filename, const std::string& ts_name, + const util::TimeStamp& ts, const bool write_nsteps) +{ + scorpio::set_attribute(filename,"GLOBAL",ts_name,ts.to_string()); + if (write_nsteps) { + scorpio::set_attribute(filename,"GLOBAL",ts_name+"_nsteps",ts.get_num_steps()); + } +} + +util::TimeStamp read_timestamp (const std::string& filename, + const std::string& ts_name, + const bool read_nsteps) +{ + auto ts = util::str_to_time_stamp(scorpio::get_attribute(filename,"GLOBAL",ts_name)); + if (read_nsteps and scorpio::has_attribute(filename,"GLOBAL",ts_name+"_nsteps")) { + ts.set_num_steps(scorpio::get_attribute(filename,"GLOBAL",ts_name+"_nsteps")); + } + return ts; +} + } // namespace scream diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 6b50fbaba576..781b376858a4 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -87,5 +87,12 @@ struct LongNames { }; +// Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file +void write_timestamp (const std::string& filename, const std::string& ts_name, + const util::TimeStamp& ts, const bool write_nsteps = false); +util::TimeStamp read_timestamp (const std::string& filename, + const std::string& ts_name, + const bool read_nsteps = false); + } // namespace scream #endif // SCREAM_IO_UTILS_HPP diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 2458a4e95c84..33e40978e2d4 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -171,7 +171,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // From restart file, get the time of last write, as well as the current size of the avg sample m_output_control.last_write_ts = read_timestamp(rhist_file,"last_write",true); m_output_control.compute_next_write_ts(); - m_output_control.nsamples_since_last_write = get_attribute(rhist_file,"num_snapshots_since_last_write"); + m_output_control.nsamples_since_last_write = get_attribute(rhist_file,"GLOBAL","num_snapshots_since_last_write"); if (m_avg_type!=OutputAvgType::Instant) { m_time_bnds.resize(2); @@ -190,24 +190,23 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // We do NOT allow changing output specs across restart. If you do want to change // any of these, you MUST start a new output stream (e.g., setting 'Perform Restart: false') - auto old_freq = scorpio::get_attribute(rhist_file,"averaging_frequency"); + auto old_freq = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency"); EKAT_REQUIRE_MSG (old_freq == m_output_control.frequency, "Error! Cannot change frequency when performing history restart.\n" " - old freq: " << old_freq << "\n" " - new freq: " << m_output_control.frequency << "\n"); - auto old_freq_units = scorpio::get_attribute(rhist_file,"averaging_frequency_units"); + auto old_freq_units = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency_units"); EKAT_REQUIRE_MSG (old_freq_units == m_output_control.frequency_units, "Error! Cannot change frequency units when performing history restart.\n" " - old freq units: " << old_freq_units << "\n" " - new freq units: " << m_output_control.frequency_units << "\n"); - auto old_avg_type = scorpio::get_attribute(rhist_file,"averaging_type"); + auto old_avg_type = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_type"); EKAT_REQUIRE_MSG (old_avg_type == e2str(m_avg_type), "Error! Cannot change avg type when performing history restart.\n" " - old avg type: " << old_avg_type + "\n" " - new avg type: " << e2str(m_avg_type) << "\n"); - - auto old_storage_type = scorpio::get_attribute(rhist_file,"file_max_storage_type"); + auto old_storage_type = scorpio::get_attribute(rhist_file,"GLOBAL","file_max_storage_type"); EKAT_REQUIRE_MSG (old_storage_type == e2str(m_output_file_specs.storage.type), "Error! Cannot change file storage type when performing history restart.\n" " - old file_max_storage_type: " << old_storage_type << "\n" @@ -216,7 +215,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, " Restart:\n" " force_new_file: true\n"); if (old_storage_type=="num_snapshot") { - auto old_max_snaps = scorpio::get_attribute(rhist_file,"max_snapshots_per_file"); + auto old_max_snaps = scorpio::get_attribute(rhist_file,"GLOBAL","max_snapshots_per_file"); EKAT_REQUIRE_MSG (old_max_snaps == m_output_file_specs.storage.max_snapshots_in_file, "Error! Cannot change max snapshots per file when performing history restart.\n" " - old max snaps: " << old_max_snaps << "\n" @@ -226,7 +225,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, " force_new_file: true\n"); } std::string fp_precision = m_params.get("Floating Point Precision"); - auto old_fp_precision = scorpio::get_attribute(rhist_file,"fp_precision"); + auto old_fp_precision = scorpio::get_attribute(rhist_file,"GLOBAL","fp_precision"); EKAT_REQUIRE_MSG (old_fp_precision == fp_precision, "Error! Cannot change floating point precision when performing history restart.\n" " - old fp precision: " << old_fp_precision << "\n" @@ -234,12 +233,12 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // Check if the prev run wrote any output file (it may have not, if the restart was written // before the 1st output step). If there is a file, check if there's still room in it. - const auto& last_output_filename = get_attribute(rhist_file,"last_output_filename"); + const auto& last_output_filename = get_attribute(rhist_file,"GLOBAL","last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { scorpio::register_file(last_output_filename,scorpio::Read,m_output_file_specs.iotype); int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); - scorpio::eam_pio_closefile(last_output_filename); + scorpio::release_file(last_output_filename); m_output_file_specs.filename = last_output_filename; m_output_file_specs.is_open = true; @@ -249,7 +248,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // since those are a property of the run, not of the file. setup_file(m_output_file_specs,m_output_control); } - scorpio::eam_pio_closefile(rhist_file); + scorpio::release_file(rhist_file); } } @@ -342,7 +341,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) snapshot_start += m_time_bnds[0]; } if (not filespecs.storage.snapshot_fits(snapshot_start)) { - eam_pio_closefile(filespecs.filename); + release_file(filespecs.filename); filespecs.close(); } @@ -388,14 +387,14 @@ void OutputManager::run(const util::TimeStamp& timestamp) setup_output_file(m_output_control,m_output_file_specs); // Update time (must be done _before_ writing fields) - pio_update_time(m_output_file_specs.filename,timestamp.days_from(m_case_t0)); + update_time(m_output_file_specs.filename,timestamp.days_from(m_case_t0)); } if (is_checkpoint_step) { setup_output_file(m_checkpoint_control,m_checkpoint_file_specs); if (is_full_checkpoint_step) { // Update time (must be done _before_ writing fields) - pio_update_time(m_checkpoint_file_specs.filename,timestamp.days_from(m_case_t0)); + update_time(m_checkpoint_file_specs.filename,timestamp.days_from(m_case_t0)); } } stop_timer(timer_root+"::get_new_file"); @@ -436,44 +435,63 @@ void OutputManager::run(const util::TimeStamp& timestamp) if (m_is_model_restart_output) { // Only write nsteps on model restart - set_attribute(filespecs.filename,"nsteps",timestamp.get_num_steps()); + set_attribute(filespecs.filename,"GLOBAL","nsteps",timestamp.get_num_steps()); } else { if (filespecs.ftype==FileType::HistoryRestart) { // Update the date of last write and sample size - scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); - scorpio::set_attribute (filespecs.filename,"last_output_filename",m_output_file_specs.filename); - scorpio::set_attribute (filespecs.filename,"num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); + write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); + scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); + scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess // output, and the latter b/c we want to make sure these params don't change across restarts - set_attribute(filespecs.filename,"averaging_type",e2str(m_avg_type)); - set_attribute(filespecs.filename,"averaging_frequency_units",m_output_control.frequency_units); - set_attribute(filespecs.filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filespecs.filename,"file_max_storage_type",e2str(m_output_file_specs.storage.type)); + set_attribute(filespecs.filename,"GLOBAL","averaging_type",e2str(m_avg_type)); + set_attribute(filespecs.filename,"GLOBAL","averaging_frequency_units",m_output_control.frequency_units); + set_attribute(filespecs.filename,"GLOBAL","averaging_frequency",m_output_control.frequency); + set_attribute(filespecs.filename,"GLOBAL","file_max_storage_type",e2str(m_output_file_specs.storage.type)); if (m_output_file_specs.storage.type==NumSnaps) { - set_attribute(filespecs.filename,"max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); + set_attribute(filespecs.filename,"GLOBAL","max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); } const auto& fp_precision = m_params.get("Floating Point Precision"); - set_attribute(filespecs.filename,"fp_precision",fp_precision); + set_attribute(filespecs.filename,"GLOBAL","fp_precision",fp_precision); } // Write all stored globals for (const auto& it : m_globals) { const auto& name = it.first; const auto& any = it.second; - set_any_attribute(filespecs.filename,name,any); + if (any.isType()) { + set_attribute(filespecs.filename,"GLOBAL",name,ekat::any_cast(any)); + } else if (any.isType()) { + set_attribute(filespecs.filename,"GLOBAL",name,ekat::any_cast(any)); + } else if (any.isType()) { + set_attribute(filespecs.filename,"GLOBAL",name,ekat::any_cast(any)); + } else if (any.isType()) { + set_attribute(filespecs.filename,"GLOBAL",name,ekat::any_cast(any)); + } else if (any.isType()) { + set_attribute(filespecs.filename,"GLOBAL",name,ekat::any_cast(any)); + } else { + EKAT_ERROR_MSG ( + "Error! Invalid concrete type for IO global.\n" + " - global name: " + it.first + "\n" + " - type id : " + any.content().type().name() + "\n"); + } } // We're adding one snapshot to the file filespecs.storage.update_storage(timestamp); - if (m_time_bnds.size()>0) { - scorpio::grid_write_data_array(filespecs.filename, "time_bnds", m_time_bnds.data(), 2); + // NOTE: for checkpoint files, unless we write restart data, we did not update time, + // which means we cannot write any variable (the check var.num_records==time.length + // would fail) + if (m_time_bnds.size()>0 and + (filespecs.ftype!=FileType::HistoryRestart or is_full_checkpoint_step)) { + scorpio::write_var(filespecs.filename, "time_bnds", m_time_bnds.data()); } // Check if we need to flush the output file if (filespecs.file_needs_flush()) { - eam_flush_file (filespecs.filename); + flush_file (filespecs.filename); } }; @@ -500,10 +518,10 @@ void OutputManager::finalize() { // Close any output file still open if (m_output_file_specs.is_open) { - scorpio::eam_pio_closefile (m_output_file_specs.filename); + scorpio::release_file (m_output_file_specs.filename); } if (m_checkpoint_file_specs.is_open) { - scorpio::eam_pio_closefile (m_checkpoint_file_specs.filename); + scorpio::release_file (m_checkpoint_file_specs.filename); } // Swapping with an empty mgr is the easiest way to cleanup. @@ -691,8 +709,6 @@ void OutputManager:: setup_file ( IOFileSpecs& filespecs, const IOControl& control) { - using namespace scorpio; - const bool is_checkpoint_step = &control==&m_checkpoint_control; std::string fp_precision = is_checkpoint_step @@ -701,59 +717,47 @@ setup_file ( IOFileSpecs& filespecs, const auto& filename = filespecs.filename; // Register new netCDF file for output. Check if we need to append to an existing file - auto mode = m_resume_output_file ? Append : Write; - register_file(filename,mode,filespecs.iotype); + auto mode = m_resume_output_file ? scorpio::Append : scorpio::Write; + scorpio::register_file(filename,mode,filespecs.iotype); if (m_resume_output_file) { - eam_pio_redef(filename); - } + scorpio::redef(filename); + } else { + // Register time (and possibly time_bnds) var(s) + auto time_units="days since " + m_case_t0.get_date_string() + " " + m_case_t0.get_time_string(); + scorpio::define_time(filename,time_units,"time"); - // Note: length=0 is how scorpio recognizes that this is an 'unlimited' dimension, which - // allows to write as many timesnaps as we desire. - register_dimension(filename,"time","time",0,false); + scorpio::define_var(filename,"time",time_units,{}, "double", "double",true); + if (use_leap_year()) { + scorpio::set_attribute (filename,"time","calendar","gregorian"); + } else { + scorpio::set_attribute (filename,"time","calendar","noleap"); + } - // Register time (and possibly time_bnds) var(s) - auto time_units="days since " + m_case_t0.get_date_string() + " " + m_case_t0.get_time_string(); - register_variable(filename,"time","time",time_units,{"time"}, "double", "double","time"); - if (use_leap_year()) { - set_variable_metadata (filename,"time","calendar","gregorian"); - } else { - set_variable_metadata (filename,"time","calendar","noleap"); - } - if (m_avg_type!=OutputAvgType::Instant) { - // First, ensure a 'dim2' dimension with len=2 is registered. - register_dimension(filename,"dim2","dim2",2,false); - register_variable(filename,"time_bnds","time_bnds",time_units,{"dim2","time"},"double","double","time-dim2"); - - // Make it clear how the time_bnds should be interpreted - set_variable_metadata(filename,"time_bnds","note","right endpoint accumulation"); - - // I'm not sure what's the point of this, but CF conventions seem to require it - set_variable_metadata (filename,"time","bounds","time_bnds"); - } + if (m_avg_type!=OutputAvgType::Instant) { + // First, ensure a 'dim2' dimension with len=2 is registered. + scorpio::define_dim(filename,"dim2",2); + scorpio::define_var(filename,"time_bnds",time_units,{"dim2"},"double","double",true); + + // Make it clear how the time_bnds should be interpreted + scorpio::set_attribute (filename,"time_bnds","note","right endpoint accumulation"); + + // I'm not sure what's the point of this, but CF conventions seem to require it + scorpio::set_attribute (filename,"time","bounds","time_bnds"); + } - if (not m_resume_output_file) { - // Finish the definition phase for this file. write_timestamp(filename,"case_t0",m_case_t0); write_timestamp(filename,"run_t0",m_run_t0); - set_attribute(filename,"averaging_type",e2str(m_avg_type)); - set_attribute(filename,"averaging_frequency_units",m_output_control.frequency_units); - set_attribute(filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filespecs.filename,"file_max_storage_type",e2str(m_output_file_specs.storage.type)); + scorpio::set_attribute(filename,"GLOBAL","averaging_type",e2str(m_avg_type)); + scorpio::set_attribute(filename,"GLOBAL","averaging_frequency_units",m_output_control.frequency_units); + scorpio::set_attribute(filename,"GLOBAL","averaging_frequency",m_output_control.frequency); + scorpio::set_attribute(filespecs.filename,"GLOBAL","file_max_storage_type",e2str(m_output_file_specs.storage.type)); if (m_output_file_specs.storage.type==NumSnaps) { - set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); + scorpio::set_attribute(filename,"GLOBAL","max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); } - set_attribute(filename,"fp_precision",fp_precision); + scorpio::set_attribute(filename,"GLOBAL","fp_precision",fp_precision); set_file_header(filespecs); } - // Set degree of freedom for "time" and "time_bnds" - scorpio::offset_t time_dof[1] = {0}; - set_dof(filename,"time",1,time_dof); - if (m_avg_type!=OutputAvgType::Instant) { - scorpio::offset_t time_bnds_dofs[2] = {0,1}; - set_dof(filename,"time_bnds",2,time_bnds_dofs); - } - // Make all output streams register their dims/vars for (auto& it : m_output_streams) { it->setup_output_file(filename,fp_precision,mode); @@ -767,10 +771,7 @@ setup_file ( IOFileSpecs& filespecs, } } - // When resuming a file, PIO opens it in data mode. - // NOTE: all the above register_dimension/register_variable are already checking that - // the dims/vars are already in the file (we don't allow adding dims/vars) - eam_pio_enddef (filename); + scorpio::enddef (filename); if (m_save_grid_data and not filespecs.is_restart_file() and not m_resume_output_file) { // Immediately run the geo data streams @@ -786,11 +787,6 @@ setup_file ( IOFileSpecs& filespecs, /*===============================================================================================*/ void OutputManager::set_file_header(const IOFileSpecs& file_specs) { - using namespace scorpio; - - // TODO: All attributes marked TODO below need to be set. Hopefully by a universal value that reflects - // what the attribute is. For example, git-hash should be the git-hash associated with this version of - // the code at build time for this executable. auto& p = m_params.sublist("provenance"); auto now = std::chrono::system_clock::now(); std::time_t time = std::chrono::system_clock::to_time_t(now); @@ -801,20 +797,24 @@ void OutputManager::set_file_header(const IOFileSpecs& file_specs) const auto& filename = file_specs.filename; - set_attribute(filename,"case",p.get("caseid","NONE")); - set_attribute(filename,"source","E3SM Atmosphere Model (EAMxx)"); - set_attribute(filename,"eamxx_version",EAMXX_VERSION); - set_attribute(filename,"git_version",p.get("git_version",EAMXX_GIT_VERSION)); - set_attribute(filename,"hostname",p.get("hostname","UNKNOWN")); - set_attribute(filename,"username",p.get("username","UNKNOWN")); - set_attribute(filename,"atm_initial_conditions_file",p.get("initial_conditions_file","NONE")); - set_attribute(filename,"topography_file",p.get("topography_file","NONE")); - set_attribute(filename,"contact","e3sm-data-support@llnl.gov"); - set_attribute(filename,"institution_id","E3SM-Projet"); - set_attribute(filename,"realm","atmos"); - set_attribute(filename,"history",ts_str); - set_attribute(filename,"Conventions","CF-1.8"); - set_attribute(filename,"product",e2str(file_specs.ftype)); + auto set_str_att = [&](const std::string& name, const std::string& val) { + scorpio::set_attribute(filename,"GLOBAL",name,val); + }; + + set_str_att("case",p.get("caseid","NONE")); + set_str_att("source","E3SM Atmosphere Model (EAMxx)"); + set_str_att("eamxx_version",EAMXX_VERSION); + set_str_att("git_version",p.get("git_version",EAMXX_GIT_VERSION)); + set_str_att("hostname",p.get("hostname","UNKNOWN")); + set_str_att("username",p.get("username","UNKNOWN")); + set_str_att("atm_initial_conditions_file",p.get("initial_conditions_file","NONE")); + set_str_att("topography_file",p.get("topography_file","NONE")); + set_str_att("contact","e3sm-data-support@llnl.gov"); + set_str_att("institution_id","E3SM-Projet"); + set_str_att("realm","atmos"); + set_str_att("history",ts_str); + set_str_att("Conventions","CF-1.8"); + set_str_att("product",e2str(file_specs.ftype)); } /*===============================================================================================*/ void OutputManager:: diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 deleted file mode 100644 index bfd9ad68f2db..000000000000 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ /dev/null @@ -1,1889 +0,0 @@ -module scream_scorpio_interface - -!==============================================================================! -! This module handles the Fortran interface to the PIO library of input/output -! subroutines. The essential set of steps to enable and use PIO for creating -! output are as follows: -! OUTPUT -! Initialization: -! 1) Gather the pio_subsystem and pio_iotype information for the EAM component -! as assigned by the component coupler. -! This is accomplished during eam_init_pio_1 by calling -! 'eam_init_pio_subsystem' -! 2) For each output file "create" a file in PIO and record the unique file -! descriptor. -! This is accomplished during the eam_init_pio_1 by calling -! 'eam_pio_createfile' -! 3) For each output file define the "header" information, which is essentially -! a set of metadata strings that describe the output file. -! This is accomplished during eam_init_pio_1 by calling 'eam_pio_createHeader' -! 4) Define all of the dimensions that variables in this file will be defined -! on. Examples would time, lat, lon, vertical coordinate, # of consituents, -! etc. -! This is accomplished during 'register_dimension' by which calls 'PIO_def_dim' -! 5) Define all of the variables that will be written to this file. This -! includes any dimension that should also be defined as a variable, such as lat, -! lon, time. -! This is accomplished during 'register_variable' which calls to 'PIO_def_var' -! 6) Determine the unique PIO identifiers for each domain decomposition. This -! is essentially a decomposition of what will be written to the file, multiple -! variables can have the same decomposition. Every arrangement of dimensions -! requires a pio decomposition. -! This is accomplished during 'register_variable' by calling -! 'PIO_initdecomp' -! 7) Close the PIO file definition step. In other words, tell PIO that all of -! the dimensions, variables and decompositions associated with this output have -! been defined and no new ones will be added. -! This is accomplished during eam_pio_enddef by calling 'PIO_enddef' on all -! defined files. -! -! Writing Output: -! -! Finalization: -!==============================================================================! - - !------------ - use pio_types, only: iosystem_desc_t, file_desc_t, var_desc_t, io_desc_t, & - pio_noerr, pio_global, & - PIO_int, PIO_real, PIO_double, PIO_float=>PIO_real,& - pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_adios - use pio_kinds, only: PIO_OFFSET_KIND - - use mpi, only: mpi_abort, mpi_comm_size, mpi_comm_rank - - use iso_c_binding, only: c_float, c_double, c_int - implicit none - save - - public :: & - lookup_pio_atm_file, & ! Checks if a pio file is present - eam_pio_closefile, & ! Close a specfic pio file. - eam_pio_flush_file, & ! Flushes I/O buffers to file - eam_pio_enddef, & ! Ends define mode phase, enters data mode phase - eam_pio_redef, & ! Pause data mode phase, re-enter define mode phase - eam_init_pio_subsystem, & ! Gather pio specific data from the component coupler - is_eam_pio_subsystem_inited, & ! Query whether the pio subsystem is inited already - eam_pio_finalize, & ! Run any final PIO commands - register_file, & ! Creates/opens a pio input/output file - register_variable, & ! Register a variable with a particular pio output file - set_variable_metadata_char, & ! Sets a variable metadata (char data) - set_variable_metadata_float, & ! Sets a variable metadata (float data) - set_variable_metadata_double,& ! Sets a variable metadata (double data) - get_variable_metadata_char, & ! Gets a variable metadata (char data) - get_variable_metadata_float, & ! Gets a variable metadata (float data) - get_variable_metadata_double,& ! Gets a variable metadata (double data) - register_dimension, & ! Register a dimension with a particular pio output file - set_decomp, & ! Set the pio decomposition for all variables in file. - set_dof, & ! Set the pio dof decomposition for specific variable in file. - grid_write_data_array, & ! Write gridded data to a pio managed netCDF file - grid_read_data_array, & ! Read gridded data from a pio managed netCDF file - eam_update_time, & ! Update the timestamp (i.e. time variable) for a given pio netCDF file - read_time_at_index ! Returns the time stamp for a specific time index - - private :: errorHandle, get_coord, is_read, is_write, is_append, scream_iotype_to_pio_iotype - - ! Universal PIO variables for the module - integer :: atm_mpicom - integer :: pio_iotype - type(iosystem_desc_t), pointer, public :: pio_subsystem - integer :: pio_rearranger - integer :: pio_mode - integer :: time_dimid = -1 - - ! TYPES to handle history coordinates and files - integer,parameter :: max_hcoordname_len = 16 - integer,parameter :: max_chars = 256 - integer,parameter :: max_hvarname_len = 64 - integer,parameter :: max_hvar_dimlen = 5 - integer,parameter :: file_purpose_not_set = 0 ! Not set (to catch uninited stuff errors) - integer,parameter :: file_purpose_in = 1 ! Read - integer,parameter :: file_purpose_app = 2 ! Write (append if file exists) - integer,parameter :: file_purpose_out = 4 ! Write (replace if file exists) - - type, public :: hist_coord_t - character(len=max_hcoordname_len) :: name = '' ! coordinate name - integer :: dimid ! Unique PIO Id for this dimension - character(len=max_chars) :: long_name = '' ! 'long_name' attribute - character(len=max_chars) :: units = '' ! 'units' attribute - logical :: is_partitioned ! whether the dimension is partitioned across ranks - logical :: is_time_dim ! whether the dimension is the time (unlimited) dim - end type hist_coord_t - - type, public :: hist_var_t - character(len=max_hvarname_len) :: name ! coordinate name - character(len=max_chars) :: long_name ! 'long_name' attribute - character(len=max_chars) :: pio_decomp_tag ! PIO decomposition label used by this variable. - character(len=max_chars) :: units ! 'units' attribute - type(var_desc_t) :: piovar ! netCDF variable ID - integer :: dtype ! data type used to pass data to read/write routines - integer :: nc_dtype ! data type used in the netcdf files - integer :: numdims ! Number of dimensions in out field - type(io_desc_t), pointer :: iodesc => NULL() ! PIO decomp associated with this variable - type(iodesc_list_t), pointer :: iodesc_list ! PIO decomp list with metadata about PIO decomp - integer(kind=pio_offset_kind), allocatable :: compdof(:) ! Global locations in output array for this process - integer, allocatable :: dimid(:) ! array of PIO dimension id's for this variable - integer, allocatable :: dimlen(:) ! array of PIO dimension lengths for this variable - logical :: has_t_dim ! true, if variable has a time dimension - logical :: is_set = .false. ! Safety measure to ensure a deallocated hist_var_t is never used - logical :: is_partitioned ! Whether at least one of the dims is partitioned - end type hist_var_t - - ! The iodesc_list allows us to cache existing PIO decompositions - ! The tag needs the dim lengths, the dtype and map id (+ optional permutation) - ! Define a recursive structure because we do not know ahead of time how many - ! decompositions will be require - type iodesc_list_t - character(max_chars) :: tag ! Unique tag associated with this decomposition - type(io_desc_t), pointer :: iodesc => NULL() ! PIO - decomposition - type(iodesc_list_t), pointer :: next => NULL() ! Needed for recursive definition, the next list - type(iodesc_list_t), pointer :: prev => NULL() ! Needed for recursive definition, the list that points to this one - logical :: iodesc_set = .false. - integer :: num_customers = 0 ! Track the number of currently active variables that use this pio decomposition - integer :: location = 0 ! where am in the recursive list - end type iodesc_list_t - - ! Define the first iodesc_list_t - type(iodesc_list_t), pointer :: iodesc_list_top -!---------------------------------------------------------------------- - type hist_coord_list_t - type(hist_coord_t), pointer :: coord => NULL() ! Pointer to a history dimension structure - type(hist_coord_list_t), pointer :: next => NULL() ! Needed for recursive definition - end type hist_coord_list_t -!---------------------------------------------------------------------- - type hist_var_list_t - type(hist_var_t), pointer :: var => NULL() ! Pointer to a history variable structure - type(hist_var_list_t), pointer :: next => NULL() ! Needed for recursive definition - end type hist_var_list_t -!---------------------------------------------------------------------- - type pio_file_list_t - type(pio_atm_file_t), pointer :: pio_file => NULL() ! Pointer to an atm. pio file - type(pio_file_list_t), pointer :: next => NULL() ! Needed for recursive definition - type(pio_file_list_t), pointer :: prev => NULL() ! A doubly-linked list is easier to handle - end type pio_file_list_t - ! Define the first pio_file_list - type(pio_file_list_t), pointer :: pio_file_list_front - type(pio_file_list_t), pointer :: pio_file_list_back - -!---------------------------------------------------------------------- - type, public :: pio_atm_file_t - character(len=max_chars) :: filename = "" - integer :: purpose = file_purpose_not_set ! Input or Output file - type(file_desc_t) :: pioFileDesc ! Contains data identifying the file. - type(hist_coord_list_t) :: coord_list_top ! Recursive list of variables - type(hist_var_list_t) :: var_list_top ! Recursive list of variables - integer :: numRecs ! Number of history records on file - logical :: is_enddef = .false. ! Whether definition phase is open - integer :: num_customers ! The number of customer that requested to open the file. - end type pio_atm_file_t - -!---------------------------------------------------------------------- - interface grid_read_data_array - module procedure grid_read_darray_double - module procedure grid_read_darray_float - module procedure grid_read_darray_int - end interface grid_read_data_array -!---------------------------------------------------------------------- - interface grid_write_data_array - module procedure grid_write_darray_float - module procedure grid_write_darray_double - module procedure grid_write_darray_int - end interface -!---------------------------------------------------------------------- - -contains -!=====================================================================! - ! Register a PIO file to be used for input/output operations. - ! If file is already open, ensures file_purpose matches the current one - subroutine register_file(filename,file_purpose,iotype) - - character(len=*), intent(in) :: filename - integer, intent(in) :: file_purpose - integer, intent(in) :: iotype - - type(pio_atm_file_t), pointer :: pio_file - - if (.not.associated(pio_subsystem)) then - call errorHandle("PIO ERROR: local pio_subsystem pointer has not been established yet.",-999) - endif - - call get_pio_atm_file(filename,pio_file,file_purpose,iotype) - end subroutine register_file -!=====================================================================! - ! Mandatory call to finish the variable and dimension definition phase - ! of a new PIO file. Once this routine is called it is not possible - ! to add new dimensions or variables to the file. - subroutine eam_pio_enddef(filename) - use pio, only: PIO_enddef - - character(len=*), intent(in) :: filename - - type(pio_atm_file_t), pointer :: current_atm_file - integer :: ierr - logical :: found - - call lookup_pio_atm_file(filename,current_atm_file,found) - if (.not.found) then - call errorHandle("PIO ERROR: error running enddef on file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! It could happen that we are running a test, with an input file opening the - ! same file that an output stream just wrote. In this case, the def phase ended - ! during the output setup. - if (.not. current_atm_file%is_enddef) then - ! Gather the pio decomposition for all variables in this file, and assign them pointers. - call set_decomp(trim(filename)) - ! Officially close the definition step for this file. - ierr = PIO_enddef(current_atm_file%pioFileDesc) - call errorHandle("PIO ERROR: issue arose with PIO_enddef for file"//trim(current_atm_file%filename),ierr) - current_atm_file%is_enddef = .true. - endif - - end subroutine eam_pio_enddef -!=====================================================================! - ! Mandatory call to finish the variable and dimension definition phase - ! of a new PIO file. Once this routine is called it is not possible - ! to add new dimensions or variables to the file. - subroutine eam_pio_redef(filename) - use pio, only: pio_redef - - character(len=*), intent(in) :: filename - - type(pio_atm_file_t), pointer :: current_atm_file - integer :: ierr - logical :: found - - call lookup_pio_atm_file(filename,current_atm_file,found) - if (.not.found) then - call errorHandle("PIO ERROR: error running redef on file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! It could happen that we are running a test, with an input file opening the - ! same file that an output stream just wrote. In this case, the def phase ended - ! during the output setup. - if (current_atm_file%is_enddef) then - ! Re-open define phase - ierr = PIO_redef(current_atm_file%pioFileDesc) - call errorHandle("PIO ERROR: issue arose with PIO_redef for file"//trim(current_atm_file%filename),ierr) - current_atm_file%is_enddef = .false. - endif - - end subroutine eam_pio_redef -!=====================================================================! - ! Register a dimension with a specific pio output file. Mandatory inputs - ! include: - ! filename: Name of file to add the dimension to. - ! shortname: Short name descriptor for this dimension. This will be - ! name to find the dimension in the netCDF file. - ! longname: A longer character string with a more descriptive name of - ! the dimension. - ! length: The dimension length (must be >=0). Choosing 0 marks the - ! dimensions as having "unlimited" length which is used for - ! dimensions such as time. - ! NOTE: if the file is in Read or Append mode, we are supposed to have already checked - ! that the specs match the ones in the file during the C++ wrapper functions - subroutine register_dimension(filename,shortname,longname,length,is_partitioned) - use pio_types, only: pio_unlimited - use pio, only: PIO_def_dim - - character(len=*), intent(in) :: filename ! Name of file to register the dimension on. - character(len=*), intent(in) :: shortname,longname ! Short- and long- names for this dimension, short: brief identifier and name for netCDF output, long: longer descriptor sentence to be included as meta-data in file. - integer, intent(in) :: length ! Length of the dimension, 0: unlimited (like time), >0 actual length of dimension - logical, intent(in) :: is_partitioned ! whether this dimension is partitioned across ranks - - type(pio_atm_file_t), pointer :: pio_atm_file - type(hist_coord_t), pointer :: hist_coord - type(hist_coord_list_t), pointer :: curr, prev - logical :: found, dim_found - integer :: ierr - - dim_found = .false. - - ! Make sure the dimension length is reasonable - if (length<0) call errorHandle("PIO Error: dimension "//trim(shortname)//", can't have a negative dimension length",-999) - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: error registering dimension "//trim(shortname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! Get a new dimension pointer in coord_list - curr => pio_atm_file%coord_list_top - do while (associated(curr)) - if (associated(curr%coord)) then - if(trim(curr%coord%name)==trim(shortname)) then - dim_found = .true. - exit - endif - end if - prev => curr - curr => prev%next - end do - - ! If the dim was not found, create it - if (.not. dim_found) then - allocate(prev%next) - curr => prev%next - allocate(curr%coord) - hist_coord => curr%coord - - ! Register this dimension - hist_coord%name = trim(shortname) - hist_coord%long_name = trim(longname) - hist_coord%is_partitioned = is_partitioned - hist_coord%is_time_dim = length .eq. 0 - - if (is_write(pio_atm_file%purpose)) then - if (length.eq.0) then - ierr = PIO_def_dim(pio_atm_file%pioFileDesc, trim(shortname), pio_unlimited , hist_coord%dimid) - time_dimid = hist_coord%dimid - else - ierr = PIO_def_dim(pio_atm_file%pioFileDesc, trim(shortname), length , hist_coord%dimid) - end if - call errorHandle("PIO ERROR: could not define dimension "//trim(shortname)//" on file: "//trim(filename),ierr) - endif - endif - end subroutine register_dimension -!=====================================================================! - ! Register a variable with a specific pio input/output file. Mandatory inputs - ! include: - ! pio_atm_filename: The name of the netCDF file this variable will be - ! registered with. - ! shortname: A shortname descriptor (tag) for this variable. This will be - ! used to label the variable in the netCDF file as well. - ! longname: A longer character string describing the variable. - ! numdims: The number of dimensions associated with this variable, - ! including time (if applicable). - ! var_dimensions: An array of character strings with the dimension shortnames - ! for each dimension used by this variable. Should have - ! 'numdims' entries. - ! dtype: The data type for this variable using the proper netCDF - ! integer tag. - ! pio_decomp_tag: A string that describes this particular dimension - ! arrangement which will be used to create a unique PIO - ! decomposition for reading this variable. It is ok to reuse - ! the pio_decomp_tag for variables that have the same - ! dimensionality. See get_decomp for more details. - ! NOTE: if the file is in Read or Append mode, we are supposed to have already checked - ! that the specs match the ones in the file during the C++ wrapper functions - subroutine register_variable(filename,shortname,longname,units, & - numdims,var_dimensions, & - dtype,nc_dtype,pio_decomp_tag) - use pio, only: PIO_def_var, PIO_inq_dimid, PIO_inq_dimlen, PIO_inq_varid, PIO_put_att - - character(len=256), intent(in) :: filename ! Name of the file to register this variable with - character(len=256), intent(in) :: shortname,longname ! short and long names for the variable. Short: variable name in file, Long: more descriptive name - character(len=256), intent(in) :: units ! units for variable - integer, intent(in) :: numdims ! Number of dimensions for this variable, including time dimension - character(len=256), intent(in) :: var_dimensions(numdims) ! String array with shortname descriptors for each dimension of variable. - character(len=256), intent(in) :: pio_decomp_tag ! Unique tag for this variables decomposition type, to be used to determine if the io-decomp already exists. - integer, intent(in) :: dtype ! datatype for arrays that will be passed to read/write routines - integer, intent(in) :: nc_dtype ! datatype for this variable in nc files (unused if file mode is Read) - - ! Local variables - type(pio_atm_file_t),pointer :: pio_atm_file - type(hist_var_t), pointer :: hist_var - integer :: dim_ii - logical :: found,var_found - integer :: ierr - type(hist_coord_t), pointer :: hist_coord - - type(hist_var_list_t), pointer :: curr, prev - - var_found = .false. - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: error registering variable "//trim(shortname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! Get a new variable pointer in var_list - if (len_trim(shortname)>max_hvarname_len) call errorHandle("PIO Error: variable shortname "//trim(shortname)//" is too long, consider increasing max_hvarname_len or changing the variable shortname",-999) - curr => pio_atm_file%var_list_top - do while ( associated(curr) ) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(shortname) .and. curr%var%is_set) then - exit - end if - end if - prev => curr - curr => prev%next - end do - - allocate(prev%next) - curr => prev%next - allocate(curr%var) - hist_var => curr%var - ! Populate meta-data associated with this variable - hist_var%name = trim(shortname) - hist_var%long_name = trim(longname) - - hist_var%units = trim(units) - hist_var%numdims = numdims - hist_var%dtype = dtype - hist_var%nc_dtype = nc_dtype - hist_var%pio_decomp_tag = trim(pio_decomp_tag) - ! Determine the dimension id's saved in the netCDF file and associated with - ! this variable, check if variable has a time dimension - hist_var%has_t_dim = .false. - hist_var%is_partitioned = .false. - allocate(hist_var%dimid(numdims),hist_var%dimlen(numdims)) - do dim_ii = 1,numdims - ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim(var_dimensions(dim_ii)),hist_var%dimid(dim_ii)) - call errorHandle("EAM_PIO ERROR: Unable to find dimension id for "//trim(var_dimensions(dim_ii)),ierr) - ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,hist_var%dimid(dim_ii),hist_var%dimlen(dim_ii)) - call errorHandle("EAM_PIO ERROR: Unable to determine length for dimension "//trim(var_dimensions(dim_ii)),ierr) - - call get_coord (filename,var_dimensions(dim_ii),hist_coord) - if (hist_coord%is_partitioned) then - hist_var%is_partitioned = .true. - endif - if (hist_coord%is_time_dim) then - hist_var%has_t_dim = .true. - endif - end do - - if (is_write(pio_atm_file%purpose)) then - ierr = PIO_def_var(pio_atm_file%pioFileDesc, trim(shortname), hist_var%nc_dtype, hist_var%dimid(:numdims), hist_var%piovar) - call errorHandle("PIO ERROR: could not define variable "//trim(shortname)//" in file "//trim(filename),ierr) - ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'units', hist_var%units ) - call errorHandle("PIO ERROR: could not set attribute 'units' for variable "//trim(shortname)//" in file "//trim(filename),ierr) - ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'long_name', hist_var%long_name ) - call errorHandle("PIO ERROR: could not set attribute 'long_name' for variable "//trim(shortname)//" in file "//trim(filename),ierr) - else - ierr = PIO_inq_varid(pio_atm_file%pioFileDesc,trim(shortname),hist_var%piovar) - call errorHandle("PIO ERROR: could not retrieve id for variable "//trim(shortname)//" from file "//trim(filename),ierr) - endif - - ! Set that new variable has been created - hist_var%is_set = .true. - - end subroutine register_variable -!=====================================================================! - subroutine set_variable_metadata_float(filename, varname, metaname, metaval) - use pio, only: PIO_put_att - - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: varname - character(len=256), intent(in) :: metaname - real(kind=c_float), intent(in) :: metaval - - ! Local variables - type(pio_atm_file_t),pointer :: pio_file - type(hist_var_t), pointer :: var - integer :: ierr - logical :: found - - type(hist_var_list_t), pointer :: curr - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! Find the variable in the file - curr => pio_file%var_list_top - - found = .false. - do while (associated(curr)) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then - found = .true. - var => curr%var - exit - endif - endif - curr => curr%next - end do - if (.not.found ) then - call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n Variable not found.",-999) - endif - - ierr = PIO_put_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) - if (ierr .ne. 0) then - call errorHandle("Error setting attribute '" // trim(metaname) & - // "' on variable '" // trim(varname) & - // "' in pio file " // trim(filename) // ".", -999) - endif - - end subroutine set_variable_metadata_float -!=====================================================================! - subroutine set_variable_metadata_double(filename, varname, metaname, metaval) - use pio, only: PIO_put_att - - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: varname - character(len=256), intent(in) :: metaname - real(kind=c_double), intent(in) :: metaval - - ! Local variables - type(pio_atm_file_t),pointer :: pio_file - type(hist_var_t), pointer :: var - integer :: ierr - logical :: found - - type(hist_var_list_t), pointer :: curr - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! Find the variable in the file - curr => pio_file%var_list_top - - found = .false. - do while (associated(curr)) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then - found = .true. - var => curr%var - exit - endif - endif - curr => curr%next - end do - if (.not.found ) then - call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n Variable not found.",-999) - endif - - ierr = PIO_put_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) - if (ierr .ne. 0) then - call errorHandle("Error setting attribute '" // trim(metaname) & - // "' on variable '" // trim(varname) & - // "' in pio file " // trim(filename) // ".", -999) - endif - - end subroutine set_variable_metadata_double -!=====================================================================! - function get_variable_metadata_float(filename, varname, metaname) result(metaval) - use pio, only: PIO_get_att - - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: varname - character(len=256), intent(in) :: metaname - real(kind=c_float) :: metaval - - - ! Local variables - type(pio_atm_file_t),pointer :: pio_file - type(hist_var_t), pointer :: var - integer :: ierr - logical :: found - - type(hist_var_list_t), pointer :: curr - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) - endif - - ! Find the variable in the file - curr => pio_file%var_list_top - - found = .false. - do while (associated(curr)) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then - found = .true. - var => curr%var - exit - endif - endif - curr => curr%next - end do - if (.not.found ) then - call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) - endif - - ! TODO: Maybe we shouldn't throw an error when the metadata isn't there, maybe we provide an output like ierr as an integer? - ! That way we can query for a metadata and on the EAMxx side decide what to do if the metadata is missing. - ierr = PIO_get_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) - if (ierr .ne. 0) then - call errorHandle("Error getting attribute '" // trim(metaname) & - // "' on variable '" // trim(varname) & - // "' in pio file " // trim(filename) // ".", -999) - endif - - end function get_variable_metadata_float -!=====================================================================! - function get_variable_metadata_double(filename, varname, metaname) result(metaval) - use pio, only: PIO_get_att - - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: varname - character(len=256), intent(in) :: metaname - real(kind=c_double) :: metaval - - ! Local variables - type(pio_atm_file_t),pointer :: pio_file - type(hist_var_t), pointer :: var - integer :: ierr - logical :: found - - type(hist_var_list_t), pointer :: curr - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) - endif - - ! Find the variable in the file - curr => pio_file%var_list_top - - found = .false. - do while (associated(curr)) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then - found = .true. - var => curr%var - exit - endif - endif - curr => curr%next - end do - if (.not.found ) then - call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) - endif - - ierr = PIO_get_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) - if (ierr .ne. 0) then - call errorHandle("Error getting attribute '" // trim(metaname) & - // "' on variable '" // trim(varname) & - // "' in pio file " // trim(filename) // ".", -999) - endif - - end function get_variable_metadata_double -!=====================================================================! - subroutine set_variable_metadata_char(filename, varname, metaname, metaval) - use pio, only: PIO_put_att - - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: varname - character(len=256), intent(in) :: metaname - character(len=256), intent(in) :: metaval - - ! Local variables - type(pio_atm_file_t),pointer :: pio_file - type(hist_var_t), pointer :: var - integer :: ierr - logical :: found - - type(hist_var_list_t), pointer :: curr - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - ! Find the variable in the file - curr => pio_file%var_list_top - - found = .false. - do while (associated(curr)) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then - found = .true. - var => curr%var - exit - endif - endif - curr => curr%next - end do - if (.not.found ) then - call errorHandle("PIO ERROR: error setting metadata for variable "//trim(varname)//" in file "//trim(filename)//".\n Variable not found.",-999) - endif - - ierr = PIO_put_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) - if (ierr .ne. 0) then - call errorHandle("Error setting attribute '" // trim(metaname) & - // "' on variable '" // trim(varname) & - // "' in pio file " // trim(filename) // ".", -999) - endif - - end subroutine set_variable_metadata_char -!=====================================================================! - function get_variable_metadata_char(filename, varname, metaname) result(metaval) - use pio, only: PIO_get_att - - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: varname - character(len=256), intent(in) :: metaname - character(len=256) :: metaval - - ! Local variables - type(pio_atm_file_t),pointer :: pio_file - type(hist_var_t), pointer :: var - integer :: ierr - logical :: found - - type(hist_var_list_t), pointer :: curr - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (.not.found ) then - call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) - endif - - ! Find the variable in the file - curr => pio_file%var_list_top - - found = .false. - do while (associated(curr)) - if (associated(curr%var)) then - if (trim(curr%var%name)==trim(varname) .and. curr%var%is_set) then - found = .true. - var => curr%var - exit - endif - endif - curr => curr%next - end do - if (.not.found ) then - call errorHandle("PIO ERROR: PIO file not open.\nError getting metadata for variable "//trim(varname)//" in file "//trim(filename)//".",-999) - endif - - ! TODO: Maybe we shouldn't throw an error when the metadata isn't there, maybe we provide an output like ierr as an integer? - ! That way we can query for a metadata and on the EAMxx side decide what to do if the metadata is missing. - ierr = PIO_get_att(pio_file%pioFileDesc, var%piovar, metaname, metaval) - if (ierr .ne. 0) then - call errorHandle("Error getting attribute '" // trim(metaname) & - // "' on variable '" // trim(varname) & - // "' in pio file " // trim(filename) // ".", -999) - endif - end function get_variable_metadata_char -!=====================================================================! - ! Update the time dimension for a specific PIO file. This is needed when - ! reading or writing multiple time levels. Unlimited dimensions are treated - ! differently in netCDF than typical static length variables. Note, here - ! "time" is hardcoded as the only unlimited variable. If, in the future, - ! scream decides to allow for other "unlimited" dimensions to be used our - ! input/output than this routine will need to be adjusted. - subroutine eam_update_time(filename,time) - use pio, only: PIO_put_var - - character(len=*), intent(in) :: filename ! PIO filename - real(c_double), intent(in) :: time - - type(hist_var_t), pointer :: var - type(pio_atm_file_t),pointer :: pio_atm_file - integer :: ierr - logical :: found - - call lookup_pio_atm_file(filename,pio_atm_file,found) - pio_atm_file%numRecs = pio_atm_file%numRecs + 1 - call get_var(pio_atm_file,'time',var) - ! Only update time on the file if a valid time is provided - if (time>=0) then - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,(/ pio_atm_file%numRecs /), (/ 1 /), (/ time /)) - call errorHandle("PIO ERROR: something went wrong while writing time var on file="//trim(filename),ierr) - endif - end subroutine eam_update_time -!=====================================================================! - ! Assign institutions to header metadata for a specific pio output file. - subroutine eam_pio_createHeader(File) - use pio, only: PIO_put_att - - type(file_desc_t), intent(in) :: File ! Pio file Handle - integer :: retval - - ! We are able to have EAMxx directly set most attributes in the HEADER - ! except the list of institutions which appears to have a string that is too - ! long to accomodate using `set_str_attribute` as it is currently defined. - ! So we keep the setting of institutions here. - ! TODO: revise the set_str_attribute code to allow the - ! scream_output_manager.cpp to handle institutions too. - ! NOTE: The use of //char(10)// causes each institution to be written on it's own line, makes it easier to read. - retval=pio_put_att (File, PIO_GLOBAL, 'institutions', 'LLNL (Lawrence Livermore National Laboratory, Livermore, CA 94550, USA);' & - //char(10)//'ANL (Argonne National Laboratory, Argonne, IL 60439, USA); ' & - //char(10)//'BNL (Brookhaven National Laboratory, Upton, NY 11973, USA);' & - //char(10)//'LANL (Los Alamos National Laboratory, Los Alamos, NM 87545, USA);' & - //char(10)//'LBNL (Lawrence Berkeley National Laboratory, Berkeley, CA 94720, USA);' & - //char(10)//'ORNL (Oak Ridge National Laboratory, Oak Ridge, TN 37831, USA);' & - //char(10)//'PNNL (Pacific Northwest National Laboratory, Richland, WA 99352, USA);' & - //char(10)//'SNL (Sandia National Laboratories, Albuquerque, NM 87185, USA).' & - //char(10)//'Mailing address: LLNL Climate Program, c/o David C. Bader, Principal Investigator, L-103, 7000 East Avenue, Livermore, CA 94550, USA') - - end subroutine eam_pio_createHeader -!=====================================================================! - ! Ensures a pio system is in place, by either creating a new one - ! or getting the one created by CIME (for CIME builds) - subroutine eam_init_pio_subsystem(mpicom,atm_id) -#ifdef SCREAM_CIME_BUILD - use shr_pio_mod, only: shr_pio_getrearranger, shr_pio_getiosys, & - shr_pio_getiotype, shr_pio_getioformat -#else - use pio_types, only: pio_rearr_subset, PIO_iotype_netcdf, PIO_64BIT_DATA - use pio, only: pio_init - - integer :: ierr, stride, atm_rank, atm_size, num_aggregator -#endif - - integer, intent(in) :: mpicom - integer, intent(in) :: atm_id - - if (associated(pio_subsystem)) call errorHandle("PIO ERROR: local pio_subsystem pointer has already been established.",-999) - - atm_mpicom = mpicom - -#ifdef SCREAM_CIME_BUILD - pio_subsystem => shr_pio_getiosys(atm_id) - pio_iotype = shr_pio_getiotype(atm_id) - pio_rearranger = shr_pio_getrearranger(atm_id) - pio_mode = shr_pio_getioformat(atm_id) -#else - ! WARNING: we're assuming *every atm rank* is an I/O rank - call MPI_Comm_rank(atm_mpicom, atm_rank, ierr) - call MPI_Comm_size(atm_mpicom, atm_size, ierr) - - ! Just for removing unused dummy warnings - if (.false.) print *, atm_id - - stride = 1 - num_aggregator = 0 - - allocate(pio_subsystem) - pio_rearranger = pio_rearr_subset - pio_iotype = PIO_iotype_netcdf - pio_mode = PIO_64BIT_DATA ! Default to 64 bit - call PIO_init(atm_rank, atm_mpicom, atm_size, num_aggregator, stride, & - pio_rearr_subset, pio_subsystem, base=0) -#endif - - ! Init the list of pio files so that begin==end==null - pio_file_list_back => null() - pio_file_list_front => null() - - ! Init the iodecomp - iodesc_list_top => null() - - end subroutine eam_init_pio_subsystem -!=====================================================================! - ! Query whether the pio subsystem is inited already - ! This can be useful to avoid double-init or double-finalize calls. - function is_eam_pio_subsystem_inited() result(is_it) bind(c) - use iso_c_binding, only: c_bool - - logical(kind=c_bool) :: is_it - - is_it = LOGICAL(associated(pio_subsystem),kind=c_bool) - end function is_eam_pio_subsystem_inited -!=====================================================================! - function scream_iotype_to_pio_iotype(siotype) result(piotype) - integer, intent(in) :: siotype - integer :: piotype - - if(siotype == 0) then - piotype = pio_iotype - else if(siotype == 1) then - piotype = pio_iotype_netcdf - else if(siotype == 2) then - piotype = pio_iotype_pnetcdf - else if(siotype == 3) then - piotype = pio_iotype_adios - else - piotype = pio_iotype - end if - end function scream_iotype_to_pio_iotype -!=====================================================================! - ! Create a pio netCDF file with the appropriate name. - subroutine eam_pio_createfile(File,fname,iotype) - use pio, only: pio_createfile - use pio_types, only: pio_clobber - - type(file_desc_t), intent(inout) :: File ! Pio file Handle - character(len=*), intent(in) :: fname ! Pio file name - integer, intent(in) :: iotype - !-- - integer :: retval ! PIO error return value - integer :: mode ! Mode for how to handle the new file - integer :: piotype - - mode = ior(pio_mode,pio_clobber) ! Set to CLOBBER for now, TODO: fix to allow for optional mode type like in CAM - piotype = scream_iotype_to_pio_iotype(iotype) - retval = pio_createfile(pio_subsystem,File,piotype,fname,mode) - call errorHandle("PIO ERROR: unable to create file: "//trim(fname),retval) - - end subroutine eam_pio_createfile -!=====================================================================! - ! Open an already existing netCDF file. - subroutine eam_pio_openfile(pio_file,fname,iotype) - use pio, only: pio_openfile - use pio_types, only: pio_write, pio_nowrite - - type(pio_atm_file_t), pointer, intent(in) :: pio_file ! Pointer to pio file struct associated with this filename - character(len=*), intent(in) :: fname ! Pio file name - !-- - integer :: retval ! PIO error return value - integer :: mode ! Mode for how to handle the new file - integer, intent(in) :: iotype - - integer :: piotype - - if (is_read(pio_file%purpose)) then - mode = pio_nowrite - else - mode = pio_write - endif - piotype = scream_iotype_to_pio_iotype(iotype) - retval = pio_openfile(pio_subsystem,pio_file%pioFileDesc,piotype,fname,mode) - call errorHandle("PIO ERROR: unable to open file: "//trim(fname),retval) - - if (is_append(pio_file%purpose)) then - pio_file%is_enddef = .true. - endif - - end subroutine eam_pio_openfile -!=====================================================================! - ! Close a netCDF file. To be done as a last step after all input or output - ! for that file has been finished. - subroutine eam_pio_closefile(fname) - use pio, only: PIO_syncfile, PIO_closefile - - character(len=*), intent(in) :: fname ! Pio file name - !-- - type(pio_atm_file_t),pointer :: pio_atm_file - type(pio_file_list_t), pointer :: pio_file_list_ptr - logical :: found - type(hist_var_list_t), pointer :: curr_var_list - type(hist_var_t), pointer :: var - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(fname),pio_atm_file,found,pio_file_list_ptr) - - if (found) then - if (pio_atm_file%num_customers .eq. 1) then - if ( is_write(pio_atm_file%purpose) ) then - call PIO_syncfile(pio_atm_file%pioFileDesc) - endif - call PIO_closefile(pio_atm_file%pioFileDesc) - pio_atm_file%num_customers = pio_atm_file%num_customers - 1 - - ! Remove all variables from this file as customers for the stored pio - ! decompostions - curr_var_list => pio_atm_file%var_list_top ! Start with the first variable in the file - do while (associated(curr_var_list)) - var => curr_var_list%var ! The actual variable pointer - if (associated(var)) then - if (associated(var%iodesc_list)) then - ! Remove this variable as a customer of the associated iodesc - var%iodesc_list%num_customers = var%iodesc_list%num_customers - 1 - ! Dellocate select memory from this variable. Note we can't just - ! deallocate the whole var structure because this would also - ! deallocate the iodesc_list. - call deallocate_hist_var_t(var) - end if ! associated(var%iodesc_list) - end if ! associated(var) - curr_var_list => curr_var_list%next ! Move on to the next variable - end do ! associated(curr_var_list) - - ! Adjust pointers in the pio file list - if (associated(pio_file_list_ptr%prev)) then - pio_file_list_ptr%prev%next => pio_file_list_ptr%next - else - ! We're deleting the first item in the lists. Update pio_file_list_front - pio_file_list_front => pio_file_list_ptr%next - endif - if (associated(pio_file_list_ptr%next)) then - pio_file_list_ptr%next%prev => pio_file_list_ptr%prev - else - ! We're deleting the last item in the lists. Update pio_file_list_back - pio_file_list_back => pio_file_list_ptr%prev - endif - - ! Now that we have closed this pio file and purged it from the list we - ! can deallocate the structure. - deallocate(pio_atm_file) - else if (pio_atm_file%num_customers .gt. 1) then - pio_atm_file%num_customers = pio_atm_file%num_customers - 1 - else - call errorHandle("PIO ERROR: while closing file: "//trim(fname)//", found num_customers<=0",-999) - endif - else - call errorHandle("PIO ERROR: unable to close file: "//trim(fname)//", was not found",-999) - end if - - ! Final step, free any pio decompostion memory that is no longer needed. - ! Update: We are trying to reuse decompostions maximally, so we're - ! skipping this step. - !call free_decomp() - - end subroutine eam_pio_closefile -!=====================================================================! - ! Flushes IO buffers to file - subroutine eam_pio_flush_file(fname) - use pio, only: PIO_syncfile - - character(len=*), intent(in) :: fname ! Pio file name - !-- - type(pio_atm_file_t),pointer :: pio_atm_file - logical :: found - - ! Find the pointer for this file - call lookup_pio_atm_file(trim(fname),pio_atm_file,found) - - if (found) then - if ( is_write(pio_atm_file%purpose) .or. is_append(pio_atm_file%purpose) ) then - call PIO_syncfile(pio_atm_file%pioFileDesc) - else - call errorHandle("PIO ERROR: unable to flush file: "//trim(fname)//", is not open in write mode",-999) - endif - else - call errorHandle("PIO ERROR: unable to flush file: "//trim(fname)//", was not found",-999) - end if - end subroutine eam_pio_flush_file -!=====================================================================! - ! Helper function to debug list of decomps - subroutine print_decomp() - type(iodesc_list_t), pointer :: iodesc_ptr - - integer :: total - integer :: cnt - logical :: assoc - - if (associated(iodesc_list_top)) then - total = 0 - cnt = 0 - write(*,*) " PRINT DECOMP " - write(*,*) " ------------ " - write(*,'(8X,A10,A15,A50,A15)') "Location", "Associated?", "IODESC TAG", "# Customers" - else - write(*,*) "No DECOMP List to print" - return - end if - iodesc_ptr => iodesc_list_top - do while(associated(iodesc_ptr)) - total = total + 1 - assoc = .false. - if (associated(iodesc_ptr%iodesc).and.iodesc_ptr%iodesc_set) then - cnt = cnt + 1 - assoc = .true. - end if - write(*,'(I3,A5,I10,L15,A50,I15)') total, ": ", iodesc_ptr%location, assoc , trim(iodesc_ptr%tag), iodesc_ptr%num_customers - iodesc_ptr => iodesc_ptr%next - end do - write(*,*) " total: ", total, ", cnt: ",cnt - write(*,*) " ------------ " - - end subroutine print_decomp -!=====================================================================! - ! Free the memory stored in a hist_var_t derived type - subroutine deallocate_hist_var_t(var) - - type(hist_var_t), pointer :: var - - deallocate(var%compdof) - deallocate(var%dimid) - deallocate(var%dimlen) - var%is_set = .false. - - end subroutine deallocate_hist_var_t - !=====================================================================! - ! Free pio decomposition memory in PIO for any decompositions that are no - ! longer needed. Previously, we thought that this is an important memory - ! management step that should be taken whenever a file is closed. Now we're - ! trying to keep decomps persistent so they can be reused. Thus, calling - ! this routine is optional. - subroutine free_decomp() - use pio, only: PIO_freedecomp - type(iodesc_list_t), pointer :: iodesc_ptr, next - - ! Free all decompositions from PIO - iodesc_ptr => iodesc_list_top - do while(associated(iodesc_ptr)) - next => iodesc_ptr%next - if (associated(iodesc_ptr%iodesc).and.iodesc_ptr%iodesc_set) then - if (iodesc_ptr%num_customers .eq. 0) then - ! Free decomp - call pio_freedecomp(pio_subsystem,iodesc_ptr%iodesc) - ! Nullify this decomp - ! If we are at iodesc_list_top we need to make iodesc_ptr%next the new - ! iodesc_list_top: - if (associated(iodesc_ptr%prev)) then - iodesc_ptr%prev%next => iodesc_ptr%next - else - ! We are deleting the first item in the list, update - ! iodesc_list_front - iodesc_list_top => iodesc_ptr%next - end if - if (associated(iodesc_ptr%next)) then - iodesc_ptr%next%prev => iodesc_ptr%prev - end if - deallocate(iodesc_ptr) - end if - end if - iodesc_ptr => next - end do - - end subroutine free_decomp -!=====================================================================! - ! Finalize a PIO session within scream. Close all open files and deallocate - ! the pio_subsystem session. - subroutine eam_pio_finalize() - use pio, only: PIO_finalize, pio_freedecomp - ! May not be needed, possibly handled by PIO directly. - -#if !defined(SCREAM_CIME_BUILD) - integer :: ierr -#endif - type(pio_file_list_t), pointer :: curr_file_ptr, prev_file_ptr - type(iodesc_list_t), pointer :: iodesc_ptr - - ! Close all the PIO Files - curr_file_ptr => pio_file_list_front - do while (associated(curr_file_ptr)) - call eam_pio_closefile(curr_file_ptr%pio_file%filename) - prev_file_ptr => curr_file_ptr - curr_file_ptr => curr_file_ptr%next - deallocate(prev_file_ptr) - pio_file_list_front => curr_file_ptr ! be sure not to iterate over deallocated item - end do - ! Free all decompositions from PIO - iodesc_ptr => iodesc_list_top - do while(associated(iodesc_ptr)) - if (associated(iodesc_ptr%iodesc).and.iodesc_ptr%iodesc_set) then - call pio_freedecomp(pio_subsystem,iodesc_ptr%iodesc) - end if - iodesc_ptr => iodesc_ptr%next - end do - -#if !defined(SCREAM_CIME_BUILD) - call PIO_finalize(pio_subsystem, ierr) - call errorHandle("PIO ERROR: something went wrong when calling PIO_finalize.",ierr) - nullify(pio_subsystem) -#endif - - end subroutine eam_pio_finalize -!=====================================================================! - ! Handle any errors that occur in this module and print to screen an error - ! message. - subroutine errorHandle(errMsg, retVal) - use iso_c_binding - implicit none - - interface - subroutine finalize_scream_session() bind(C) - ! No inputs or anything, just interface to C code. - end subroutine finalize_scream_session - end interface - - character(len=*), intent(in) :: errMsg - integer, intent(in) :: retVal - - integer :: ierr - - if (retVal .ne. PIO_NOERR) then - write(*,'(I8,2x,A512)') retVal,trim(errMsg) - ! Kill run - call eam_pio_finalize() - call finalize_scream_session() - call mpi_abort(atm_mpicom,retVal,ierr) - end if - - end subroutine errorHandle -!=====================================================================! - ! Determine the unique pio_decomposition for this output grid, if it hasn't - ! been defined create a new one. - subroutine get_decomp(tag,dtype,dimension_len,compdof,iodesc_list) - use pio, only: pio_initdecomp - ! TODO: CAM code creates the decomp tag for the user. Theoretically it is - ! unique because it is based on dimensions and datatype. But the tag ends - ! up not being very descriptive. The todo item is to revisit how tags are - ! handled and decide if we want the code to create a tag or let the use - ! assign a tag. - character(len=*) :: tag ! Unique tag string describing this output grid - integer, intent(in) :: dtype ! Datatype associated with the output - integer, intent(in) :: dimension_len(:) ! Array of the dimension lengths for this decomp - integer(kind=pio_offset_kind), intent(in) :: compdof(:) ! The degrees of freedom this rank is responsible for - type(iodesc_list_t), pointer :: iodesc_list ! The pio decomposition list that holds this iodesc - - logical :: found ! Whether a decomp has been found among the previously defined decompositions - type(iodesc_list_t),pointer :: curr, prev ! Used to toggle through the recursive list of decompositions - integer :: loc_len ! Used to keep track of how many dimensions there are in decomp - - ! Assign a PIO decomposition to variable, if none exists, create a new one: - found = .false. - curr => iodesc_list_top - prev => iodesc_list_top - ! Cycle through all current iodesc to see if the decomp has already been - ! created - do while(associated(curr) .and. (.not.found)) - if (trim(tag) == trim(curr%tag)) then - found = .true. - else - prev => curr - curr => curr%next - end if - end do - ! If we didn't find an iodesc then we need to create one - if (.not.found) then - curr => prev ! Go back and allocate the new iodesc in curr%next - ! We may have no iodesc to begin with, so we need to associate the - ! beginning of the list. - if(.not.associated(curr)) then - allocate(curr) - curr%location = 1 - iodesc_list_top => curr - end if - if(associated(curr%iodesc)) then - allocate(curr%next) - curr%next%prev => curr - curr => curr%next - nullify(curr%iodesc) ! Extra step to ensure clean iodesc - nullify(curr%next) ! Extra step to ensure clean iodesc - end if - allocate(curr%iodesc) - curr%tag = trim(tag) - if (associated(curr%prev)) then - curr%location = prev%location+1 - end if - loc_len = size(dimension_len) - if ( loc_len.eq.1 .and. dimension_len(loc_len).eq.0 ) then - allocate(curr%iodesc) - else - call pio_initdecomp(pio_subsystem, dtype, dimension_len, compdof, curr%iodesc, rearr=pio_rearranger) - curr%iodesc_set = .true. - end if - curr%num_customers = 0 - end if - iodesc_list => curr - - end subroutine get_decomp -!=====================================================================! - ! Set the degrees of freedom (dof) this MPI rank is responsible for - ! reading/writing from/to file. - ! Briefly, PIO distributes the work of reading/writing the total array - ! of any variable data over all of the MPI ranks assigned to PIO. - ! DOF's are assigned considering a 1D flattening of any array, ignoring - ! unlimited dimensions, which are handled elsewhere. - ! Once it is known which dof a rank is responsible for this routine is used to - ! set those locally for use with all read and write statements. - ! Arguments: - ! filename: name of PIO input/output file. - ! varname: shortname of variable to set the dof for. - ! dof_len: number of dof associated for this rank and this variable - ! dof_vec: a vector of length dof_len which includes the global indices of - ! the flattened "varname" array that this MPI rank is responsible - ! for. - ! --- Example: If variable F has dimensions (x,y,t) of size (2,5,t), which means a --- - ! total of 2x5=10 dof, not counting the time dimension. - ! If 3 MPI ranks are used then a typical breakdown for dof might be: - ! Rank 1: (1,2,3) - ! Rank 2: (4,5,6) - ! Rank 3: (7,8,9,10) - subroutine set_dof(filename,varname,dof_len,dof_vec) - character(len=*), intent(in) :: filename - character(len=*), intent(in) :: varname - integer, intent(in) :: dof_len - integer(kind=pio_offset_kind), intent(in) :: dof_vec(dof_len) - - type(pio_atm_file_t),pointer :: pio_atm_file - type(hist_var_t), pointer :: var - logical :: found - integer :: ii - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - if (allocated(var%compdof)) deallocate(var%compdof) - allocate( var%compdof(dof_len) ) - do ii = 1,dof_len - var%compdof(ii) = dof_vec(ii) - end do - - end subroutine set_dof -!=====================================================================! - ! Get and assign all pio decompositions for a specific PIO file. This is a - ! mandatory step to be taken after all dimensions and variables have been - ! registered with an input or output file. - subroutine set_decomp(filename) - - character(len=*) :: filename ! Name of the pio file to set decomp for - - type(pio_atm_file_t), pointer :: current_atm_file - type(hist_var_list_t), pointer :: curr ! Used to cycle through recursive list of variables - type(hist_var_t), pointer :: hist_var ! Pointer to the variable structure that has been found - integer :: loc_len - logical :: found - - call lookup_pio_atm_file(filename,current_atm_file,found) - if (.not. found) then - call errorHandle("PIO ERROR: pio file '"//trim(filename)//"' not found.",999) - endif - - curr => current_atm_file%var_list_top - do while (associated(curr)) - ! Skip already deallocated vars, and vars for which decomp was already set - ! NOTE: fortran does not mandate/prohibit logical op short circuit, so do the - ! two following if statements separately - if (associated(curr%var)) then - if (.not. associated(curr%var%iodesc)) then - hist_var => curr%var - if (.not.associated(hist_var)) call errorHandle("PIO ERROR: unable to set decomp for file, var: "//trim(current_atm_file%filename)//", "//trim(hist_var%name)//". Set DOF.",999) - ! Assign decomp - if (hist_var%has_t_dim) then - loc_len = max(1,hist_var%numdims-1) - call get_decomp(hist_var%pio_decomp_tag,hist_var%dtype,hist_var%dimlen(:loc_len),hist_var%compdof,hist_var%iodesc_list) - else - call get_decomp(hist_var%pio_decomp_tag,hist_var%dtype,hist_var%dimlen,hist_var%compdof,hist_var%iodesc_list) - end if - hist_var%iodesc => hist_var%iodesc_list%iodesc - hist_var%iodesc_list%num_customers = hist_var%iodesc_list%num_customers + 1 ! Add this variable as a customer of this pio decomposition - endif - end if - curr => curr%next - end do - - end subroutine set_decomp -!=====================================================================! - ! Query the hist_var_t pointer for a specific variable on a specific file. - subroutine get_var(pio_file,varname,var) - - type(pio_atm_file_t), pointer :: pio_file ! Pio output file structure - character(len=*) :: varname ! Name of the variable to query - type(hist_var_t), pointer :: var ! Pointer to the variable structure that has been found - - type(hist_var_list_t), pointer :: curr ! Used to cycle through recursive list of variables - - curr => pio_file%var_list_top - do while (associated(curr)) - var => curr%var - if (associated(var)) then - if (trim(varname) == trim(var%name) .and. var%is_set) return - end if - curr => curr%next - end do - - ! If we got this far we didn't find the variable - call errorHandle("PIO ERROR: unable to find variable: "//trim(varname)//" in file: "//trim(pio_file%filename),999) - - end subroutine get_var -!=====================================================================! - ! Lookup pointer for pio file based on filename. - subroutine lookup_pio_atm_file(filename,pio_file,found,pio_file_list_ptr_in) - - character(len=*),intent(in) :: filename ! Name of file to be found - type(pio_atm_file_t), pointer :: pio_file ! Pointer to pio_atm_output structure associated with this filename - logical, intent(out) :: found ! whether or not the file was found - type(pio_file_list_t), pointer, optional :: pio_file_list_ptr_in - - type(pio_file_list_t), pointer :: pio_file_list_ptr - - ! Scan pio file list, search for this filename - found = .false. - pio_file_list_ptr => pio_file_list_front - pio_file => null() - do while (associated(pio_file_list_ptr)) - if (trim(filename)==trim(pio_file_list_ptr%pio_file%filename)) then - pio_file => pio_file_list_ptr%pio_file - found = .true. - if (present(pio_file_list_ptr_in)) then - pio_file_list_ptr_in => pio_file_list_ptr - endif - return - end if - pio_file_list_ptr => pio_file_list_ptr%next - end do - - end subroutine lookup_pio_atm_file -!=====================================================================! - ! Create a new pio file pointer based on filename. - subroutine get_pio_atm_file(filename,pio_file,purpose,iotype) - use pio, only: PIO_inq_dimid, PIO_inq_dimlen - - character(len=*),intent(in) :: filename ! Name of file to be found - type(pio_atm_file_t), pointer :: pio_file ! Pointer to pio_atm_output structure associated with this filename - integer,intent(in) :: purpose ! Purpose for this file lookup, 0 = find already existing, 1 = create new as output, 2 = open new as input - integer,intent(in) :: iotype - - logical :: found - type(pio_file_list_t), pointer :: new_list_item - - integer :: ierr, time_id - - ! Sanity checks - if ( .not. (is_read(purpose) .or. is_write(purpose) .or. is_append(purpose)) ) then - call errorHandle("PIO Error: unrecognized open mode requested for file '"//filename//"'.",-999) - endif - if ( is_read(purpose) .and. is_write(purpose) ) then - call errorHandle("PIO Error: both READ and WRITE mode requested for file '"//filename//"'.",-999) - endif - if ( is_read(purpose) .and. is_append(purpose) ) then - call errorHandle("PIO Error: APPEND mode requested along with READ mode for file '"//filename//"'.",-999) - endif - - ! If the file already exists, return that file - call lookup_pio_atm_file(trim(filename),pio_file,found) - if (found) then - if (is_write(purpose) .or. is_write(pio_file%purpose) ) then - ! We only allow multiple customers of the file if they all use it in read mode. - call errorHandle("PIO Error: file '"//trim(filename)//"' was already open for writing.",-999) - endif - pio_file%num_customers = pio_file%num_customers + 1 - else - allocate(new_list_item) - allocate(new_list_item%pio_file) - pio_file => new_list_item%pio_file - - ! Create and initialize the new pio file: - pio_file%filename = trim(filename) - pio_file%numRecs = 0 - pio_file%num_customers = 1 - pio_file%purpose = purpose - if (is_read(purpose) .or. is_append(purpose)) then - ! Either read or append to existing file. Either way, file must exist on disk - call eam_pio_openfile(pio_file,trim(pio_file%filename),iotype) - ! Update the numRecs to match the number of recs in this file. - ierr = pio_inq_dimid(pio_file%pioFileDesc,"time",time_id) - if (ierr.ne.0) then - ! time is not present in the file, set the number of Recs to 1 - pio_file%numRecs = 1 - else - ! time is present, set the number of Recs accordingly - ierr = pio_inq_dimlen(pio_file%pioFileDesc,time_id,pio_file%numRecs) - call errorHandle("EAM_PIO ERROR: Unable to determine length for dimension time in file "//trim(pio_file%filename),ierr) - end if - elseif (is_write(purpose)) then - ! New output file - call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename),iotype) - call eam_pio_createHeader(pio_file%pioFileDesc) - else - call errorHandle("PIO Error: get_pio_atm_file with filename = "//trim(filename)//", purpose (int) assigned to this lookup is not valid" ,-999) - end if - - ! Update the pio file list - if (associated(pio_file_list_back)) then - ! 1) Link new file to the new_list_itement back of the list - new_list_item%prev => pio_file_list_back - ! 2) Link the current last element of the list to the new one - pio_file_list_back%next => new_list_item - ! 3) and update the pointer to the last - pio_file_list_back => new_list_item - else - ! The list was empty. Set both front/back to point to the new item - pio_file_list_front => new_list_item - pio_file_list_back => new_list_item - endif - endif - end subroutine get_pio_atm_file -!=====================================================================! - ! Retrieve the time value for a specific time_index - ! If the input arg time_index is not provided, then it is assumed the user wants - ! the last time entry. If time_index is present, it MUST be valid - function read_time_at_index(filename,time_index) result(val) - use pio, only: PIO_get_var, PIO_inq_varid, PIO_inq_dimid, PIO_inq_dimlen - - character(len=*), intent(in) :: filename - integer, intent(in), optional :: time_index - real(c_double) :: val - real(c_double) :: val_buf(1) - - type(pio_atm_file_t), pointer :: pio_atm_file - logical :: found - integer :: dim_id, time_len, ierr - type(var_desc_t) :: varid ! netCDF variable ID - integer :: strt(1), cnt(1), timeidx - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - if (.not.found) call errorHandle("read_time_at_index ERROR: File "//trim(filename)//" not found",-999) - ierr = PIO_inq_varid(pio_atm_file%pioFileDesc,"time",varid) - call errorHandle('read_time_at_index: Error finding variable ID for "time" in file '//trim(filename)//'.',ierr); - - ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim("time"),dim_id) - call errorHandle("read_time_at_index ERROR: dimension 'time' not found in file "//trim(filename)//".",ierr) - ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,dim_id,time_len) - call errorHandle("PIO Error! Something went wrong when inquiring time dim length on file file "//trim(filename)//".",ierr) - - if (present(time_index)) then - timeidx = time_index - else - timeidx = time_len - endif - if (timeidx .gt. time_len) then - call errorHandle("read_time_at_index ERROR: time_index arg larger than length of time dimension",-999) - elseif (timeidx .le. 0) then - call errorHandle("read_time_at_index ERROR: time_index arg must be positive",-999) - end if - - strt(1) = timeidx - cnt(1) = 1 - ierr = PIO_get_var(pio_atm_file%pioFileDesc,varid,strt,cnt,val_buf) - call errorHandle('read_time_at_index: Error reading variable "time" in file '//trim(filename)//'.',ierr); - val = val_buf(1) - end function read_time_at_index -!=====================================================================! - ! Write output to file based on type (int or real) - ! --Note-- that any dimensionality could be written if it is flattened to 1D - ! before calling a write routine. - ! Mandatory inputs are: - ! filename: the netCDF filename to be written to. - ! varname: The shortname for the variable being written. - ! var_data_ptr: An array of data that will be used to write the output. NOTE: - ! If PIO MPI ranks > 1, var_data_ptr should be the subset of the global array - ! which includes only those degrees of freedom that have been - ! assigned to this rank. See set_dof above. - !--------------------------------------------------------------------------- - ! - ! grid_write_darray_1d: Write a variable defined on this grid - ! - !--------------------------------------------------------------------------- - subroutine grid_write_darray_float(filename, varname, buf, buf_size) - use pio, only: PIO_put_var, PIO_setframe, PIO_write_darray - use pio_types, only: PIO_max_var_dims - - ! Dummy arguments - character(len=*), intent(in) :: filename ! PIO filename - character(len=*), intent(in) :: varname - integer(kind=c_int), intent(in) :: buf_size - real(kind=c_float), intent(in) :: buf(buf_size) - - ! Local variables - - type(pio_atm_file_t), pointer :: pio_atm_file - type(hist_var_t), pointer :: var - integer :: ierr,jdim - integer :: start(pio_max_var_dims), count(pio_max_var_dims) - logical :: found - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - - if (var%has_t_dim) then - ! Set the time index we are writing - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(max(1,pio_atm_file%numRecs),kind=pio_offset_kind)) - endif - - if (var%is_partitioned) then - call pio_write_darray(pio_atm_file%pioFileDesc, var%piovar, var%iodesc, buf, ierr) - else - if (var%has_t_dim) then - do jdim=1,var%numdims - if (var%dimid(jdim) .eq. time_dimid) then - start (jdim) = int(max(1,pio_atm_file%numRecs)) - count (jdim) = 1 - else - start (jdim) = 1 - count (jdim) = var%dimlen(jdim) - endif - enddo - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,start(:var%numdims),count(:var%numdims),buf) - else - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,buf) - endif - endif - - call errorHandle( 'eam_grid_write_darray_float: Error writing variable '//trim(varname),ierr) - end subroutine grid_write_darray_float - subroutine grid_write_darray_double(filename, varname, buf, buf_size) - use pio, only: PIO_put_var, PIO_setframe, PIO_write_darray - use pio_types, only: PIO_max_var_dims - - ! Dummy arguments - character(len=*), intent(in) :: filename ! PIO filename - character(len=*), intent(in) :: varname - integer(kind=c_int), intent(in) :: buf_size - real(kind=c_double), intent(in) :: buf(buf_size) - - ! Local variables - - type(pio_atm_file_t), pointer :: pio_atm_file - type(hist_var_t), pointer :: var - integer :: ierr,jdim - integer :: start(pio_max_var_dims), count(pio_max_var_dims) - logical :: found - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - - if (var%has_t_dim) then - ! Set the time index we are writing - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(max(1,pio_atm_file%numRecs),kind=pio_offset_kind)) - endif - - if (var%is_partitioned) then - call pio_write_darray(pio_atm_file%pioFileDesc, var%piovar, var%iodesc, buf, ierr) - else - if (var%has_t_dim) then - do jdim=1,var%numdims - if (var%dimid(jdim) .eq. time_dimid) then - start (jdim) = int(max(1,pio_atm_file%numRecs)) - count (jdim) = 1 - else - start (jdim) = 1 - count (jdim) = var%dimlen(jdim) - endif - enddo - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,start(:var%numdims),count(:var%numdims),buf) - else - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,buf) - endif - endif - - call errorHandle( 'eam_grid_write_darray_double: Error writing variable '//trim(varname),ierr) - end subroutine grid_write_darray_double - subroutine grid_write_darray_int(filename, varname, buf, buf_size) - use pio, only: PIO_put_var, PIO_setframe, PIO_write_darray - use pio_types, only: PIO_max_var_dims - - ! Dummy arguments - character(len=*), intent(in) :: filename ! PIO filename - character(len=*), intent(in) :: varname - integer(kind=c_int), intent(in) :: buf_size - integer(kind=c_int), intent(in) :: buf(buf_size) - - ! Local variables - - type(pio_atm_file_t), pointer :: pio_atm_file - type(hist_var_t), pointer :: var - integer :: ierr,jdim - integer :: start(pio_max_var_dims), count(pio_max_var_dims) - logical :: found - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - - if (var%has_t_dim) then - ! Set the time index we are writing - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(max(1,pio_atm_file%numRecs),kind=pio_offset_kind)) - endif - - if (var%is_partitioned) then - call pio_write_darray(pio_atm_file%pioFileDesc, var%piovar, var%iodesc, buf, ierr) - else - if (var%has_t_dim) then - do jdim=1,var%numdims - if (var%dimid(jdim) .eq. time_dimid) then - start (jdim) = int(max(1,pio_atm_file%numRecs)) - count (jdim) = 1 - else - start (jdim) = 1 - count (jdim) = var%dimlen(jdim) - endif - enddo - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,start(:var%numdims),count(:var%numdims),buf) - else - ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,buf) - endif - endif - - call errorHandle( 'eam_grid_write_darray_int: Error writing variable '//trim(varname),ierr) - end subroutine grid_write_darray_int -!=====================================================================! - ! Read output from file based on type (int or real) - ! --Note-- that any dimensionality could be read if it is flattened to 1D - ! before calling a write routine. - ! Mandatory inputs are: - ! filename: the netCDF filename to be read from. - ! varname: The shortname for the variable being read.i - ! var_data_ptr: An c_ptr of data that will be used to read the input. NOTE: - ! If PIO MPI ranks > 1, var_data_ptr should be the subset of the global array - ! which includes only those degrees of freedom that have been - ! assigned to this rank. See set_dof above. - ! time_index: The 1-based time index for this variable (0 is ignored). - !--------------------------------------------------------------------------- - ! - ! grid_read_darray_1d: Read a variable defined on this grid - ! - !--------------------------------------------------------------------------- - subroutine grid_read_darray_double(filename, varname, buf, buf_size, time_index) - use pio, only: PIO_setframe, PIO_read_darray - - ! Dummy arguments - character(len=*), intent(in) :: filename ! PIO filename - character(len=*), intent(in) :: varname - integer (kind=c_int), intent(in) :: buf_size - real(kind=c_double), intent(out) :: buf(buf_size) - integer, intent(in) :: time_index - - ! Local variables - type(pio_atm_file_t),pointer :: pio_atm_file - type(hist_var_t), pointer :: var - integer :: ierr, var_size - logical :: found - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - ! Set the timesnap we are reading - if (time_index .gt. 0) then - ! The user has set a valid time index to read from - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(time_index,kind=pio_offset_kind)) - else - ! Otherwise default to the last time_index in the file - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(pio_atm_file%numRecs,kind=pio_offset_kind)) - end if - - ! We don't want the extent along the 'time' dimension - var_size = SIZE(var%compdof) - - ! Now we know the exact size of the array, and can shape the f90 pointer - call pio_read_darray(pio_atm_file%pioFileDesc, var%piovar, var%iodesc, buf, ierr) - call errorHandle( 'eam_grid_read_darray_double: Error reading variable '//trim(varname),ierr) - end subroutine grid_read_darray_double - subroutine grid_read_darray_float(filename, varname, buf, buf_size, time_index) - use pio, only: PIO_setframe, PIO_read_darray - - ! Dummy arguments - character(len=*), intent(in) :: filename ! PIO filename - character(len=*), intent(in) :: varname - integer (kind=c_int), intent(in) :: buf_size - real(kind=c_float), intent(out) :: buf(buf_size) - integer, intent(in) :: time_index - - ! Local variables - type(pio_atm_file_t),pointer :: pio_atm_file - type(hist_var_t), pointer :: var - integer :: ierr, var_size - logical :: found - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - - ! Set the timesnap we are reading - if (time_index .gt. 0) then - ! The user has set a valid time index to read from - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(time_index,kind=pio_offset_kind)) - else - ! Otherwise default to the last time_index in the file - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(pio_atm_file%numRecs,kind=pio_offset_kind)) - end if - - ! We don't want the extent along the 'time' dimension - var_size = SIZE(var%compdof) - - ! Now we know the exact size of the array, and can shape the f90 pointer - call pio_read_darray(pio_atm_file%pioFileDesc, var%piovar, var%iodesc, buf, ierr) - call errorHandle( 'eam_grid_read_darray_float: Error reading variable '//trim(varname),ierr) - end subroutine grid_read_darray_float - subroutine grid_read_darray_int(filename, varname, buf, buf_size, time_index) - use pio, only: PIO_setframe, PIO_read_darray - - ! Dummy arguments - character(len=*), intent(in) :: filename ! PIO filename - character(len=*), intent(in) :: varname - integer (kind=c_int), intent(in) :: buf_size - integer (kind=c_int), intent(out) :: buf(buf_size) - integer, intent(in) :: time_index - - ! Local variables - type(pio_atm_file_t),pointer :: pio_atm_file - type(hist_var_t), pointer :: var - integer :: ierr, var_size - logical :: found - - call lookup_pio_atm_file(trim(filename),pio_atm_file,found) - call get_var(pio_atm_file,varname,var) - - ! Set the timesnap we are reading - if (time_index .gt. 0) then - ! The user has set a valid time index to read from - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(time_index,kind=pio_offset_kind)) - else - ! Otherwise default to the last time_index in the file - call PIO_setframe(pio_atm_file%pioFileDesc,var%piovar,int(pio_atm_file%numRecs,kind=pio_offset_kind)) - end if - - ! We don't want the extent along the 'time' dimension - var_size = SIZE(var%compdof) - - ! Now we know the exact size of the array, and can shape the f90 pointer - call pio_read_darray(pio_atm_file%pioFileDesc, var%piovar, var%iodesc, buf, ierr) - call errorHandle( 'eam_grid_read_darray_int: Error reading variable '//trim(varname),ierr) - end subroutine grid_read_darray_int -!=====================================================================! - subroutine convert_int_2_str(int_in,str_out) - integer, intent(in) :: int_in - character(len=*), intent(out) :: str_out - - character(len=8) :: fmt_str - - if (int_in < 0) then - fmt_str = "(A1," - else - fmt_str = "(" - end if - - if (abs(int_in)<10) then - fmt_str = trim(fmt_str)//"I1)" - elseif (abs(int_in)<1e2) then - fmt_str = trim(fmt_str)//"I2)" - elseif (abs(int_in)<1e3) then - fmt_str = trim(fmt_str)//"I3)" - elseif (abs(int_in)<1e4) then - fmt_str = trim(fmt_str)//"I4)" - elseif (abs(int_in)<1e5) then - fmt_str = trim(fmt_str)//"I5)" - elseif (abs(int_in)<1e6) then - fmt_str = trim(fmt_str)//"I6)" - elseif (abs(int_in)<1e7) then - fmt_str = trim(fmt_str)//"I7)" - elseif (abs(int_in)<1e8) then - fmt_str = trim(fmt_str)//"I8)" - elseif (abs(int_in)<1e9) then - fmt_str = trim(fmt_str)//"I9)" - endif - - if (int_in < 0) then - write(str_out,fmt_str) "n", int_in - else - write(str_out,fmt_str) int_in - end if - - end subroutine convert_int_2_str - - subroutine get_coord (filename,shortname,hist_coord) - character(len=256), intent(in) :: filename - character(len=256), intent(in) :: shortname - type(hist_coord_t), pointer, intent(out) :: hist_coord - - type(pio_atm_file_t), pointer :: pio_atm_file - type(hist_coord_list_t), pointer :: curr, prev - logical :: file_found, dim_found - - hist_coord => NULL() - - call lookup_pio_atm_file(trim(filename),pio_atm_file,file_found) - if (.not. file_found ) then - call errorHandle("PIO ERROR: error retrieving dimension "//trim(shortname)//" in file "//trim(filename)//".\n PIO file not found or not open.",-999) - endif - - curr => pio_atm_file%coord_list_top - do while (associated(curr)) - if (associated(curr%coord)) then - if(trim(curr%coord%name)==trim(shortname)) then - dim_found = .true. - exit - endif - end if - prev => curr - curr => prev%next - end do - - if (.not. dim_found ) then - call errorHandle("PIO ERROR: error retrieving dimension "//trim(shortname)//" from file "//trim(filename)//". Dimension not found.",-999) - endif - - hist_coord => curr%coord - end subroutine get_coord - - function is_read (purpose) - integer, intent(in) :: purpose - logical :: is_read - is_read = iand(purpose,file_purpose_in) .ne. 0 - end function is_read - function is_write (purpose) - integer, intent(in) :: purpose - logical :: is_write - is_write = iand(purpose,file_purpose_out) .ne. 0 - end function is_write - function is_append (purpose) - integer, intent(in) :: purpose - logical :: is_append - is_append = iand(purpose,file_purpose_app) .ne. 0 - end function is_append -!=====================================================================! -end module scream_scorpio_interface diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index ead1aa0fbd65..2bd1eb999a6e 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -1,54 +1,125 @@ #include "scream_scorpio_interface.hpp" -#include "ekat/ekat_scalar_traits.hpp" +#include "scream_shr_interface_c2f.hpp" + #include "scream_config.h" -#include "ekat/ekat_assert.hpp" -#include "share/scream_types.hpp" +#include +#include #include -#include - - -using scream::Real; -using scream::Int; -extern "C" { - -// Fortran routines to be called from C++ - void register_file_c2f(const char*&& filename, const int& mode, const int& iotype); - int get_file_mode_c2f(const char*&& filename); - void set_decomp_c2f(const char*&& filename); - void set_dof_c2f(const char*&& filename,const char*&& varname,const Int dof_len,const std::int64_t *x_dof); - void grid_read_data_array_c2f_int(const char*&& filename, const char*&& varname, const Int time_index, int *buf, const int buf_size); - void grid_read_data_array_c2f_float(const char*&& filename, const char*&& varname, const Int time_index, float *buf, const int buf_size); - void grid_read_data_array_c2f_double(const char*&& filename, const char*&& varname, const Int time_index, double *buf, const int buf_size); - - void grid_write_data_array_c2f_int(const char*&& filename, const char*&& varname, const int* buf, const int buf_size); - void grid_write_data_array_c2f_float(const char*&& filename, const char*&& varname, const float* buf, const int buf_size); - void grid_write_data_array_c2f_double(const char*&& filename, const char*&& varname, const double* buf, const int buf_size); - void eam_init_pio_subsystem_c2f(const int mpicom, const int atm_id); - void eam_pio_finalize_c2f(); - void eam_pio_closefile_c2f(const char*&& filename); - void eam_pio_flush_file_c2f(const char*&& filename); - void pio_update_time_c2f(const char*&& filename,const double time); - void register_dimension_c2f(const char*&& filename, const char*&& shortname, const char*&& longname, const int global_length, const bool partitioned); - void register_variable_c2f(const char*&& filename, const char*&& shortname, const char*&& longname, - const char*&& units, const int numdims, const char** var_dimensions, - const int dtype, const int nc_dtype, const char*&& pio_decomp_tag); - void set_variable_metadata_char_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const char*&& meta_val); - void set_variable_metadata_float_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const float meta_val); - void set_variable_metadata_double_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, const double meta_val); - float get_variable_metadata_float_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); - double get_variable_metadata_double_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); - void get_variable_metadata_char_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name, char*&& meta_val); - void eam_pio_redef_c2f(const char*&& filename); - void eam_pio_enddef_c2f(const char*&& filename); - bool is_enddef_c2f(const char*&& filename); -} // extern C +#include namespace scream { namespace scorpio { +// This class is an implementation detail, and therefore it is hidden inside +// a cpp file. All customers of IO capabilities must use the common interfaces +// exposed in the header file of this source file. +// This class simply serves as a container for persistent IO data. +struct ScorpioSession +{ +public: + static ScorpioSession& instance () { + static ScorpioSession s; + return s; + } + + template + using strmap_t = std::map; + + strmap_t files; + strmap_t> decomps; + + // In the above map, we would like to label decomps as dtype_dim1$N1_dim2$N2... + // where N$i is the global length of dim$i. However, it *may* happen that we use + // two different decompositions for the same global layout, which would clash + // the names. For this reason, we append at the end an increasing counter, + // which disambiguate between globally-equivalent partitions. + // When adding a new decomp, we check this map to see if another decomp + // already exists with the same global layout. If so, we first check to see + // if that decomp is equivalent to the new one *on all ranks*. If yes, we + // recycle it, otherwise we create a new PIO decomp. + // strmap_t> decomp_global_layout_to_decomp_name; + // strmap_t decomp_global_layout_to_counter; + + int pio_sysid = -1; + int pio_type_default = -1; + int pio_rearranger = -1; + int pio_format = -1; + + ekat::Comm comm; + +private: + + ScorpioSession () = default; +}; + +// --------------------------------------------------------------------------------------------- // + +template +void copy_data (const S* src, D* dst, int n) { + for (int i=0; i& e) +{ + return e->name; +} + +template +std::string print_map_keys (const std::map& map) { + std::string s; + for (const auto& it : map) { + s += it.first + ","; + } + s.pop_back(); + return s; +} + // Retrieve the int codes PIO uses to specify data types int nctype (const std::string& type) { if (type=="int") { @@ -65,637 +136,1338 @@ int nctype (const std::string& type) { #else return PIO_FLOAT; #endif + } else if (type=="char") { + return PIO_CHAR; } else { EKAT_ERROR_MSG ("Error! Unrecognized/unsupported data type '" + type + "'.\n"); } } -std::string nctype2str (const int type) { - for (auto t : {"int", "int64", "float", "double"}) { - if (nctype(t)==type) return t; + +template +int nctype () { + if (std::is_same::value) { + return PIO_INT; + } else if (std::is_same::value) { + return PIO_INT64; + } else if (std::is_same::value) { + return PIO_FLOAT; + } else if (std::is_same::value) { + return PIO_DOUBLE; + } else if (std::is_same::value) { + return PIO_CHAR; + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported data type.\n"); } - return "UNKNOWN"; } -/* ----------------------------------------------------------------- */ -void eam_init_pio_subsystem(const ekat::Comm& comm) { - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - eam_init_pio_subsystem(fcomm); + +template +const void* ncdata (const T& v) { + return reinterpret_cast(&v); +} +template<> +const void* ncdata (const std::string& v) { + return reinterpret_cast(v.data()); +} + +template +PIO_Offset nclen (const T& v) { + return 1; +} +template<> +PIO_Offset nclen (const std::string& v) { + return v.size(); } -void eam_init_pio_subsystem(const int mpicom, const int atm_id) { - // TODO: Right now the compid has been hardcoded to 0 and the flag - // to create a init a subsystem in SCREAM is hardcoded to true. - // When surface coupling is established we will need to refactor this - // routine to pass the appropriate values depending on if we are running - // the full model or a unit test. - eam_init_pio_subsystem_c2f(mpicom,atm_id); +std::string refine_dtype (const std::string& dtype) { + if (dtype=="real") { +#if defined(SCREAM_DOUBLE_PRECISION) + return "double"; +#else + return "float"; +#endif + } else if (dtype=="single") { + return "float"; + } else { + return dtype; + } } -/* ----------------------------------------------------------------- */ -void eam_pio_finalize() { - eam_pio_finalize_c2f(); + +template +std::string get_dtype () { + using raw_t = typename std::remove_cv::type; + std::string s; + if (std::is_same::value) { + s = "int"; + } else if (std::is_same::value) { + s = "float"; + } else if (std::is_same::value) { + s = "double"; + } else if (std::is_integral::value && + std::is_signed::value && + sizeof(raw_t)==sizeof(long long)) { + s = "int64"; + } else { + EKAT_ERROR_MSG ("Error! Invalid/unsupported data type.\n"); + } + return s; } -/* ----------------------------------------------------------------- */ -void register_file(const std::string& filename, const FileMode mode, const int iotype) { - register_file_c2f(filename.c_str(),mode,iotype); + +size_t dtype_size (const std::string& dtype) { + if (dtype=="int") { + return sizeof(int); + } else if (dtype=="int64") { + return sizeof(long long); + } else if (dtype=="float") { + return sizeof(float); + } else if (dtype=="double") { + return sizeof(double); + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported data type '" + dtype + "'.\n"); + } + return 0; } -/* ----------------------------------------------------------------- */ -void eam_pio_closefile(const std::string& filename) { - eam_pio_closefile_c2f(filename.c_str()); +// ====================== Local utilities ========================== // + +namespace impl { +// Note: these utilities are used in this file to retrieve PIO entities, +// so that we implement all checks once (rather than in every function) + +// Small struct that allows to quickly open a file (in Read mode) if it wasn't open. +// If the file had to be open, when the struct is deleted, it will release the file. +struct PeekFile { + PeekFile(const std::string& filename_in) { + filename = filename_in; + was_open = is_file_open(filename); + if (not was_open) { + register_file(filename,Read); + } + } + + ~PeekFile () { + if (not was_open) { + // Note: this function _could_ throw, but it should not happen (unless someone + // else called release_file twice). That's b/c we either are not the only + // customer (so nothing to be done other than a ref count decrement) or + // the file was open in Read mode, in which case it doesn't need to do + // much in scorpio. + release_file(filename); + } + } +private: + std::string filename; + bool was_open; +}; + +PIOFile& get_file (const std::string& filename, + const std::string& context) +{ + auto& s = ScorpioSession::instance(); + + EKAT_REQUIRE_MSG (s.files.count(filename)==1, + "Error! Could not retrieve the file. File not open.\n" + " - filename: " + filename + "\n" + "Context:\n" + " " + context + "\n"); + + return s.files.at(filename); } -void eam_flush_file(const std::string& filename) { - eam_pio_flush_file_c2f(filename.c_str()); + +PIODim& get_dim (const std::string& filename, + const std::string& dimname, + const std::string& context) +{ + const auto& f = get_file(filename,context); + EKAT_REQUIRE_MSG (f.dims.count(dimname)==1, + "Error! Could not retrieve dimension. Dimension not found.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n" + " - dims on file: " + print_map_keys(f.dims) + "\n" + "Context:\n" + " " + context + "\n"); + + return *f.dims.at(dimname); } -/* ----------------------------------------------------------------- */ -void set_decomp(const std::string& filename) { - set_decomp_c2f(filename.c_str()); +PIOVar& get_var (const std::string& filename, + const std::string& varname, + const std::string& context) +{ + const auto& f = get_file(filename,context); + EKAT_REQUIRE_MSG (f.vars.count(varname)==1, + "Error! Could not retrieve variable. Variable not found.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n" + " - vars on file : " + print_map_keys(f.vars) + "\n" + "Context:\n" + " " + context + "\n"); + + return *f.vars.at(varname); } -/* ----------------------------------------------------------------- */ -int get_dimlen(const std::string& filename, const std::string& dimname) + +} // namespace impl + +// ====================== Global IO operations ======================= // + +void init_subsystem(const ekat::Comm& comm, const int atm_id) { - int ncid, dimid, err; - PIO_Offset len; + auto& s = ScorpioSession::instance(); + s.comm = comm; - bool was_open = is_file_open_c2f(filename.c_str(),-1); - if (not was_open) { - register_file(filename,Read); - } + EKAT_REQUIRE_MSG (s.pio_sysid==-1, + "Error! Attmept to re-initialize pio subsystem.\n"); - ncid = get_file_ncid_c2f (filename.c_str()); - err = PIOc_inq_dimid(ncid,dimname.c_str(),&dimid); - EKAT_REQUIRE_MSG (err!=PIO_EBADDIM, - "Error! Could not find dimension in the file.\n" - " - filename : " + filename + "\n" - " - dimname : " + dimname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving dimension id.\n" - " - filename : " + filename + "\n" - " - dimname : " + dimname + "\n" - " - pio error: " + std::to_string(err) + "\n"); +#ifdef SCREAM_CIME_BUILD + s.pio_sysid = shr_get_iosysid_c2f(atm_id); + s.pio_type_default = shr_get_iotype_c2f(atm_id); + s.pio_rearranger = shr_get_rearranger_c2f(atm_id); + s.pio_format = shr_get_ioformat_c2f(atm_id); +#else + // Use some reasonable defaults for standalone EAMxx tests + int stride = 1; + int base = 0; - err = PIOc_inq_dimlen(ncid,dimid,&len); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while querying dimension length.\n" - " - filename : " + filename + "\n" - " - dimname : " + dimname + "\n" - " - pio error: " + std::to_string(err) + "\n"); + s.pio_rearranger = PIO_REARR_SUBSET; + s.pio_format = PIO_64BIT_DATA; +#if PIO_USE_PNETCDF + s.pio_type_default = PIO_IOTYPE_PNETCDF; +#elif PIO_USE_NETCDF + s.pio_type_default = PIO_IOTYPE_NETCDF; +#else +#error "Standalone EAMxx requires either PNETCDF or NETCDF iotype to be available in Scorpio" +#endif - if (not was_open) { - eam_pio_closefile(filename); - } + auto err = PIOc_Init_Intracomm(comm.mpi_comm(), comm.size(), stride, base, s.pio_rearranger, &s.pio_sysid); + check_scorpio_noerr (err,"init_subsystem", "Init_Intracomm"); + + // Unused in standalone mode + (void) atm_id; +#endif - return len; + static_assert (sizeof(offset_t)==sizeof(PIO_Offset), + "Error! PIO was configured with PIO_OFFSET not a 64-bit int.\n"); } -/* ----------------------------------------------------------------- */ -bool has_dim (const std::string& filename, const std::string& dimname) + +bool is_subsystem_inited () { + return ScorpioSession::instance().pio_sysid!=-1; +} + +void finalize_subsystem () { - int ncid, dimid, err; + auto& s = ScorpioSession::instance(); - bool was_open = is_file_open_c2f(filename.c_str(),-1); - if (not was_open) { - register_file(filename,Read); - } + // TODO: should we simply return instead? I think trying to finalize twice + // *may* be a sign of possible bugs, though with Catch2 testing + // I *think* there may be some issue with how the code is run. + EKAT_REQUIRE_MSG (s.pio_sysid!=-1, + "Error! PIO subsystem was already finalized.\n"); - ncid = get_file_ncid_c2f (filename.c_str()); - err = PIOc_inq_dimid(ncid,dimname.c_str(),&dimid); - if (err==PIO_EBADDIM) { - return false; + for (auto& it : s.files) { + EKAT_REQUIRE_MSG (it.second.num_customers==0, + "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + " - filename: " + it.first + "\n"); } + s.files.clear(); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving dimension id.\n" - " - filename : " + filename + "\n" - " - dimname : " + dimname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - if (not was_open) { - eam_pio_closefile(filename); + for (auto& it : s.decomps) { + EKAT_REQUIRE_MSG (it.second.use_count()==1, + "Error! ScorpioSession::finalize called, but a decomp is still stored elsewhere.\n" + " - decomp name: " + it.first + "\n"); + + int err = PIOc_freedecomp(s.pio_sysid,it.second->ncid); + check_scorpio_noerr(err,"finalize_subsystem","freedecomp"); } + s.decomps.clear(); - return true; +#ifndef SCREAM_CIME_BUILD + // Don't finalize in CIME builds, since the coupler will take care of it + PIOc_finalize (s.pio_sysid); +#endif + + s.pio_sysid = -1; + s.pio_type_default = -1; + s.pio_format = -1; + s.pio_rearranger = -1; } -/* ----------------------------------------------------------------- */ -bool has_variable (const std::string& filename, const std::string& varname) + +// ========================= File operations ===================== // + +void register_file (const std::string& filename, + const FileMode mode, + const IOType iotype) { - int ncid, varid, err; + auto& s = ScorpioSession::instance(); + auto& f = s.files[filename]; + EKAT_REQUIRE_MSG (f.mode==Unset || f.mode==mode, + "Error! File was already opened with a different mode.\n" + " - filename: " + filename + "\n" + " - old mode: " + e2str(f.mode) + "\n" + " - new mode: " + e2str(mode) + "\n"); + EKAT_REQUIRE_MSG (f.mode==Unset || f.iotype==iotype, + "Error! File was already opened with a different iotype.\n" + " - filename: " + filename + "\n" + " - old type: " + iotype2str(f.iotype) + "\n" + " - new type: " + iotype2str(iotype) + "\n"); - bool was_open = is_file_open_c2f(filename.c_str(),-1); - if (not was_open) { - register_file(filename,Read); + if (f.mode == Unset) { + // First time we ask for this file. Call PIO open routine(s) + int err; + int iotype_int = iotype==IOType::DefaultIOType + ? s.pio_type_default + : static_cast(iotype); + if (mode & Read) { + auto write = mode & Write ? PIO_WRITE : PIO_NOWRITE; + err = PIOc_openfile(s.pio_sysid,&f.ncid,&iotype_int,filename.c_str(),write); + f.enddef = true; + } else { + err = PIOc_createfile(s.pio_sysid,&f.ncid,&iotype_int,filename.c_str(),s.pio_format); + f.enddef = false; + } + + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while opening a file.\n" + " - filename : " + filename + "\n" + " - file mode: " + e2str(mode) + "\n" + " - pio error: " + std::to_string(err) + "\n"); + + f.mode = mode; + f.iotype = iotype; + f.name = filename; + + if (mode & Read) { + // Read all dims/vars from file + PIO_Offset len; + int ndims, nvars, ngatts, unlimdimid; + err = PIOc_inq(f.ncid, &ndims, &nvars, &ngatts, &unlimdimid); + check_scorpio_noerr(err,f.name,"register_file","inq"); + + char name[PIO_MAX_NAME]; + for (int idim=0; idim(); + dim->name = name; + dim->fid = f.ncid; + dim->length = len; + dim->ncid = idim; + dim->unlimited = idim==unlimdimid; + + if (dim->unlimited) { + f.time_dim = dim; + } + } + + auto find_dim = [&](int dimid) -> std::shared_ptr { + std::shared_ptr d; + for (auto it : f.dims) { + if (it.second->ncid==dimid) { + d = it.second; + } + } + EKAT_REQUIRE_MSG (d!=nullptr, + "Error! Could not locat dimension id in the file.\n" + " - filename: " + f.name + "\n" + " - dim id : " + std::to_string(dimid) + "\n"); + return d; + }; + int dtype,natts; + int dimids[PIO_MAX_DIMS]; + for (int ivar=0; ivar(); + var->name = name; + var->ncid = ivar; + var->fid = f.ncid; + for (int idim=0; idimunlimited) { + var->time_dep = true; + var->num_records = dim->length; + } else { + var->dims.push_back(find_dim(dimids[idim])); + } + } + + for (const auto& dt : {"int","int64","float","double","char"}) { + if (nctype(dt)==dtype) { + var->dtype = var->nc_dtype = dt; + break; + } + } + EKAT_REQUIRE_MSG (var->dtype!="", + "Error! Variable data type not supported.\n" + " - filename: " + filename + "\n" + " - varname : " + var->name + "\n" + " - nc dtype: " + std::to_string(dtype) + "\n"); + + for (int iatt=0; iattunits = name; + break; + } + } + } + } } + ++f.num_customers; +} - ncid = get_file_ncid_c2f (filename.c_str()); - err = PIOc_inq_varid(ncid,varname.c_str(),&varid); - if (err==PIO_ENOTVAR) { - return false; +void release_file (const std::string& filename) +{ + auto& f = impl::get_file(filename,"scorpio::release_file"); + + --f.num_customers; + if (f.num_customers>0) { + return; } - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving variable id.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - if (not was_open) { - eam_pio_closefile(filename); + + int err; + if (f.mode & Write) { + err = PIOc_sync(f.ncid); + check_scorpio_noerr (err,f.name,"release_file","sync"); } - return true; + err = PIOc_closefile(f.ncid); + check_scorpio_noerr (err,f.name,"release_file","closefile"); + + auto& s = ScorpioSession::instance(); + s.files.erase(filename); } -bool has_attribute (const std::string& filename, const std::string& attname) +void flush_file (const std::string &filename) { - return has_attribute(filename,"GLOBAL",attname); + auto& f = impl::get_file(filename,"scorpio::sync_file"); + + EKAT_REQUIRE_MSG (f.mode & Write, + "Error! Cannot call sync_file. File is read-only.\n" + " - filename: " + filename + "\n"); + + int err = PIOc_sync(f.ncid); + check_scorpio_noerr (err,f.name,"sync_file","sync"); } -bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname) +void redef(const std::string &filename) { - int ncid, varid, attid, err; + auto& f = impl::get_file(filename,"scorpio::redef"); - bool was_open = is_file_open_c2f(filename.c_str(),-1); - if (not was_open) { - register_file(filename,Read); + EKAT_REQUIRE_MSG (f.mode & Write, + "Error! Could not call redef on the input file. File is read-only.\n" + " - filename: " + filename + "\n"); + + if (f.enddef) { + int err = PIOc_redef(f.ncid); + check_scorpio_noerr (err,f.name,"redef","redef"); + f.enddef = false; } +} + +void enddef(const std::string &filename) +{ + auto& f = impl::get_file(filename,"scorpio::enddef"); - // Get file id - ncid = get_file_ncid_c2f (filename.c_str()); + if (not f.enddef) { + int err = PIOc_enddef(f.ncid); + check_scorpio_noerr (err,f.name,"enddef","enddef"); + f.enddef = true; + } +} - // Get var id - if (varname=="GLOBAL") { - varid = PIO_GLOBAL; +bool is_file_open (const std::string& filename, const FileMode mode) +{ + auto& s = ScorpioSession::instance(); + auto it = s.files.find(filename); + if (it==s.files.end()) return false; + + return mode==Unset || (mode & it->second.mode); +} + +// =================== Dimensions operations ======================= // + +void define_dim (const std::string& filename, const std::string& dimname, const int length) +{ + auto& f = impl::get_file(filename,"scorpio::define_dim"); + + EKAT_REQUIRE_MSG (f.mode & Write, + "Error! Could not define dimension. File is read-only.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n"); + + auto& dim = f.dims[dimname]; + + bool unlimited = length==0; + + if (dim==nullptr) { + EKAT_REQUIRE_MSG (f.mode!=Append, + "Error! Cannot add a new dim when the file is open in append mode.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n"); + // Create new dimension + dim = std::make_shared(); + dim->name = dimname; + dim->fid = f.ncid; + dim->length = length; + dim->unlimited = unlimited; + + // Define the dimension in PIO + int err = PIOc_def_dim(f.ncid,dimname.c_str(),dim->length,&dim->ncid); + check_scorpio_noerr (err,f.name,"dimension",dimname,"define_dim","def_dim"); } else { - err = PIOc_inq_varid(ncid,varname.c_str(),&varid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving variable id.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - pio error: " + std::to_string(err) + "\n"); + // Already defined. Check that the dim specs are the same. + EKAT_REQUIRE_MSG (unlimited==dim->unlimited, + "Error! Redefining dimension with different unlimited flag.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n" + " - old unlimited:" + (dim->unlimited ? "yes" : "no" )+ "\n" + " - new unlimited:" + (unlimited ? "yes" : "no") + "\n"); + + EKAT_REQUIRE_MSG (unlimited || length==dim->length, + "Error! Redefining dimension with a different (local) length.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n" + " - old length:" + std::to_string(dim->length)+ "\n" + " - new length:" + std::to_string(length) + "\n"); } +} - // Get att id - err = PIOc_inq_attid(ncid,varid,attname.c_str(),&attid); - if (err==PIO_ENOTATT) { +bool has_dim (const std::string& filename, + const std::string& dimname, + const int length) +{ + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); + + const auto& f = impl::get_file(filename,"scorpio::has_dim"); + + auto it = f.dims.find(dimname); + if (it==f.dims.end()) { return false; + } else if (length==0) { + return it->second->unlimited; + } else if (length>0) { + return it->second->length==length; } - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving attribute id.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - attname : " + attname + "\n" - " - pio error: " + std::to_string(err) + "\n"); + return true; +} - if (not was_open) { - eam_pio_closefile(filename); - } +int get_dimlen (const std::string& filename, const std::string& dimname) +{ + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); - return true; + EKAT_REQUIRE_MSG (has_dim(filename,dimname), + "Error! Could not inquire dimension length. The dimension is not in the file.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n"); + + auto& s = ScorpioSession::instance(); + auto& f = s.files[filename]; + + return f.dims.at(dimname)->length; } -/* ----------------------------------------------------------------- */ -void set_dof(const std::string& filename, const std::string& varname, const Int dof_len, const std::int64_t* x_dof) { - set_dof_c2f(filename.c_str(),varname.c_str(),dof_len,x_dof); +int get_dimlen_local (const std::string& filename, const std::string& dimname) +{ + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); + + EKAT_REQUIRE_MSG (has_dim(filename,dimname), + "Error! Could not inquire dimension local length. The dimension is not in the file.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n"); + + auto& s = ScorpioSession::instance(); + auto& f = s.files[filename]; + + const auto& dim = f.dims.at(dimname); + return dim->offsets==nullptr ? dim->length : dim->offsets->size(); } -/* ----------------------------------------------------------------- */ -void pio_update_time(const std::string& filename, const double time) { - pio_update_time_c2f(filename.c_str(),time); +int get_time_len (const std::string& filename) +{ + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); + + auto& s = ScorpioSession::instance(); + auto& f = s.files[filename]; + + EKAT_REQUIRE_MSG (f.time_dim, + "Error! Could not inquire time dimension length. The time dimension is not in the file.\n" + " - filename: " + filename + "\n"); + + return f.time_dim->length; } -/* ----------------------------------------------------------------- */ -void register_dimension(const std::string &filename, const std::string& shortname, const std::string& longname, const int length, const bool partitioned) + +std::string get_time_name (const std::string& filename) { - int mode = get_file_mode_c2f(filename.c_str()); - std::string mode_str = mode==Read ? "Read" : (mode==Write ? "Write" : "Append"); - if (mode!=Write) { - // Ensure the dimension already exists, and that it has the correct size (if not unlimited) - int ncid,dimid,unlimid,err; + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); - ncid = get_file_ncid_c2f (filename.c_str()); - err = PIOc_inq_dimid(ncid,shortname.c_str(),&dimid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Could not retrieve dimension id from file open in " + mode_str + " mode.\n" - " - filename: " + filename + "\n" - " - dimension : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); + auto& s = ScorpioSession::instance(); + auto& f = s.files[filename]; - err = PIOc_inq_unlimdim(ncid,&unlimid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong querying for the unlimited dimension id.\n" + EKAT_REQUIRE_MSG (f.time_dim, + "Error! Could not inquire time dimension name. The time dimension is not in the file.\n" + " - filename: " + filename + "\n"); + + return f.time_dim->name; +} + +// =================== Decompositions operations ==================== // + +// NOTES: +// - this is a local function, we don't expose it. It's only called inside other scorpio utilities +// - we don't really *need* filename, it's only to print more context in case of errors +void set_var_decomp (PIOVar& var, + const std::string& filename) +{ + for (size_t i=1; ioffsets==nullptr, + "Error! We currently only allow decomposition on slowest-striding dimension.\n" + " Generalizing is not complicated, but it was not a priority.\n" " - filename: " + filename + "\n" - " - dimension : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - if (length==0) { - EKAT_REQUIRE_MSG ( unlimid==dimid, - "Error! Input dimension is unlimited, but does not appear to be unlimited in the file (open in " + mode_str + " mode).\n" - " - filename: " + filename + "\n" - " - dimension : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - } else { - EKAT_REQUIRE_MSG ( unlimid!=dimid, - "Error! Input dimension is not unlimited, but it appears to be unlimited in the file (open in " + mode_str + " mode).\n" - " - filename: " + filename + "\n" - " - dimension : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); + " - varname : " + var.name + "\n" + " - var dims: " + ekat::join(var.dims,get_entity_name,",") + "\n" + " - bad dim : " + var.dims[i]->name + "\n"); + } + EKAT_REQUIRE_MSG (var.dims[0]->offsets!=nullptr, + "Error! Calling set_var_decomp, but the var first dimension does not appear to be decomposed.\n" + " - filename: " + filename + "\n" + " - varname : " + var.name + "\n" + " - var dims: " + ekat::join(var.dims,get_entity_name,",") + "\n"); + EKAT_REQUIRE_MSG (var.decomp==nullptr, + "Error! You should have invalidated var.decomp before attempting to reset it.\n" + " - filename : " + filename + "\n" + " - varname : " + var.name + "\n" + " - var decomp: " + var.decomp->name + "\n"); - int len_from_file = get_dimlen(filename,shortname); - EKAT_REQUIRE_MSG (length==len_from_file, - "Error! Input dimension length does not match the one from the file (open in " + mode_str + " mode).\n" - " - filename: " + filename + "\n" - " - dimension : " + shortname + "\n" - " - input dim length: " + std::to_string(length) + "\n" - " - file dim length : " + std::to_string(len_from_file) + "\n"); + // Create decomp name: dtype-dim1_dim2_..._dimk + std::shared_ptr decomp_dim; + std::string decomp_tag = var.dtype + "-"; + for (auto d : var.dims) { + decomp_tag += d->name + "<" + std::to_string(d->length) + ">_"; + } + decomp_tag.pop_back(); // remove trailing underscore + + // Check if a decomp with this name already exists + auto& s = ScorpioSession::instance(); + auto& decomp = s.decomps[decomp_tag]; +#ifndef NDEBUG + // Extra check: all ranks must agree on whether they have the decomposition! + // If they don't agree, some rank will be stuck in a PIO call, waiting for others + int found = decomp==nullptr ? 0 : 1; + int min_found, max_found; + const auto& comm = ScorpioSession::instance().comm; + comm.all_reduce(&found,&min_found,1,MPI_MIN); + comm.all_reduce(&found,&max_found,1,MPI_MAX); + EKAT_REQUIRE_MSG(min_found==max_found, + "Error! Decomposition already present on some ranks but not all.\n" + " - filename: " + filename + "\n" + " - varname : " + var.name + "\n" + " - var dims: " + ekat::join(var.dims,get_entity_name,",") + "\n" + " - decopm tag: " + decomp_tag + "\n"); +#endif + + if (decomp==nullptr) { + // We haven't create this decomp yet. Go ahead and create one + decomp = std::make_shared(); + decomp->name = decomp_tag; + decomp->dim = var.dims[0]; + + int ndims = var.dims.size(); + + // Get ALL dims global lengths, and compute prod of *non-decomposed* dims + std::vector gdimlen = {decomp->dim->length}; + int non_decomp_dim_prod = 1; + for (int idim=1; idimlength); + non_decomp_dim_prod *= d->length; + } + + // Create offsets list + const auto& dim_offsets = *decomp->dim->offsets; + int dim_loc_len = dim_offsets.size(); + decomp->offsets.resize (non_decomp_dim_prod*dim_loc_len); + for (int idof=0; idofoffsets.begin()+ idof*non_decomp_dim_prod; + auto end = beg + non_decomp_dim_prod; + std::iota (beg,end,non_decomp_dim_prod*dof_offset); } + + // Create PIO decomp + int maplen = decomp->offsets.size(); + PIO_Offset* compmap = reinterpret_cast(decomp->offsets.data()); + int err = PIOc_init_decomp(s.pio_sysid,nctype(var.dtype),ndims,gdimlen.data(), + maplen,compmap, &decomp->ncid,s.pio_rearranger, + nullptr,nullptr); + + check_scorpio_noerr(err,filename,"decomp",decomp_tag,"set_var_decomp","InitDecomp"); } - register_dimension_c2f(filename.c_str(), shortname.c_str(), longname.c_str(), length, partitioned); + // Set decomp data in the var + var.decomp = decomp; } -/* ----------------------------------------------------------------- */ -void register_variable(const std::string& filename, const std::string& shortname, const std::string& longname, - const std::vector& var_dimensions, - const std::string& dtype, const std::string& pio_decomp_tag) + +void set_dim_decomp (const std::string& filename, + const std::string& dimname, + const std::vector& my_offsets, + const bool allow_reset) { - // This overload does not require to specify an nc data type, so it *MUST* be used when the - // file access mode is either Read or Append. Either way, a) the var should be on file already, - // and b) so should be the dimensions - EKAT_REQUIRE_MSG (has_variable(filename,shortname), - "Error! This overload of register_variable *assumes* the variable is already in the file, but wasn't found.\n" + auto& s = ScorpioSession::instance(); + auto& f = impl::get_file(filename,"scorpio::set_decomp"); + auto& dim = impl::get_dim(filename,dimname,"scorpio::set_dim_decomp"); + + EKAT_REQUIRE_MSG (not dim.unlimited, + "Error! Cannot partition an unlimited dimension.\n" " - filename: " + filename + "\n" - " - varname : " + shortname + "\n"); - for (const auto& dimname : var_dimensions) { - int len = get_dimlen (filename,dimname); - // WARNING! If the dimension was not yet registered, it will be registered as a NOT partitioned dim. - // If this dim should be partitioned, then register it *before* the variable - register_dimension(filename,dimname,dimname,len,false); - } - register_variable(filename,shortname,longname,"",var_dimensions,dtype,"",pio_decomp_tag); -} -void register_variable(const std::string &filename, const std::string& shortname, const std::string& longname, - const std::string& units_in, const std::vector& var_dimensions, - const std::string& dtype, const std::string& nc_dtype_in, const std::string& pio_decomp_tag) -{ - // Local copies, since we can modify them in case of defaults - auto units = units_in; - auto nc_dtype = nc_dtype_in; - - int mode = get_file_mode_c2f(filename.c_str()); - std::string mode_str = mode==Read ? "Read" : (mode==Write ? "Write" : "Append"); - - bool has_var = has_variable(filename,shortname); - if (mode==Write) { - EKAT_REQUIRE_MSG ( units!="" and nc_dtype!="", - "Error! Missing valid units and/or nc_dtype arguments for file open in Write mode.\n" - " - filename: " + filename + "\n" - " - varname : " + shortname + "\n"); - } else { - EKAT_REQUIRE_MSG ( has_var, - "Error! Variable not found in file open in " + mode_str + " mode.\n" + " - dimname : " + dimname + "\n"); + + if (dim.offsets!=nullptr) { + if (allow_reset) { + // We likely won't need the previously created decomps that included this dimension. + // So, as we remove decomps from vars that have this dim, keep track of their name, + // so that we can free them later *if no other users of them remain*. + std::set decomps_to_remove; + for (auto it : f.vars) { + auto v = it.second; + if (v->decomp!=nullptr and v->decomp->dim->name==dimname) { + decomps_to_remove.insert(v->decomp->name); + v->decomp = nullptr; + } + } + for (const auto& dn : decomps_to_remove) { + if (s.decomps.at(dn).use_count()==1) { + auto decomp = s.decomps.at(dn); + // There is no other customer of this decomposition, so we can safely free it + int err = PIOc_freedecomp(s.pio_sysid,decomp->ncid); + check_scorpio_noerr(err,filename,"decomp",dn,"set_dim_decomp","freedecomp"); + s.decomps.erase(dn); + } + } + } else { + // Check that the offsets are (globally) the same + int same = *dim.offsets==my_offsets; + const auto& comm = ScorpioSession::instance().comm; + comm.all_reduce(&same,1,MPI_MIN); + EKAT_REQUIRE_MSG(same==1, + "Error! Attempt to redefine a decomposition with a different dofs distribution.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n" + "If you are attempting to redefine the decomp, call this function with throw_if_changing_decomp=false.\n"); + + // Same decomposition, so we can just return + return; + } + } + + // Check that offsets are less than the global dimension length + for (auto o : my_offsets) { + EKAT_REQUIRE_MSG (o>=0 && o>(my_offsets); - if (has_var) { - // The file already exists or the var was already registered. - // Make sure we're registering the var with the same specs - int ncid = get_file_ncid_c2f (filename.c_str()); - int vid,ndims,err; - err = PIOc_inq_varid(ncid,shortname.c_str(),&vid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong retrieving variable id.\n" - " - filename: " + filename + "\n" - " - varname : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - err = PIOc_inq_varndims(ncid,vid,&ndims); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong inquiring the number of dimensions of a variable.\n" - " - filename: " + filename + "\n" - " - varname : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - std::vector dims(ndims); - err = PIOc_inq_vardimid(ncid,vid,dims.data()); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong inquiring the dimensions ids of a variable.\n" + // If vars were already defined, we need to process them, + // and create the proper PIODecomp objects. + for (auto it : f.vars) { + if (ekat::contains(it.second->dim_names(),dimname)) { + set_var_decomp (*it.second,filename); + } + } +} + +void set_dim_decomp (const std::string& filename, + const std::string& dimname, + const offset_t start, const offset_t count, + const bool allow_reset) +{ + std::vector offsets(count); + std::iota(offsets.begin(),offsets.end(),start); + set_dim_decomp(filename,dimname,offsets,allow_reset); +} + +void set_dim_decomp (const std::string& filename, + const std::string& dimname, + const bool allow_reset) +{ + const auto& comm = ScorpioSession::instance().comm; + + const int glen = get_dimlen(filename,dimname); + int len = glen / comm.size(); + if (comm.rank() < (glen % comm.size())) { + ++len; + } + + offset_t offset = len; + comm.scan(&offset,1,MPI_SUM); + offset -= len; // scan is inclusive, but we need exclusive + + set_dim_decomp (filename,dimname,offset,len,allow_reset); +} + +// ================== Variable operations ================== // + +// Define var on output file (cannot call on Read/Append files) +void define_var (const std::string& filename, const std::string& varname, + const std::string& units, const std::vector& dimensions, + const std::string& dtype, const std::string& nc_dtype, + const bool time_dep) +{ + auto& f = impl::get_file(filename,"scorpio::define_var"); + + EKAT_REQUIRE_MSG (f.mode & Write, + "Error! Could not define variable. File is read-only.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n"); + + EKAT_REQUIRE_MSG (not time_dep || f.time_dim!=nullptr, + "Error! Cannot define time-dependent variable: no time dimension defined.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n"); + + if (f.vars.count(varname)==0) { + EKAT_REQUIRE_MSG (f.mode!=Append, + "Error! Cannot add a new var when the file is open in append mode.\n" " - filename: " + filename + "\n" - " - varname : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - std::vector dims_from_file(ndims); - for (int i=0; i(); + var->name = varname; + var->fid = f.ncid; + var->units = units; + var->dtype = refine_dtype(dtype); + var->nc_dtype = refine_dtype(nc_dtype); + var->time_dep = time_dep; + int ndims = dimensions.size() + (time_dep ? 1 : 0); + std::vector dimids; + if (time_dep) { + dimids.push_back(f.time_dim->ncid); + } + for (const auto& dname : dimensions) { + EKAT_REQUIRE_MSG (has_dim(filename,dname), + "Error! Cannot create variable. Dimension not found.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - var dims : " + ekat::join(dimensions,",") + "\n" + " - file dims: " + print_map_keys(f.dims) + "\n"); + auto dim = f.dims.at(dname); + var->dims.push_back(dim); + dimids.push_back(dim->ncid); } - // Here, let's only try to access var_dimensions[0] when we know for sure - // that var_dimensions is actually dimensioned (i.e., .size()>0) - if (var_dimensions.size()>0) { - if (mode==Read && (dims_from_file[0]=="time" && var_dimensions[0]!="time")) { - // For Read operations, we may not consider "time" as a field dimension, so if the - // input file has "time", simply disregard it in this check. - dims_from_file.erase(dims_from_file.begin()); - } - } else { - if (mode==Read && (dims_from_file[0]=="time")) { - // For Read operations, we may not consider "time" as a field dimension, so if the - // input file has "time", simply disregard it in this check. - dims_from_file.erase(dims_from_file.begin()); - } + // Define the variable in PIO + int err = PIOc_def_var(f.ncid,varname.c_str(),nctype(nc_dtype),ndims,dimids.data(),&var->ncid); + check_scorpio_noerr(err,f.name,"variable",varname,"define_var","def_var"); + + f.vars[varname] = var; + + if (units!="") { + // Add units attribute + set_attribute(filename,varname,"units",units); } - std::reverse(dims_from_file.begin(),dims_from_file.end()); - EKAT_REQUIRE_MSG(var_dimensions==dims_from_file, - "Error! Input variable dimensions do not match the ones from the file.\n" - " - filename : " + filename + "\n" - " - varname : " + shortname + "\n" - " - input dims: (" + ekat::join(var_dimensions,",") + ")\n" - " - file dims : (" + ekat::join(dims_from_file,",") + ")\n"); - - // Check nc dtype only if user bothered specifying it (if mode=Read, probably the user doesn't care) - nc_type type; - err = PIOc_inq_vartype(ncid,vid,&type); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while inquiring variable data type.\n" - " - filename: " + filename + "\n" - " - varname : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - EKAT_REQUIRE_MSG (nc_dtype=="" or type==nctype(nc_dtype), - "Error! Input NC data type does not match the one from the file.\n" - " - filename: " + filename + "\n" - " - varname : " + shortname + "\n" - " - input dtype : " + nc_dtype + "\n" - " - file dtype : " + nctype2str(type) + "\n"); - nc_dtype = nctype2str(type); - - // Get var units (if set) - int natts; - err = PIOc_inq_varnatts(ncid,vid,&natts); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while inquiring a variable's number of attributes.\n" - " - filename: " + filename + "\n" - " - varname : " + shortname + "\n" - " - pio error: " + std::to_string(err) + "\n"); - std::string units_from_file(PIO_MAX_NAME,'\0'); - std::string att_name(PIO_MAX_NAME,'\0'); - for (int i=0; idims.size()>0 and var->dims[0]->offsets!=nullptr) { + set_var_decomp (*var,filename); } + } else { + const auto& var = f.vars.at(varname); + // The variable was already defined. Check that important metadata is the same + EKAT_REQUIRE_MSG (var->units==units, + "Error! Attempt to redefine variable with different units.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - old units: " + var->units + "\n" + " - new units: " + units + "\n"); + EKAT_REQUIRE_MSG (var->dtype==refine_dtype(dtype), + "Error! Attempt to redefine variable with different data type.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - old dtype: " + var->dtype + "\n" + " - new dtype: " + refine_dtype(dtype) + "\n"); + EKAT_REQUIRE_MSG (var->nc_dtype==refine_dtype(nc_dtype), + "Error! Attempt to redefine variable with different PIO data type.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - old pio dtype: " + var->nc_dtype + "\n" + " - new pio dtype: " + refine_dtype(nc_dtype) + "\n"); + EKAT_REQUIRE_MSG (var->time_dep==time_dep, + "Error! Attempt to redefine variable with different time dep flag.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - old time_dep: " + (var->time_dep ? "yes" : "no") + "\n" + " - new time_dep: " + ( time_dep ? "yes" : "no") + "\n"); + const auto var_dims = ekat::join(var->dims,get_entity_name,","); + EKAT_REQUIRE_MSG (var_dims==ekat::join(dimensions,","), + "Error! Attempt to redefine variable with different dimensions.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n" + " - old dims: " + var_dims + "\n" + " - new dims: " + ekat::join(dimensions,",") + "\n"); + } +} + +void define_var (const std::string& filename, const std::string& varname, + const std::vector& dimensions, + const std::string& dtype, + const bool time_dependent) +{ + define_var(filename,varname,"",dimensions,dtype,dtype,time_dependent); +} + +void change_var_dtype (const std::string& filename, const std::string& varname, + const std::string& dtype) +{ + auto& var = impl::get_var(filename,varname,"scorpio::change_var_dtype"); + + if (refine_dtype(dtype)==refine_dtype(var.dtype)) { + return; } - // Convert the vector of strings that contains the variable dimensions to a char array - const int numdims = var_dimensions.size(); - std::vector var_dimensions_c(numdims); - for (int ii = 0;iiname; + double t; + read_var(filename,time_name,&t,time_index); + return t; } -/* ----------------------------------------------------------------- */ -void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, std::string& meta_val) { - meta_val.resize(256); - get_variable_metadata_char_c2f(filename.c_str(),varname.c_str(),meta_name.c_str(),&meta_val[0]); - // If terminating char is not found, meta_val simply uses all 256 chars - if (meta_val.find('\0')!=std::string::npos) { - meta_val.resize(meta_val.find('\0')); +std::vector get_all_times (const std::string& filename) +{ + const auto& f = impl::get_file(filename,"scorpio::get_all_times"); + const auto& dim = *f.time_dim; + + std::vector times (dim.length); + for (int i=0; i=0, - "[get_any_attribute] Error! Could not retrieve file ncid.\n" - " - filename : " + filename + "\n"); - int varid; +// Read variable into user provided buffer. +// If time dim is present, read given time slice (time_index=-1 means "read last record). +// If time dim is not present, time_index must be -1 (error out otherwise) +template +void read_var (const std::string &filename, const std::string &varname, T* buf, const int time_index) +{ + EKAT_REQUIRE_MSG (buf!=nullptr, + "Error! Cannot read from provided pointer. Invalid buffer pointer.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n"); + + const auto& f = impl::get_file(filename,"scorpio::read_var"); + auto& var = impl::get_var(filename,varname,"scorpio::read_var"); + int err; - if (var_name=="GLOBAL") { - varid = PIO_GLOBAL; - } else { - err = PIOc_inq_varid(ncid, var_name.c_str(), &varid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "[get_any_attribute] Error! Something went wrong while inquiring variable id.\n" - " - filename : " + filename + "\n" - " - variable : " + var_name + "\n" - " - attribute: " + att_name + "\n" - " - pio error: " << err << "\n"); + int frame = -1; + if (var.time_dep) { + frame = time_index>=0 ? time_index : f.time_dim->length-1; + EKAT_REQUIRE_MSG (framelength, + "Error! Time index out of bounds.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n" + " - time idx: " + std::to_string(time_index) + "\n" + " - time len: " + std::to_string(f.time_dim->length)); + err = PIOc_setframe(f.ncid,var.ncid,frame); + check_scorpio_noerr (err,f.name,"variable",varname,"read_var","setframe"); + } else if (time_index>=0) { + // This is a bit of a hacky usage. We want to read a time index of a var, + // but the time dim is NOT unlimited in the input file. Apparently, SCORPIO + // supports this, and some E3SM input files are indeed like this, so we + // need to support it here too, hacky as it may be. + frame = time_index; + + EKAT_REQUIRE_MSG (framelength, + "Error! First dim index out of bounds.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - first dim idx: " + std::to_string(time_index) + "\n" + " - first dim len: " + std::to_string(var.dims[0]->length) + "\n"); } - nc_type type; - PIO_Offset len; - err = PIOc_inq_att(ncid,varid,att_name.c_str(),&type,&len); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "[get_any_attribute] Error! Something went wrong while inquiring global attribute.\n" + // The user may be using a different data type than the one that is in the file + // If that's the case, they should call `change_var_dtype` *before* trying to read it + if (get_dtype()!=var.dtype) { + EKAT_ERROR_MSG ( + "Error! You're attempting to read a variable with the wrong data type.\n" + "If you want to use a different data type, call change_var_dtype *before* attempting a read.\n" " - filename : " + filename + "\n" - " - variable : " + var_name + "\n" - " - attribute: " + att_name + "\n" - " - pio error: " << err << "\n"); + " - varname : " + varname + "\n" + " - var dtype: " + var.dtype + "\n" + " - ptr dtype: " + get_dtype() + "\n"); + } - EKAT_REQUIRE_MSG (len==1 || type==PIO_CHAR, - "[get_any_attribute] Error! Only single value attributes allowed.\n" - " - filename : " + filename + "\n" - " - variable : " + var_name + "\n" - " - attribute: " + att_name + "\n" - " - nc type : " << type << "\n" - " - att len : " << len << "\n"); - - ekat::any att; - if (type==PIO_INT) { - int val; - err = PIOc_get_att(ncid,varid,att_name.c_str(),&val); - att.reset(val); - } else if (type==PIO_DOUBLE) { - double val; - err = PIOc_get_att(ncid,varid,att_name.c_str(),&val); - att.reset(val); - } else if (type==PIO_FLOAT) { - float val; - err = PIOc_get_att(ncid,varid,att_name.c_str(),&val); - att.reset(val); - } else if (type==PIO_CHAR) { - std::string val(len,'\0'); - err = PIOc_get_att(ncid,varid,att_name.c_str(),&val[0]); - att.reset(val); + std::string pioc_func; + if (var.decomp) { + // A decomposed variable, requires read_darray + err = PIOc_read_darray(f.ncid,var.ncid,var.decomp->ncid,var.decomp->offsets.size(),buf); + pioc_func = "read_darray"; } else { - EKAT_ERROR_MSG ("[get_any_attribute] Error! Unsupported/unrecognized nc type.\n" - " - filename : " + filename + "\n" - " - variable : " + var_name + "\n" - " - attribute: " + att_name + "\n" - " - nc type : " << type << "\n"); - } - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "[get_any_attribute] Error! Something went wrong while inquiring global attribute.\n" - " - filename : " + filename + "\n" - " - variable : " + var_name + "\n" - " - attribute: " + att_name + "\n" - " - pio error: " << err << "\n"); + // A non-decomposed variable, use PIOc_get_var(a) + + // If nc data type doesn't match the input pointer, we need to use the var internal buffer + void* io_buf = buf; + if (var.dtype!=var.nc_dtype) { + if (var.size==-1) { + var.size = 1; + for (auto d : var.dims) { + var.size *= d->length; + } + var.buf.resize(var.size*dtype_size(var.nc_dtype)); + } + io_buf = var.buf.data(); + } - eam_pio_closefile(filename); - return att; + if (frame>=0) { + // We need to get the start/count for each dimension + int ndims = var.dims.size(); + std::vector start (ndims+1,0), count(ndims+1); // +1 for time + start[0] = frame; + count[0] = 1; + for (int idim=0; idimlength; + } + err = PIOc_get_vara(f.ncid,var.ncid,start.data(),count.data(),io_buf); + pioc_func = "get_vara"; + } else { + err = PIOc_get_var(f.ncid,var.ncid,io_buf); + pioc_func = "get_var"; + } + + // If we used the var tmp buffer, copy back into the user-provided pointer + if (var.dtype!=var.nc_dtype) { + if (var.nc_dtype=="int") { + copy_data(reinterpret_cast(io_buf),buf,var.size); + } else if (var.nc_dtype=="int64") { + copy_data(reinterpret_cast(io_buf),buf,var.size); + } else if (var.nc_dtype=="float") { + copy_data(reinterpret_cast(io_buf),buf,var.size); + } else if (var.nc_dtype=="double") { + copy_data(reinterpret_cast(io_buf),buf,var.size); + } + } + } + check_scorpio_noerr (err,f.name,"variable",varname,"read_var",pioc_func); } -void set_any_attribute (const std::string& filename, const std::string& att_name, const ekat::any& att) { - auto ncid = get_file_ncid_c2f (filename.c_str()); + +// Write data from user provided buffer into the requested variable +template +void write_var (const std::string &filename, const std::string &varname, const T* buf, const T* fillValue) +{ + EKAT_REQUIRE_MSG (buf!=nullptr, + "Error! Cannot write in provided pointer. Invalid buffer pointer.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n"); + + const auto& f = impl::get_file(filename,"scorpio::write_var"); + auto& var = impl::get_var(filename,varname,"scorpio::write_var"); int err; - EKAT_REQUIRE_MSG (ncid>=0, - "[set_any_attribute] Error! Could not retrieve file ncid.\n" - " - filename : " + filename + "\n"); + if (var.time_dep) { + ++var.num_records; + EKAT_REQUIRE_MSG (var.num_records==f.time_dim->length, + "Error! Number of records for variable does not match time length.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n" + " - time len: " + std::to_string(f.time_dim->length) + "\n" + " - nrecords: " + std::to_string(var.num_records) + "\n"); + err = PIOc_setframe (f.ncid,var.ncid,var.num_records-1); + check_scorpio_noerr (err,f.name,"variable",varname,"write_var","setframe"); + } - bool redef = is_enddef_c2f(filename.c_str()); - if (redef) { - err = PIOc_redef(ncid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "[set_any_attribute] Error! Something went wrong while re-opening def phase.\n" - " - filename : " + filename + "\n" - " - attribute: " + att_name + "\n" - " - pio error: " << err << "\n"); - } - - int varid = PIO_GLOBAL; - if (att.isType()) { - const int& data = ekat::any_cast(att); - err = PIOc_put_att(ncid,varid,att_name.c_str(),PIO_INT,1,&data); - } else if (att.isType()) { - const double& data = ekat::any_cast(att); - err = PIOc_put_att(ncid,varid,att_name.c_str(),PIO_DOUBLE,1,&data); - } else if (att.isType()) { - const float& data = ekat::any_cast(att); - err = PIOc_put_att(ncid,varid,att_name.c_str(),PIO_FLOAT,1,&data); - } else if (att.isType()) { - const std::string& data = ekat::any_cast(att); - err = PIOc_put_att(ncid,varid,att_name.c_str(),PIO_CHAR,data.size(),data.data()); - } else { - EKAT_ERROR_MSG ("[set_any_attribute] Error! Unsupported/unrecognized att type.\n" + // The user may be using a different data type than the one that is in the file + if (get_dtype()!=var.dtype) { + EKAT_ERROR_MSG ( + "Error! You're attempting to write a variable with the wrong data type.\n" + "If you want to use a different data type, call change_var_dtype *before* attempting a write.\n" " - filename : " + filename + "\n" - " - att name : " + att_name + "\n" - " - att value: " << att << "\n" - " - type info: " << att.content().type().name() << "\n"); + " - varname : " + varname + "\n" + " - var dtype: " + var.dtype + "\n" + " - ptr dtype: " + get_dtype() + "\n"); } - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "[set_any_attribute] Error! Something went wrong while setting global attribute.\n" - " - filename : " + filename + "\n" - " - attribute: " + att_name + "\n" - " - pio error: " << err << "\n"); + std::string pioc_func; + if (var.decomp) { + // A decomposed variable, requires write_darray + err = PIOc_write_darray(f.ncid,var.ncid,var.decomp->ncid,var.decomp->offsets.size(),buf,fillValue); + pioc_func = "write_darray"; + } else { + // A non-decomposed variable, use PIOc_put_var(a) + // If nc data type doesn't match the input pointer, we need to use the var internal buffer + const void* io_buf = buf; + if (var.dtype!=var.nc_dtype) { + if (var.size==-1) { + var.size = 1; + for (auto d : var.dims) { + var.size *= d->length; + } + var.buf.resize(var.size*dtype_size(var.nc_dtype)); + } + io_buf = var.buf.data(); + void* var_buf = var.buf.data(); + if (var.nc_dtype=="int") { + copy_data(buf,reinterpret_cast(var_buf),var.size); + } else if (var.nc_dtype=="int64") { + copy_data(buf,reinterpret_cast(var_buf),var.size); + } else if (var.nc_dtype=="float") { + copy_data(buf,reinterpret_cast(var_buf),var.size); + } else if (var.nc_dtype=="double") { + copy_data(buf,reinterpret_cast(var_buf),var.size); + } + } - if (redef) { - err = PIOc_enddef(ncid); - EKAT_REQUIRE_MSG (err==PIO_NOERR, - "[set_any_attribute] Error! Something went wrong while re-closing def phase.\n" - " - filename : " + filename + "\n" - " - attribute: " + att_name + "\n" - " - pio error: " << err << "\n"); + if (var.time_dep) { + // We need to get the start/count for each dimension + int ndims = var.dims.size(); + std::vector start (ndims+1,0), count(ndims+1); + start[0] = f.time_dim->length-1; + count[0] = 1; + for (int idim=0; idimlength; + } + err = PIOc_put_vara(f.ncid,var.ncid,start.data(),count.data(),io_buf); + pioc_func = "put_vara"; + } else { + // Easy: just pass the buffer, and write all entries + err = PIOc_put_var(f.ncid,var.ncid,io_buf); + pioc_func = "put_var"; + } } + check_scorpio_noerr (err,f.name,"variable",varname,"write_var",pioc_func); } -/* ----------------------------------------------------------------- */ -void eam_pio_enddef(const std::string &filename) { - eam_pio_enddef_c2f(filename.c_str()); -} -/* ----------------------------------------------------------------- */ -void eam_pio_redef(const std::string &filename) { - eam_pio_redef_c2f(filename.c_str()); -} -/* ----------------------------------------------------------------- */ -template<> -void grid_read_data_array(const std::string &filename, const std::string &varname, - const int time_index, int *hbuf, const int buf_size) { - grid_read_data_array_c2f_int(filename.c_str(),varname.c_str(),time_index,hbuf,buf_size); -} -template<> -void grid_read_data_array(const std::string &filename, const std::string &varname, - const int time_index, float *hbuf, const int buf_size) { - grid_read_data_array_c2f_float(filename.c_str(),varname.c_str(),time_index,hbuf,buf_size); -} -template<> -void grid_read_data_array(const std::string &filename, const std::string &varname, - const int time_index, double *hbuf, const int buf_size) { - grid_read_data_array_c2f_double(filename.c_str(),varname.c_str(),time_index,hbuf,buf_size); + +// ========================== READ/WRITE ETI ========================== // + +template void read_var (const std::string&, const std::string&, int*, const int); +template void read_var (const std::string&, const std::string&, long long*, const int); +template void read_var (const std::string&, const std::string&, float*, const int); +template void read_var (const std::string&, const std::string&, double*, const int); +template void read_var (const std::string&, const std::string&, char*, const int); + +template void write_var (const std::string&, const std::string&, const int*, const int*); +template void write_var (const std::string&, const std::string&, const long long*, const long long*); +template void write_var (const std::string&, const std::string&, const float*, const float*); +template void write_var (const std::string&, const std::string&, const double*, const double*); +template void write_var (const std::string&, const std::string&, const char*, const char*); + +// =============== Attributes operations ================== // + +bool has_global_attribute (const std::string& filename, const std::string& attname) +{ + return has_attribute(filename,"GLOBAL",attname); } -/* ----------------------------------------------------------------- */ -template<> -void grid_write_data_array(const std::string &filename, const std::string &varname, const int* hbuf, const int buf_size) { - grid_write_data_array_c2f_int(filename.c_str(),varname.c_str(),hbuf,buf_size); + +bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname) +{ + register_file(filename,Read); + const int ncid = impl::get_file(filename,"scorpio::has_attribute").ncid; + + // Get var id + int varid; + if (varname=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + const auto& var = impl::get_var(filename,varname,"scorpio::has_attribute"); + varid = var.ncid; + } + + // Get att id + int attid; + int err = PIOc_inq_attid(ncid,varid,attname.c_str(),&attid); + if (err==PIO_ENOTATT) { + return false; + } + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while retrieving attribute id.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - attname : " + attname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + + release_file (filename); + + return true; } -template<> -void grid_write_data_array(const std::string &filename, const std::string &varname, const float* hbuf, const int buf_size) { - grid_write_data_array_c2f_float(filename.c_str(),varname.c_str(),hbuf,buf_size); + +template +T get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname) +{ + const auto& f = impl::get_file (filename,"scorpio::set_any_attribute"); + + int varid; + if (varname=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + varid = impl::get_var(filename,varname,"scorpio::set_any_attribute").ncid; + } + + T val; + int err = PIOc_get_att(f.ncid,varid,attname.c_str(),reinterpret_cast(&val)); + check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); + + return val; } + +// Explicit instantiation +template int get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname); +template std::int64_t get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname); +template float get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname); +template double get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname); + +// Full specialization for strings template<> -void grid_write_data_array(const std::string &filename, const std::string &varname, const double* hbuf, const int buf_size) { - grid_write_data_array_c2f_double(filename.c_str(),varname.c_str(),hbuf,buf_size); -} -/* ----------------------------------------------------------------- */ -void write_timestamp (const std::string& filename, const std::string& ts_name, - const util::TimeStamp& ts, const bool write_nsteps) +std::string get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname) { - set_attribute(filename,ts_name,ts.to_string()); - if (write_nsteps) { - set_attribute(filename,ts_name+"_nsteps",ts.get_num_steps()); + const auto& f = impl::get_file (filename,"scorpio::set_any_attribute"); + + int varid; + if (varname=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + varid = impl::get_var(filename,varname,"scorpio::set_any_attribute").ncid; } + + int err; + PIO_Offset len; + err = PIOc_inq_attlen(f.ncid,varid,attname.c_str(),&len); + check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","inq_attlen"); + + std::string val(len,'\0'); + + err = PIOc_get_att(f.ncid,varid,attname.c_str(),val.data()); + check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); + + return val; } -/* ----------------------------------------------------------------- */ -util::TimeStamp read_timestamp (const std::string& filename, - const std::string& ts_name, - const bool read_nsteps) + +template +void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const T& att) { - auto ts = util::str_to_time_stamp(get_attribute(filename,ts_name)); - if (read_nsteps and has_attribute(filename,ts_name+"_nsteps")) { - ts.set_num_steps(get_attribute(filename,ts_name+"_nsteps")); + const auto& f = impl::get_file (filename,"scorpio::set_any_attribute"); + + int varid; + if (varname=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + varid = impl::get_var(filename,varname,"scorpio::set_any_attribute").ncid; + } + + // If the file was not in define mode, we must call enddef at the end + const bool needs_redef = f.enddef; + if (needs_redef) { + redef(filename); + } + + int err = PIOc_put_att(f.ncid,varid,attname.c_str(),nctype(),nclen(att),ncdata(att)); + check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); + + if (needs_redef) { + enddef(filename); } - return ts; } -/* ----------------------------------------------------------------- */ + +// Explicit instantiation +template void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const int& att); +template void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const std::int64_t& att); +template void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const float& att); +template void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const double& att); +template void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const std::string& att); + } // namespace scorpio } // namespace scream diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index c5ae69ea6c68..1adfc630681c 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -1,168 +1,193 @@ #ifndef SCREAM_SCORPIO_INTERFACE_HPP #define SCREAM_SCORPIO_INTERFACE_HPP -#include "share/field/field_tag.hpp" -#include "share/scream_types.hpp" -#include "share/util/scream_time_stamp.hpp" +#include "scream_scorpio_types.hpp" -#include "ekat/mpi/ekat_comm.hpp" -#include "ekat/util/ekat_string_utils.hpp" +#include +#include #include -/* C++/F90 bridge to F90 SCORPIO routines */ +/* + * This file contains interfaces to scorpio C library routines + * + * There are two reasons for these interfaces: + * + * - allow better context when exceptions/errors occour: throwing exceptions + * can even allow host code to use try-catch blocks to figure out what works. + * Moreover, the user does not need to continuously check return codes, since + * we throw any time scorpio returns something different from PIO_NOERR. + * - allow using simpler interfaces: by using string/vector and templates, + * the host code can have more compact interfaces. + */ namespace scream { namespace scorpio { - using offset_t = std::int64_t; - - // WARNING: these values must match the ones of file_purpose_in and file_purpose_out - // in the scream_scorpio_interface F90 module - enum FileMode { - Read = 1, - Append = 2, - Write = 4 - }; - - // I/O types supported - enum IOType { - // Default I/O type is used to let the code choose I/O type as needed (via CIME) - DefaultIOType = 0, - NetCDF, - PnetCDF, - Adios, - Hdf5 - }; - - inline int str2iotype(const std::string &str) - { - if(str == "default"){ - return static_cast(IOType::DefaultIOType); - } - else if(str == "netcdf"){ - return static_cast(IOType::NetCDF); - } - else if(str == "pnetcdf"){ - return static_cast(IOType::PnetCDF); - } - else if(str == "adios"){ - return static_cast(IOType::Adios); - } - else if(str == "hdf5"){ - return static_cast(IOType::Hdf5); - } - else{ - return static_cast(IOType::DefaultIOType); - } - } - - inline std::string iotype2str(int iotype) - { - switch(iotype){ - case static_cast(IOType::DefaultIOType): return "default"; - case static_cast(IOType::NetCDF): return "netcdf"; - case static_cast(IOType::PnetCDF): return "pnetcdf"; - case static_cast(IOType::Adios): return "adios"; - case static_cast(IOType::Hdf5): return "hdf5"; - default: return "default"; - } - } - - /* All scorpio usage requires that the pio_subsystem is initialized. Happens only once per simulation */ - void eam_init_pio_subsystem(const ekat::Comm& comm); - void eam_init_pio_subsystem(const int mpicom, const int atm_id = 0); - /* Cleanup scorpio with pio_finalize */ - void eam_pio_finalize(); - /* Close a file currently open in scorpio */ - void eam_pio_closefile(const std::string& filename); - void eam_flush_file(const std::string& filename); - /* Register a new file to be used for input/output with the scorpio module */ - void register_file(const std::string& filename, const FileMode mode, int iotype = IOType::DefaultIOType); - /* Sets the IO decompostion for all variables in a particular filename. Required after all variables have been registered. Called once per file. */ - int get_dimlen(const std::string& filename, const std::string& dimname); - bool has_dim(const std::string& filename, const std::string& dimname); - bool has_variable (const std::string& filename, const std::string& varname); - bool has_attribute (const std::string& filename, const std::string& attname); - bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname); - void set_decomp(const std::string& filename); - /* Sets the degrees-of-freedom for a particular variable in a particular file. Called once for each variable, for each file. */ - void set_dof(const std::string &filename, const std::string &varname, const Int dof_len, const offset_t* x_dof); - /* Register a dimension coordinate with a file. Called during the file setup. */ - void register_dimension(const std::string& filename,const std::string& shortname, const std::string& longname, const int length, const bool partitioned); - /* Register a variable with a file. Called during the file setup, for an output stream. */ - void register_variable(const std::string& filename, const std::string& shortname, const std::string& longname, - const std::string& units, const std::vector& var_dimensions, - const std::string& dtype, const std::string& nc_dtype, const std::string& pio_decomp_tag); - void register_variable(const std::string& filename, const std::string& shortname, const std::string& longname, - const std::vector& var_dimensions, - const std::string& dtype, const std::string& pio_decomp_tag); - void set_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, const std::string& meta_val); - void set_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, const float meta_val); - void set_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, const double meta_val); - void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, float& meta_val); - void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, double& meta_val); - void get_variable_metadata (const std::string& filename, const std::string& varname, const std::string& meta_name, std::string& meta_val); - /* Register a variable with a file. Called during the file setup, for an input stream. */ - ekat::any get_any_attribute (const std::string& filename, const std::string& att_name); - ekat::any get_any_attribute (const std::string& filename, const std::string& var_name, const std::string& att_name); - void set_any_attribute (const std::string& filename, const std::string& att_name, const ekat::any& att); - /* End the definition phase for a scorpio file. Last thing called after all dimensions, variables, dof's and decomps have been set. Called once per file. - * Mandatory before writing or reading can happend on file. */ - void eam_pio_enddef(const std::string &filename); - void eam_pio_redef(const std::string &filename); - /* Called each timestep to update the timesnap for the last written output. */ - void pio_update_time(const std::string &filename, const double time); - - // Read data for a specific variable from a specific file. To read data that - // isn't associated with a time index, or to read data at the most recent - // time, set time_index to -1. Otherwise use the proper zero-based time index. - template - void grid_read_data_array (const std::string &filename, const std::string &varname, - const int time_index, T* hbuf, const int buf_size); - /* Write data for a specific variable to a specific file. */ - template - void grid_write_data_array(const std::string &filename, const std::string &varname, - const T* hbuf, const int buf_size); - - template - T get_attribute (const std::string& filename, const std::string& att_name) - { - auto att = get_any_attribute(filename,att_name); - return ekat::any_cast(att); - } - - template - void set_attribute (const std::string& filename, const std::string& att_name, const T& att) - { - ekat::any a(att); - set_any_attribute(filename,att_name,a); - } - - // Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file - void write_timestamp (const std::string& filename, const std::string& ts_name, - const util::TimeStamp& ts, const bool write_nsteps = false); - util::TimeStamp read_timestamp (const std::string& filename, - const std::string& ts_name, - const bool read_nsteps = false); - -extern "C" { - /* Query whether the pio subsystem is inited or not */ - bool is_eam_pio_subsystem_inited(); - /* Checks if a file is already open, with the given mode */ - int get_file_ncid_c2f(const char*&& filename); - // If mode<0, then simply checks if file is open, regardless of mode - bool is_file_open_c2f(const char*&& filename, const int& mode); - /* Query a netCDF file for the time variable */ - bool is_enddef_c2f(const char*&& filename); - double read_time_at_index_c2f(const char*&& filename, const int& time_index); - double read_curr_time_c2f(const char*&& filename); - /* Query a netCDF file for the metadata associated w/ a variable */ - float get_variable_metadata_float_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); - double get_variable_metadata_double_c2f (const char*&& filename, const char*&& varname, const char*&& meta_name); -} // extern "C" +inline std::string default_time_name () { return "time"; } + +// Handles aliases for same type: +// single -> float +// float -> float +// double -> double +// real -> float or double (depending on SCREAM_DOUBLE_PRECISION) +std::string refine_dtype (const std::string& dtype); + +// =================== Global operations ================= // + +void init_subsystem(const ekat::Comm& comm, const int atm_id = 0); +bool is_subsystem_inited (); +void finalize_subsystem (); + +// =================== File operations ================= // + +// Opens a file, returns const handle to it (useful for Read mode, to get dims/vars) +void register_file(const std::string& filename, const FileMode mode, const IOType iotype = IOType::DefaultIOType); + +// Release a file (if in Write mode, sync and close the file); +void release_file (const std::string& filename); + +// Check if file is open. If mode!=Unset, also checks that it's open with given mode +bool is_file_open (const std::string& filename, const FileMode mode = Unset); + +// Force a flush to file (for Write mode only) +void flush_file (const std::string &filename); + +// Reopen/ends the definition phase +void redef (const std::string &filename); +void enddef (const std::string &filename); + +// =================== Dimensions operations ======================= // + +// Define dim on output file (cannot call on Read/Append files) +void define_dim (const std::string& filename, const std::string& dimname, const int length); + +// Check that the given dimension is in the file. If length>0, also check that the length is as expected. +bool has_dim (const std::string& filename, + const std::string& dimname, + const int length = -1); + +int get_dimlen (const std::string& filename, const std::string& dimname); +int get_dimlen_local (const std::string& filename, const std::string& dimname); + +int get_time_len (const std::string& filename); +std::string get_time_name (const std::string& filename); + +// =================== Decompositions operations ==================== // + +// Create a decomposition along a particular dimension +// Notes: +// - we declare a decomposition along a single dimension. +// - set_dim_decomp requires *offsets* in the global array, not global indices +// (for 0-based indices, they're the same, but for 1-based indices they're not) +// - the second version is a shortcut for contiguous decompositions. Notice that NO +// check is performed to ensure that the decomposition covers the entire dimension, +// or that there are no overlaps (in fact, one *may* need overlap, in certain reads). +// - the third version is a shortcut of the second, where we compute start/count based +// on a linear decomposition of the dimension along all ranks in the IO comm stored +// in the ScorpioInstance. The return value is the local length of the dimension +// - if allow_reset=true, we simply reset the decomposition (if present). +// - if allow_reset=false, if a decomposition for this dim is already set, we error out + +void set_dim_decomp (const std::string& filename, + const std::string& dimname, + const std::vector& my_offsets, + const bool allow_reset = false); + +void set_dim_decomp (const std::string& filename, + const std::string& dimname, + const offset_t start, const offset_t count, + const bool allow_reset = false); + +void set_dim_decomp (const std::string& filename, + const std::string& dimname, + const bool allow_reset = false); + +// ================== Variable operations ================== // + +// Define var on output file (cannot call on Read/Append files) +void define_var (const std::string& filename, const std::string& varname, + const std::string& units, const std::vector& dimensions, + const std::string& dtype, const std::string& nc_dtype, + const bool time_dependent = false); + +// Shortcut when units are not used, and dtype==nc_dtype +void define_var (const std::string& filename, const std::string& varname, + const std::vector& dimensions, + const std::string& dtype, + const bool time_dependent = false); + +// This is useful when reading data sets. E.g., if the pio file is storing +// a var as float, but we need to read it as double, we need to call this. +void change_var_dtype (const std::string& filename, + const std::string& varname, + const std::string& dtype); + +// Check that the given variable is in the file. +bool has_var (const std::string& filename, const std::string& varname); + +// Allows to easily query var metadata, such as dims, units, data type,... +const PIOVar& get_var (const std::string& filename, + const std::string& varname); + +// Defines both a time dimension and a time variable +void define_time (const std::string& filename, const std::string& units, + const std::string& time_name = default_time_name()); + +// Update value of time variable, increasing time dim length +void update_time(const std::string &filename, const double time); + +// Retrieves the time variable value(s) +double get_time (const std::string& filename, const int time_index = -1); +std::vector get_all_times (const std::string& filename); + +// Read variable into user provided buffer. +// If time dim is present, read given time slice (time_index=-1 means "read last record). +// If time dim is not present and time_index>=0, it is interpreted as the index of the +// first dimension (which is not unlimited). +// NOTE: ETI in the cpp file for int, float, double. +template +void read_var (const std::string &filename, const std::string &varname, T* buf, const int time_index = -1); + +// Write data from user provided buffer into the requested variable +// NOTE: ETI in the cpp file for int, float, double. +template +void write_var (const std::string &filename, const std::string &varname, const T* buf, const T* fillValue = nullptr); + +// =============== Attributes operations ================== // + +// To specify GLOBAL attributes, pass "GLOBAL" as varname +// NOTE: set/get_attribute are implemented in the cpp file, +// with Explicit Instantiation only for: +// int, std::int64_t, float, double, std::string + +bool has_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname); + +template +T get_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname); + +template +void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const T& att); + +// Shortcut, to allow calling set_attribute with compile-time strings, like so +// set_attribute(my_file,my_var,my_att_name,"my_value"); +template +inline void set_attribute (const std::string& filename, + const std::string& varname, + const std::string& attname, + const char (&att)[N]) +{ + set_attribute(filename,varname,attname,att); +} } // namespace scorpio } // namespace scream -#endif // define SCREAM_SCORPIO_INTERFACE_HPP +#endif // define SCREAM_SCORPIO_INTERFACE_HPP diff --git a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 deleted file mode 100644 index 2f1263beef7d..000000000000 --- a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 +++ /dev/null @@ -1,520 +0,0 @@ -module scream_scorpio_interface_iso_c2f - use iso_c_binding, only: c_int, c_double, c_float, c_bool, c_ptr - implicit none - -! -! This file contains bridges from scream c++ to shoc fortran. -! -contains -!=====================================================================! - subroutine eam_init_pio_subsystem_c2f(mpicom,compid) bind(c) - use scream_scorpio_interface, only : eam_init_pio_subsystem - integer(kind=c_int), value, intent(in) :: mpicom,compid - - call eam_init_pio_subsystem(mpicom,compid) - end subroutine eam_init_pio_subsystem_c2f -!=====================================================================! - subroutine eam_pio_finalize_c2f() bind(c) - use scream_scorpio_interface, only : eam_pio_finalize - - call eam_pio_finalize() - end subroutine eam_pio_finalize_c2f -!=====================================================================! - function get_file_ncid_c2f(filename_in) result(ncid) bind(c) - use scream_scorpio_interface, only : lookup_pio_atm_file, pio_atm_file_t - type(c_ptr), intent(in) :: filename_in - - type(pio_atm_file_t), pointer :: atm_file - character(len=256) :: filename - integer(kind=c_int) :: ncid - logical :: found - - call convert_c_string(filename_in,filename) - call lookup_pio_atm_file(filename,atm_file,found) - if (found) then - ncid = int(atm_file%pioFileDesc%fh,kind=c_int) - else - ncid = -1 - endif - end function get_file_ncid_c2f -!=====================================================================! - function get_file_mode_c2f(filename_in) result(mode) bind(c) - use scream_scorpio_interface, only : lookup_pio_atm_file, pio_atm_file_t - - type(c_ptr), intent(in) :: filename_in - - type(pio_atm_file_t), pointer :: atm_file - character(len=256) :: filename - integer(kind=c_int) :: mode - logical :: found - - call convert_c_string(filename_in,filename) - call lookup_pio_atm_file(filename,atm_file,found) - if (found) then - mode = atm_file%purpose - else - mode = 0 - endif - end function get_file_mode_c2f - function is_file_open_c2f(filename_in,purpose) result(res) bind(c) - use scream_scorpio_interface, only : lookup_pio_atm_file, pio_atm_file_t - - type(c_ptr), intent(in) :: filename_in - integer(kind=c_int), intent(in) :: purpose - - type(pio_atm_file_t), pointer :: atm_file - character(len=256) :: filename - logical (kind=c_bool) :: res - logical :: found - - call convert_c_string(filename_in,filename) - call lookup_pio_atm_file(filename,atm_file,found) - if (found) then - res = LOGICAL(purpose .lt. 0 .or. atm_file%purpose .eq. purpose,kind=c_bool) - else - res = .false. - endif - end function is_file_open_c2f -!=====================================================================! - subroutine register_file_c2f(filename_in,purpose,iotype) bind(c) - use scream_scorpio_interface, only : register_file - type(c_ptr), intent(in) :: filename_in - integer(kind=c_int), intent(in) :: purpose - integer(kind=c_int), intent(in) :: iotype - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call register_file(trim(filename),purpose,iotype) - - end subroutine register_file_c2f -!=====================================================================! - subroutine set_decomp_c2f(filename_in) bind(c) - use scream_scorpio_interface, only : set_decomp - type(c_ptr), intent(in) :: filename_in - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call set_decomp(trim(filename)) - end subroutine set_decomp_c2f -!=====================================================================! - subroutine set_dof_c2f(filename_in,varname_in,dof_len,dof_vec) bind(c) - use scream_scorpio_interface, only : set_dof, pio_offset_kind - use iso_c_binding, only: c_int64_t - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), value, intent(in) :: dof_len - integer(kind=c_int64_t), intent(in), dimension(dof_len) :: dof_vec - - character(len=256) :: filename - character(len=256) :: varname - integer :: ii - integer(kind=pio_offset_kind), allocatable :: dof_vec_f90(:) - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - ! Need to add 1 to the dof_vec because C++ starts indices at 0 not 1: - allocate(dof_vec_f90(dof_len)) - do ii = 1,dof_len - dof_vec_f90(ii) = dof_vec(ii) + 1 - end do - call set_dof(trim(filename),trim(varname),dof_len,dof_vec_f90) - deallocate(dof_vec_f90) - end subroutine set_dof_c2f -!=====================================================================! - subroutine eam_pio_closefile_c2f(filename_in) bind(c) - use scream_scorpio_interface, only : eam_pio_closefile - type(c_ptr), intent(in) :: filename_in - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call eam_pio_closefile(trim(filename)) - - end subroutine eam_pio_closefile_c2f -!=====================================================================! - subroutine eam_pio_flush_file_c2f(filename_in) bind(c) - use scream_scorpio_interface, only : eam_pio_flush_file - type(c_ptr), intent(in) :: filename_in - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call eam_pio_flush_file(trim(filename)) - - end subroutine eam_pio_flush_file_c2f -!=====================================================================! - subroutine pio_update_time_c2f(filename_in,time) bind(c) - use scream_scorpio_interface, only : eam_update_time - type(c_ptr), intent(in) :: filename_in - real(kind=c_double), value, intent(in) :: time - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call eam_update_time(trim(filename),time) - - end subroutine pio_update_time_c2f -!=====================================================================! - subroutine register_variable_c2f(filename_in, shortname_in, longname_in, & - units_in, numdims, var_dimensions_in, & - dtype, nc_dtype, pio_decomp_tag_in) bind(c) - use scream_scorpio_interface, only : register_variable - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: shortname_in - type(c_ptr), intent(in) :: longname_in - type(c_ptr), intent(in) :: units_in - integer(kind=c_int), value, intent(in) :: numdims - type(c_ptr), intent(in) :: var_dimensions_in(numdims) - integer(kind=c_int), value, intent(in) :: dtype, nc_dtype - type(c_ptr), intent(in) :: pio_decomp_tag_in - - character(len=256) :: filename - character(len=256) :: shortname - character(len=256) :: longname - character(len=256) :: units - character(len=256) :: var_dimensions(numdims) - character(len=256) :: pio_decomp_tag - integer :: ii - - call convert_c_string(filename_in,filename) - call convert_c_string(shortname_in,shortname) - call convert_c_string(longname_in,longname) - call convert_c_string(units_in,units) - call convert_c_string(pio_decomp_tag_in,pio_decomp_tag) - do ii = 1,numdims - call convert_c_string(var_dimensions_in(ii), var_dimensions(ii)) - end do - - call register_variable(filename,shortname,longname,units,numdims,var_dimensions,dtype,nc_dtype,pio_decomp_tag) - - end subroutine register_variable_c2f -!=====================================================================! - subroutine set_variable_metadata_char_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) - use scream_scorpio_interface, only : set_variable_metadata_char - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - type(c_ptr), intent(in) :: metaname_in - type(c_ptr), intent(in) :: metaval_in - - character(len=256) :: filename - character(len=256) :: varname - character(len=256) :: metaname - character(len=256) :: metaval - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call convert_c_string(metaname_in,metaname) - call convert_c_string(metaval_in,metaval) - - call set_variable_metadata_char(filename,varname,metaname,metaval) - - end subroutine set_variable_metadata_char_c2f -!=====================================================================! - subroutine set_variable_metadata_float_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) - use scream_scorpio_interface, only : set_variable_metadata_float - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - type(c_ptr), intent(in) :: metaname_in - real(kind=c_float), value, intent(in) :: metaval_in - - character(len=256) :: filename - character(len=256) :: varname - character(len=256) :: metaname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call convert_c_string(metaname_in,metaname) - - call set_variable_metadata_float(filename,varname,metaname,metaval_in) - - end subroutine set_variable_metadata_float_c2f -!=====================================================================! - subroutine set_variable_metadata_double_c2f(filename_in, varname_in, metaname_in, metaval_in) bind(c) - use scream_scorpio_interface, only : set_variable_metadata_double - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - type(c_ptr), intent(in) :: metaname_in - real(kind=c_double), value, intent(in) :: metaval_in - - character(len=256) :: filename - character(len=256) :: varname - character(len=256) :: metaname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call convert_c_string(metaname_in,metaname) - - call set_variable_metadata_double(filename,varname,metaname,metaval_in) - - end subroutine set_variable_metadata_double_c2f -!=====================================================================! - function get_variable_metadata_float_c2f(filename_in, varname_in, metaname_in) result(metaval_out) bind(c) - use scream_scorpio_interface, only : get_variable_metadata_float - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - type(c_ptr), intent(in) :: metaname_in - real(kind=c_float) :: metaval_out - - character(len=256) :: filename - character(len=256) :: varname - character(len=256) :: metaname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call convert_c_string(metaname_in,metaname) - - metaval_out = get_variable_metadata_float(filename,varname,metaname) - - end function get_variable_metadata_float_c2f -!=====================================================================! - function get_variable_metadata_double_c2f(filename_in, varname_in, metaname_in) result(metaval_out) bind(c) - use scream_scorpio_interface, only : get_variable_metadata_double - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - type(c_ptr), intent(in) :: metaname_in - real(kind=c_double) :: metaval_out - - character(len=256) :: filename - character(len=256) :: varname - character(len=256) :: metaname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call convert_c_string(metaname_in,metaname) - - metaval_out = get_variable_metadata_double(filename,varname,metaname) - - end function get_variable_metadata_double_c2f -!=====================================================================! - subroutine get_variable_metadata_char_c2f(filename_in, varname_in, metaname_in, metaval_out) bind(c) - use scream_scorpio_interface, only : get_variable_metadata_char - use iso_c_binding, only: C_NULL_CHAR, c_char, c_f_pointer - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - type(c_ptr), intent(in) :: metaname_in - type(c_ptr), intent(in) :: metaval_out - - character(len=256) :: filename - character(len=256) :: varname - character(len=256) :: metaname - character(len=256) :: metaval - character(len=256, kind=c_char), pointer :: temp_string - integer :: slen - - call c_f_pointer(metaval_out,temp_string) - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call convert_c_string(metaname_in,metaname) - - metaval = get_variable_metadata_char(filename,varname,metaname) - - slen = len(trim(metaval)) - ! If string is 255 or less, add terminating char. If not, it's still - ! ok (the C++ string will have length=max_length=256) - if (slen .le. 255) then - temp_string = trim(metaval) // C_NULL_CHAR - else - temp_string = metaval - endif - end subroutine get_variable_metadata_char_c2f -!=====================================================================! - subroutine register_dimension_c2f(filename_in, shortname_in, longname_in, length, partitioned) bind(c) - use scream_scorpio_interface, only : register_dimension - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: shortname_in - type(c_ptr), intent(in) :: longname_in - integer(kind=c_int), value, intent(in) :: length - logical(kind=c_bool), value, intent(in) :: partitioned - - character(len=256) :: filename - character(len=256) :: shortname - character(len=256) :: longname - - call convert_c_string(filename_in,filename) - call convert_c_string(shortname_in,shortname) - call convert_c_string(longname_in,longname) - call register_dimension(filename,shortname,longname,length,LOGICAL(partitioned)) - - end subroutine register_dimension_c2f -!=====================================================================! - function read_curr_time_c2f(filename_in) result(val) bind(c) - use scream_scorpio_interface, only : read_time_at_index - type(c_ptr), intent(in) :: filename_in - real(kind=c_double) :: val - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - val = read_time_at_index(filename) - - end function read_curr_time_c2f -!=====================================================================! - function read_time_at_index_c2f(filename_in,time_index) result(val) bind(c) - use scream_scorpio_interface, only : read_time_at_index - type(c_ptr), intent(in) :: filename_in - integer(kind=c_int), intent(in) :: time_index ! zero-based - real(kind=c_double) :: val - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - val = read_time_at_index(filename,time_index) - - end function read_time_at_index_c2f -!=====================================================================! - function is_enddef_c2f(filename_in) bind(c) result(enddef) - use scream_scorpio_interface, only : lookup_pio_atm_file, pio_atm_file_t - type(c_ptr), intent(in) :: filename_in - - type(pio_atm_file_t), pointer :: atm_file - character(len=256) :: filename - logical (kind=c_bool) :: enddef - logical :: found - - call convert_c_string(filename_in,filename) - call lookup_pio_atm_file(filename,atm_file,found) - if (found) then - enddef = LOGICAL(atm_file%is_enddef, kind=c_bool) - endif - end function is_enddef_c2f -!=====================================================================! - subroutine eam_pio_enddef_c2f(filename_in) bind(c) - use scream_scorpio_interface, only : eam_pio_enddef - type(c_ptr), intent(in) :: filename_in - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call eam_pio_enddef(filename) - end subroutine eam_pio_enddef_c2f -!=====================================================================! - subroutine eam_pio_redef_c2f(filename_in) bind(c) - use scream_scorpio_interface, only : eam_pio_redef - type(c_ptr), intent(in) :: filename_in - - character(len=256) :: filename - - call convert_c_string(filename_in,filename) - call eam_pio_redef(filename) - end subroutine eam_pio_redef_c2f -!=====================================================================! - subroutine convert_c_string(c_string_ptr,f_string) - use iso_c_binding, only: c_f_pointer, C_NULL_CHAR - ! Purpose: To convert a c_string pointer to the proper fortran string format. - type(c_ptr), intent(in) :: c_string_ptr - character(len=256), intent(out) :: f_string - character(len=256), pointer :: temp_string - integer :: str_len - - call c_f_pointer(c_string_ptr,temp_string) - str_len = index(temp_string, C_NULL_CHAR) - 1 - f_string = trim(temp_string(1:str_len)) - - end subroutine convert_c_string -!=====================================================================! - subroutine grid_write_data_array_c2f_int(filename_in,varname_in,buf,buf_size) bind(c) - use scream_scorpio_interface, only: grid_write_data_array - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), intent(in), value :: buf_size - integer(kind=c_int), intent(in) :: buf(buf_size) - - character(len=256) :: filename - character(len=256) :: varname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call grid_write_data_array(filename,varname,buf,buf_size) - - end subroutine grid_write_data_array_c2f_int - subroutine grid_write_data_array_c2f_float(filename_in,varname_in,buf,buf_size) bind(c) - use scream_scorpio_interface, only: grid_write_data_array - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), intent(in), value :: buf_size - real(kind=c_float), intent(in) :: buf(buf_size) - - character(len=256) :: filename - character(len=256) :: varname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call grid_write_data_array(filename,varname,buf,buf_size) - - end subroutine grid_write_data_array_c2f_float - subroutine grid_write_data_array_c2f_double(filename_in,varname_in,buf,buf_size) bind(c) - use scream_scorpio_interface, only: grid_write_data_array - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), intent(in), value :: buf_size - real(kind=c_double), intent(in) :: buf(buf_size) - - character(len=256) :: filename - character(len=256) :: varname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call grid_write_data_array(filename,varname,buf,buf_size) - - end subroutine grid_write_data_array_c2f_double -!=====================================================================! - subroutine grid_read_data_array_c2f_int(filename_in,varname_in,time_index,buf,buf_size) bind(c) - use scream_scorpio_interface, only: grid_read_data_array - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), value, intent(in) :: time_index ! zero-based - integer(kind=c_int), intent(in), value :: buf_size - integer(kind=c_int), intent(out) :: buf(buf_size) - - character(len=256) :: filename - character(len=256) :: varname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call grid_read_data_array(filename,varname,buf,buf_size,time_index+1) - - end subroutine grid_read_data_array_c2f_int -!=====================================================================! - subroutine grid_read_data_array_c2f_float(filename_in,varname_in,time_index,buf,buf_size) bind(c) - use scream_scorpio_interface, only: grid_read_data_array - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), value, intent(in) :: time_index ! zero-based - integer(kind=c_int), intent(in), value :: buf_size - real(kind=c_float), intent(out) :: buf(buf_size) - - character(len=256) :: filename - character(len=256) :: varname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call grid_read_data_array(filename,varname,buf,buf_size,time_index+1) - - end subroutine grid_read_data_array_c2f_float -!=====================================================================! - subroutine grid_read_data_array_c2f_double(filename_in,varname_in,time_index,buf,buf_size) bind(c) - use scream_scorpio_interface, only: grid_read_data_array - - type(c_ptr), intent(in) :: filename_in - type(c_ptr), intent(in) :: varname_in - integer(kind=c_int), value, intent(in) :: time_index ! zero-based - integer(kind=c_int), intent(in), value :: buf_size - real(kind=c_double), intent(out) :: buf(buf_size) - - character(len=256) :: filename - character(len=256) :: varname - - call convert_c_string(filename_in,filename) - call convert_c_string(varname_in,varname) - call grid_read_data_array(filename,varname,buf,buf_size,time_index+1) - - end subroutine grid_read_data_array_c2f_double -!=====================================================================! -end module scream_scorpio_interface_iso_c2f diff --git a/components/eamxx/src/share/io/scream_scorpio_types.cpp b/components/eamxx/src/share/io/scream_scorpio_types.cpp new file mode 100644 index 000000000000..71c3325c465b --- /dev/null +++ b/components/eamxx/src/share/io/scream_scorpio_types.cpp @@ -0,0 +1,59 @@ +#include "scream_scorpio_types.hpp" + +#include + +namespace scream { +namespace scorpio { + +std::string e2str (const FileMode mode) +{ + auto mode_int = static_cast::type>(mode); + std::string s; + switch (mode) { + case Unset: s = "UNSET"; break; + case Read: s = "READ"; break; + case Write: s = "WRITE"; break; + case Append: s = "APPEND"; break; + default: + EKAT_ERROR_MSG ( + "Error! Unsupported/unrecognized FileMode value.\n" + " - value: " + std::to_string(mode_int) + "\n"); + } + return s; +} + +IOType str2iotype(const std::string &str) +{ + if(str == "default"){ + return IOType::DefaultIOType; + } else if(str == "netcdf") { + return IOType::NetCDF; + } else if(str == "pnetcdf") { + return IOType::PnetCDF; + } else if(str == "adios") { + return IOType::Adios; + } else if(str == "hdf5") { + return IOType::Hdf5; + } else { + return IOType::Invalid; + } +} + +std::string iotype2str(const IOType iotype) +{ + std::string s; + switch(iotype){ + case IOType::DefaultIOType: s = "default"; break; + case IOType::NetCDF: s = "netcdf"; break; + case IOType::PnetCDF: s = "pnetcdf"; break; + case IOType::Adios: s = "adios"; break; + case IOType::Hdf5: s = "hdf5"; break; + case IOType::Invalid: s = "invalid"; break; + default: + EKAT_ERROR_MSG ("Unrecognized iotype.\n"); + } + return s; +} + +} // namespace scorpio +} // namespace scream diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp new file mode 100644 index 000000000000..4781f7ad62ef --- /dev/null +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -0,0 +1,154 @@ +#ifndef SCREAM_SCORPIO_TYPES_HPP +#define SCREAM_SCORPIO_TYPES_HPP + +#include +#include +#include +#include + +namespace scream { +namespace scorpio { + +// Enum denoting how we open a file +enum FileMode { + Unset = 0, + Read = 1, + Write = 2, + Append = Read | Write +}; +std::string e2str (const FileMode mode); + +// I/O types supported +enum IOType { + // Default I/O type is used to let the code choose I/O type as needed (via CIME) + DefaultIOType = 0, + NetCDF, + PnetCDF, + Adios, + Hdf5, + Invalid +}; + +IOType str2iotype(const std::string &str); +std::string iotype2str(const IOType iotype); + +// The type used by PIOc for offsets +using offset_t = std::int64_t; + +/* + * The following PIOxyz types each represent an entity that + * is associated in scorpio with an id. For each of them, we add some + * additional metadata, to avoid having to call PIOc_inq_xyz every time + * we need such information. + * While these types are defined in this public header, the customers + * of scream_io will never be able to get any object of these types out + * of the internal database. In particular, all data is stored in a + * ScorpioSession singleton class, whose declaration is hidden inside + * scream_scorpio_interface.cpp. + */ + +// The basic common data of any PIO entity +struct PIOEntity { + int ncid = -1; // PIO access all data via their id + std::string name; // In EAMxx, we prefer to use a string name than a number id +}; + +// An entity that is linked to a specific file +// This allows an entity to retrieve its file +struct PIOFileEntity : public PIOEntity { + int fid = -1; // Allow retrieving file from the entity +}; + +// struct PIOAtt : public PIOFileEntity { +// int varid = -1; +// ekat::any val; +// }; + +// A dimension +struct PIODim : public PIOFileEntity { + int length = -1; + bool unlimited = false; + + // In case we decompose the dimension, this will store + // the owned offsets on this rank + // NOTE: use a pointer, so we can detect if a decomposition already + // existed or not when we set one. + std::shared_ptr> offsets; +}; + +// A decomposition +// NOTE: the offsets of a PIODecomp are not the same as the offsets of +// stored in its dim. The latter are the offsets *along that dim*. +// A PIODecomp is associated with a Nd layout, which includes decomp_dim +// among its dimensions. PIODecomp::offsets are the offsets of the full +// array layout owned by this rank. Hence, there can be many PIODecomp +// all storing the same dim +struct PIODecomp : public PIOEntity { + std::vector offsets; // Owned offsets + std::shared_ptr dim; +}; + +// // A decomposition of a single dimension +// struct PIODimDecomp : public PIOEntity { +// std::shared_ptr dim; // The dim being decomposed +// std::vector offsets; // Owned offsets along decomposed dimension +// }; + +// // A decomposition of a full variable layout +// struct PIOVarDecomp : public PIOEntity { +// std::shared_ptr dim_decomp; // The decomposition along the dimension +// std::vector offsets; // Owned offsets along the full var layout +// }; + +// A variable +struct PIOVar : public PIOFileEntity { + // Note: if time_dep=true, we will add it to the list of dims passed + // to scorpio, but the time dim will not appear in this list. + std::vector> dims; + // std::map> atts; + + std::vector dim_names () const { + std::vector n; + for (auto d : dims) { + n.push_back(d->name); + } + return n; + } + + std::string dtype; + std::string nc_dtype; + std::string units; + + bool time_dep = false; + + // Extra safety measure: for time_dep vars, use this to check that + // we are not writing more slices than the current time dim length. + int num_records = 0; + + std::shared_ptr decomp; + + // Used only if a) var is not decomposed, and b) dtype!=nc_dtype + int size = -1; // Product of all dims + std::vector buf; +}; + +// A file, which is basically a container for dims and vars +struct PIOFile : public PIOEntity { + std::map> dims; + std::map> vars; + // std::map> atts; + + std::shared_ptr time_dim; + FileMode mode; + IOType iotype; + bool enddef = false; + + // We keep track of how many places are currently using this file, so that we + // can close it only when they are all done. + int num_customers = 0; +}; + +} // namespace scorpio +} // namespace scream + +#endif // define SCREAM_SCORPIO_INTERFACE_HPP diff --git a/components/eamxx/src/share/io/scream_shr_interface_c2f.F90 b/components/eamxx/src/share/io/scream_shr_interface_c2f.F90 new file mode 100644 index 000000000000..514a1478648f --- /dev/null +++ b/components/eamxx/src/share/io/scream_shr_interface_c2f.F90 @@ -0,0 +1,53 @@ +module eamxx_shr_interface_mod + use iso_c_binding, only: c_int + implicit none + +! +! This file contains bridges from EAMxx to E3SM shr module +! +contains + +!=====================================================================! + function shr_get_iosysid_c2f(atm_id) result (iosysid) bind(c) + use pio, only: iosystem_desc_t + use shr_pio_mod, only: shr_pio_getiosys + integer(kind=c_int), value, intent(in) :: atm_id + + integer (kind=c_int) :: iosysid + type(iosystem_desc_t), pointer :: iosystem + + iosystem => shr_pio_getiosys(atm_id) + iosysid = iosystem%iosysid + end function shr_get_iosysid_c2f + +!=====================================================================! + function shr_get_iotype_c2f(atm_id) result (iotype) bind(c) + use shr_pio_mod, only: shr_pio_getiotype + integer(kind=c_int), value, intent(in) :: atm_id + + integer (kind=c_int) :: iotype + + iotype = shr_pio_getiotype(atm_id) + end function shr_get_iotype_c2f + +!=====================================================================! + function shr_get_rearranger_c2f(atm_id) result (rearr) bind(c) + use shr_pio_mod, only: shr_pio_getrearranger + integer(kind=c_int), value, intent(in) :: atm_id + + integer (kind=c_int) :: rearr + + rearr = shr_pio_getrearranger(atm_id) + end function shr_get_rearranger_c2f + +!=====================================================================! + function shr_get_ioformat_c2f(atm_id) result (ioformat) bind(c) + use shr_pio_mod, only: shr_pio_getioformat + integer(kind=c_int), value, intent(in) :: atm_id + + integer (kind=c_int) :: ioformat + + ioformat = shr_pio_getioformat(atm_id) + end function shr_get_ioformat_c2f + +end module eamxx_shr_interface_mod diff --git a/components/eamxx/src/share/io/scream_shr_interface_c2f.hpp b/components/eamxx/src/share/io/scream_shr_interface_c2f.hpp new file mode 100644 index 000000000000..4e8615425157 --- /dev/null +++ b/components/eamxx/src/share/io/scream_shr_interface_c2f.hpp @@ -0,0 +1,16 @@ +#ifndef SCREAM_SHR_INTERFACE_HPP +#define SCREAM_SHR_INTERFACE_HPP + +extern "C" +{ + +#ifdef SCREAM_CIME_BUILD +int shr_get_iosysid_c2f(int atm_id); +int shr_get_iotype_c2f(int atm_id); +int shr_get_rearranger_c2f(int atm_id); +int shr_get_ioformat_c2f(int atm_id); +#endif + +} // extern "C" + +#endif // SCREAM_SHR_INTERFACE_HPP diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 94bcee1790b0..16ce2efd4376 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -5,6 +5,12 @@ include(ScreamUtils) include (BuildCprnc) BuildCprnc() +## Test scorpio interfaces +CreateUnitTest(scorpio_interface_tests "scorpio_interface_tests.cpp" + LIBS scream_scorpio_interface LABELS "io" + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) + ## Test io utils CreateUnitTest(io_utils "io_utils.cpp" LIBS scream_io LABELS io diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 378a956f8932..11905fd8480e 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -257,14 +257,12 @@ void read (const std::string& avg_type, const std::string& freq_units, } // Check that the expected metadata was appropriately set for each variable - Real fill_out; - std::string att_test; for (const auto& fn: fnames) { - scorpio::get_variable_metadata(filename,fn,"_FillValue",fill_out); - REQUIRE(fill_out==constants::DefaultFillValue().value); + auto att_fill = scorpio::get_attribute(filename,fn,"_FillValue"); + REQUIRE(att_fill==constants::DefaultFillValue().value); - scorpio::get_variable_metadata(filename,fn,"test",att_test); - REQUIRE (att_test==fn); + auto att_str = scorpio::get_attribute(filename,fn,"test"); + REQUIRE (att_str==fn); } } @@ -284,7 +282,7 @@ TEST_CASE ("io_basic") { }; ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); @@ -308,7 +306,7 @@ TEST_CASE ("io_basic") { print(" PASS\n"); } } - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index b2fe7f994db6..7daa13c1904e 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -250,7 +250,7 @@ void read (const int seed, const ekat::Comm& comm) TEST_CASE ("io_diags") { ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); // Make MyDiag available via diag factory auto& diag_factory = AtmosphereDiagnosticFactory::instance(); @@ -272,7 +272,7 @@ TEST_CASE ("io_diags") { write(seed,comm); read(seed,comm); print(" PASS\n"); - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp index ba5eb7d871d8..0901e7e7c9e2 100644 --- a/components/eamxx/src/share/io/tests/io_filled.cpp +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -247,10 +247,10 @@ void read (const std::string& avg_type, const std::string& freq_units, } // Check that the fill value gets appropriately set for each variable - Real fill_out; for (const auto& fn: fnames) { - scorpio::get_variable_metadata(filename,fn,"_FillValue",fill_out); - REQUIRE(fill_out==constants::DefaultFillValue().value); + // NOTE: use float, since default fp_precision for I/O is 'single' + auto att_fill = scorpio::get_attribute(filename,fn,"_FillValue"); + REQUIRE(att_fill==constants::DefaultFillValue().value); } } @@ -270,7 +270,7 @@ TEST_CASE ("io_filled") { }; ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); @@ -294,7 +294,7 @@ TEST_CASE ("io_filled") { print(" PASS\n"); } } - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_monthly.cpp b/components/eamxx/src/share/io/tests/io_monthly.cpp index 4cf18fff1c71..82b064211968 100644 --- a/components/eamxx/src/share/io/tests/io_monthly.cpp +++ b/components/eamxx/src/share/io/tests/io_monthly.cpp @@ -212,7 +212,7 @@ void read (const int seed, const ekat::Comm& comm) TEST_CASE ("io_monthly") { ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); @@ -224,7 +224,7 @@ TEST_CASE ("io_monthly") { if (comm.am_i_root()) { std::cout << " -> Testing output with one file per month ... PASS\n"; } - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index 585fc7b26f69..5796dbbe8b20 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -178,7 +178,7 @@ void read (const int freq, const int seed, const int ps_write, const int ps_read TEST_CASE ("io_packs") { ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); @@ -202,7 +202,7 @@ TEST_CASE ("io_packs") { print(" PASS\n"); } } - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index bfed937636e2..c579dea5c7dc 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -54,8 +54,7 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> Test Setup ...\n",io_comm); - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + scorpio::init_subsystem(io_comm); const int ncols_src = 64*io_comm.size(); const int nlevs_src = 2*packsize + 1; // Construct a timestamp @@ -81,7 +80,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> Create remap file ... \n",io_comm); const int ncols_tgt_l = ncols_src_l/2; const int ncols_tgt = ncols_src/2; - std::vector col, row, S; + std::vector col, row; + std::vector S; const Real wgt = 0.4; for (int ii=0; ii dofs_cols (ncols_src_l); - std::iota(dofs_cols.begin(),dofs_cols.end(),io_comm.rank()*ncols_src_l); // For vertical remapping we will prokject onto a set of equally // spaced pressure levels from p_top to b_bot that is nearly half // the number of source columns. - std::vector dofs_levs(nlevs_tgt); - std::iota(dofs_levs.begin(),dofs_levs.end(),0); std::vector p_tgt; for (int ii=0; ii Create remap file ... done\n",io_comm); /* @@ -296,11 +288,11 @@ TEST_CASE("io_remap_test","io_remap_test") std::string att_val; const auto& filename = vert_in.get("Filename"); for (auto& fname : fnames) { - scorpio::get_variable_metadata(filename,fname,"test",att_val); + att_val = scorpio::get_attribute(filename,fname,"test"); REQUIRE (att_val==fname); } std::string f_at_lev_name = "Y_int_at_" + std::to_string(p_ref) + "Pa"; - scorpio::get_variable_metadata(filename,f_at_lev_name,"test",att_val); + att_val = scorpio::get_attribute(filename,f_at_lev_name,"test"); REQUIRE (att_val=="Y_int"); test_input.finalize(); @@ -366,11 +358,11 @@ TEST_CASE("io_remap_test","io_remap_test") std::string att_val; const auto& filename = horiz_in.get("Filename"); for (auto& fname : fnames) { - scorpio::get_variable_metadata(filename,fname,"test",att_val); + att_val = scorpio::get_attribute(filename,fname,"test"); REQUIRE (att_val==fname); } std::string f_at_lev_name = "Y_int_at_" + std::to_string(p_ref) + "Pa"; - scorpio::get_variable_metadata(filename,f_at_lev_name,"test",att_val); + att_val = scorpio::get_attribute(filename,f_at_lev_name,"test"); REQUIRE (att_val=="Y_int"); test_input.finalize(); @@ -452,11 +444,11 @@ TEST_CASE("io_remap_test","io_remap_test") std::string att_val; const auto& filename = vh_in.get("Filename"); for (auto& fname : fnames) { - scorpio::get_variable_metadata(filename,fname,"test",att_val); + att_val = scorpio::get_attribute(filename,fname,"test"); REQUIRE (att_val==fname); } std::string f_at_lev_name = "Y_int_at_" + std::to_string(p_ref) + "Pa"; - scorpio::get_variable_metadata(filename,f_at_lev_name,"test",att_val); + att_val = scorpio::get_attribute(filename,f_at_lev_name,"test"); REQUIRE (att_val=="Y_int"); test_input.finalize(); @@ -551,7 +543,7 @@ TEST_CASE("io_remap_test","io_remap_test") // ------------------------------------------------------------------------------------------------------ // All Done print (" -> Test Remapped Output ... done\n",io_comm); - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } /*==========================================================================================================*/ diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index edf5f5dbbcd9..b610002c8aca 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -47,8 +47,7 @@ TEST_CASE("se_grid_io") int num_levs = 2 + SCREAM_PACK_SIZE; // Initialize the pio_subsystem for this test: - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + scorpio::init_subsystem(io_comm); // First set up a field manager and grids manager to interact with the output functions auto gm = get_test_gm(io_comm,num_my_elems,np,num_levs); @@ -95,7 +94,7 @@ TEST_CASE("se_grid_io") ins_input.finalize(); // All Done - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } /*===================================================================================================*/ diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 7e59574b1b9b..40c8ffee528e 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -68,8 +68,7 @@ TEST_CASE("output_restart","io") const auto& out_fields = fm0->get_groups_info().at("output")->m_fields_names; // Initialize the pio_subsystem for this test: - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); + scorpio::init_subsystem(comm); // Timestamp of the simulation initial time util::TimeStamp t0 ({2000,1,1},{0,0,0}); @@ -150,7 +149,7 @@ TEST_CASE("output_restart","io") print(" DONE\n"); } // Finalize everything - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } /*=============================================================================================*/ diff --git a/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp b/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp new file mode 100644 index 000000000000..ea2999eef575 --- /dev/null +++ b/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp @@ -0,0 +1,217 @@ +#include + +#include "share/io/scream_scorpio_interface.hpp" +#include + +namespace scream { + +using namespace scorpio; + +TEST_CASE ("io_subsystem") { + ekat::Comm comm (MPI_COMM_WORLD); + + init_subsystem (comm); + REQUIRE_THROWS (init_subsystem(comm)); // ERROR: already inited + + finalize_subsystem (); + REQUIRE_THROWS (finalize_subsystem()); // ERROR: no subsystem active +} + +TEST_CASE ("write_and_read") { + ekat::Comm comm (MPI_COMM_WORLD); + + EKAT_REQUIRE_MSG (comm.size()<=4, + "Error! This test is tailored for an MPI_Comm of size 1, 2, 3, or 4.\n" + " - MPI_Comm size: " + std::to_string(comm.size()) + "\n"); + + init_subsystem (comm); + + std::string filename = "scorpio_interface_write_test_np" + std::to_string(comm.size()) + ".nc"; + + const int dim1 = 2; + const int dim2 = 4; + const int dim3 = 12; + const int ldim3 = dim3 / comm.size(); + + // Offsets for dim3 decomp owned by this rank + std::vector my_offsets; + for (int i=0; i var1 (dim1); + std::vector var2 (dim1*dim2); + std::vector var3 (1); + std::vector var45 (ldim3*dim1); + + // Write first time slice + update_time (filename,0.0); + std::iota (var1.begin(),var1.end(),100); + std::iota (var2.begin(),var2.end(),100); + std::iota (var3.begin(),var3.end(),100); + std::iota (var45.begin(),var45.end(),100+comm.rank()*ldim3*dim1); + write_var (filename,"var1",var1.data()); + write_var (filename,"var2",var2.data()); + write_var (filename,"var3",var3.data()); + write_var (filename,"var4",var45.data()); + write_var (filename,"var5",var45.data()); + + REQUIRE_THROWS (write_var (filename,"var3",static_cast(nullptr))); // ERROR: invalid pointer + + // Write second time slice + update_time (filename,0.5); + std::iota (var2.begin(),var2.end(), 200); + std::iota (var3.begin(),var3.end(), 200); + std::iota (var45.begin(),var45.end(),200+comm.rank()*ldim3*dim1); + write_var (filename,"var2",var2.data()); + write_var (filename,"var3",var3.data()); + double minus_one = -1; + write_var (filename,"var5",var45.data(),&minus_one); + + // Cleanup + release_file (filename); + + REQUIRE_THROWS (release_file (filename)); // ERROR: file not open + REQUIRE (not is_file_open(filename)); + } + + // Read phase + { + using strvec_t = std::vector; + + register_file (filename,Read); + REQUIRE (is_file_open(filename)); + + // Check dims + REQUIRE (has_dim(filename,"the_time",2)); + REQUIRE (has_dim(filename,"dim1",dim1)); + REQUIRE (has_dim(filename,"dim2",dim2)); + REQUIRE (has_dim(filename,"dim3",dim3)); + + // Check vars + REQUIRE (has_var(filename,"the_time")); + const auto& timevar = get_var(filename,"the_time"); + REQUIRE (timevar.dim_names().size()==0); + REQUIRE (timevar.time_dep==true); + + REQUIRE (has_var(filename,"var1")); + const auto& piovar1 = get_var(filename,"var1"); + REQUIRE (piovar1.dim_names()==strvec_t{"dim1"}); + REQUIRE (piovar1.time_dep==false); + + REQUIRE (has_var(filename,"var2")); + const auto& piovar2 = get_var(filename,"var2"); + REQUIRE (piovar2.dim_names()==strvec_t{"dim1","dim2"}); + REQUIRE (piovar2.time_dep==true); + + REQUIRE (has_var(filename,"var3")); + const auto& piovar3 = get_var(filename,"var3"); + REQUIRE (piovar3.dim_names().size()==0); + REQUIRE (piovar3.time_dep==true); + + REQUIRE (has_var(filename,"var4")); + const auto& piovar4 = get_var(filename,"var4"); + REQUIRE (piovar4.dim_names()==strvec_t{"dim3","dim1"}); + REQUIRE (piovar4.time_dep==false); + + REQUIRE (has_var(filename,"var5")); + const auto piovar5= get_var(filename,"var5"); + REQUIRE (piovar5.dim_names()==strvec_t{"dim3","dim1"}); + REQUIRE (piovar5.time_dep==true); + + REQUIRE (not has_var(filename,"var0")); // Var not in file + + set_dim_decomp (filename,"dim3",my_offsets); + + std::vector var1 (dim1); + std::vector var2 (dim1*dim2); + std::vector var3 (1); + std::vector var45 (ldim3*dim1); + + std::vector tgt_var1 (dim1); + std::vector tgt_var2 (dim1*dim2); + std::vector tgt_var3 (1); + std::vector tgt_var45 (ldim3*dim1); + std::iota (tgt_var1.begin(),tgt_var1.end(), 100); + std::iota (tgt_var2.begin(),tgt_var2.end(), 100); + std::iota (tgt_var3.begin(),tgt_var3.end(), 100); + std::iota (tgt_var45.begin(),tgt_var45.end(),100+comm.rank()*ldim3*dim1); + + read_var (filename,"var1",var1.data()); + REQUIRE (tgt_var1==var1); + + // Read first time slice + REQUIRE_THROWS (read_var (filename,"var3",var3.data(),3)); // ERROR: time_idx out of bounds + REQUIRE_THROWS (read_var (filename,"var3",static_cast(nullptr))); // ERROR: invalid pointer + + read_var (filename,"var2",var2.data(),0); + REQUIRE (tgt_var2==var2); + + read_var (filename,"var3",var3.data(),0); + REQUIRE (tgt_var3==var3); + + read_var (filename,"var4",var45.data(),0); + REQUIRE (tgt_var45==var45); + + read_var (filename,"var5",var45.data(),0); + REQUIRE (tgt_var45==var45); + + // Read second time slice + std::iota (tgt_var2.begin(),tgt_var2.end(), 200); + std::iota (tgt_var3.begin(),tgt_var3.end(), 200); + std::iota (tgt_var45.begin(),tgt_var45.end(),200+comm.rank()*ldim3*dim1); + + read_var (filename,"var2",var2.data(),1); + REQUIRE (tgt_var2==var2); + + read_var (filename,"var3",var3.data(),1); + REQUIRE (tgt_var3==var3); + + read_var (filename,"var5",var45.data(),1); + REQUIRE (tgt_var45==var45); + + // Cleanup + release_file (filename); + } + + finalize_subsystem (); +} + +} // namespace scream From 413dfcb75f94ce9e0fb91b450f1f94fa9ecd321b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 18 Apr 2024 18:22:25 -0600 Subject: [PATCH 137/476] EAMxx: propagate IO lib changes to the share library --- .../share/grid/mesh_free_grids_manager.cpp | 8 ++-- .../grid/remap/horiz_interp_remapper_data.cpp | 45 +++++-------------- .../share/grid/remap/vertical_remapper.cpp | 10 +---- .../share/tests/coarsening_remapper_tests.cpp | 34 ++++++-------- .../tests/eamxx_time_interpolation_tests.cpp | 4 +- .../tests/refining_remapper_p2p_tests.cpp | 37 +++++++-------- .../tests/refining_remapper_rma_tests.cpp | 37 +++++++-------- .../share/tests/vertical_remapper_tests.cpp | 21 +++------ .../share/util/eamxx_time_interpolation.cpp | 41 ++++++++++++----- 9 files changed, 99 insertions(+), 138 deletions(-) diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index 604ee6ba8cad..e0f78e10cf06 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -166,13 +166,13 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const lev.sync_to_dev(); } else if (geo_data_source=="IC_FILE"){ const auto& filename = m_params.get("ic_filename"); - if (scorpio::has_variable(filename,"lat") && - scorpio::has_variable(filename,"lon")) { + if (scorpio::has_var(filename,"lat") && + scorpio::has_var(filename,"lon")) { load_lat_lon(grid,filename); } - if (scorpio::has_variable(filename,"hyam") && - scorpio::has_variable(filename,"hybm")) { + if (scorpio::has_var(filename,"hyam") && + scorpio::has_var(filename,"hybm")) { load_vertical_coordinates(grid,filename); } } diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index d44fc6aa2f0f..b26501a499b9 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -40,40 +40,19 @@ get_my_triplets (const std::string& map_file) const // 1. Load the map file chunking it evenly across all ranks scorpio::register_file(map_file,scorpio::FileMode::Read); - // 1.1 Create a "helper" grid, with as many dofs as the number - // of triplets in the map file, and divided linearly across ranks - const int ngweights = scorpio::get_dimlen(map_file,"n_s"); - int nlweights = ngweights / comm.size(); - if (comm.rank() < (ngweights % comm.size())) { - nlweights += 1; - } - - gid_type offset = nlweights; - comm.scan(&offset,1,MPI_SUM); - offset -= nlweights; // scan is inclusive, but we need exclusive + // Inform scorpio that we will provide "int" pointers for row/col indices + scorpio::change_var_dtype(map_file,"row","int"); + scorpio::change_var_dtype(map_file,"col","int"); - // Create a unique decomp tag, which ensures all refining remappers have - // their own decomposition - static int tag_counter = 0; - const std::string int_decomp_tag = "RR::gmtg,int,grid-idx=" + std::to_string(tag_counter++); - const std::string real_decomp_tag = "RR::gmtg,real,grid-idx=" + std::to_string(tag_counter++); + // Decompose n_s dim linearly across ranks + scorpio::set_dim_decomp (map_file,"n_s"); + int nlweights = scorpio::get_dimlen_local(map_file,"n_s"); - // 1.2 Read a chunk of triplets col indices + // 1.1 Read a chunk of triplets col indices std::vector cols(nlweights); std::vector rows(nlweights); std::vector S(nlweights); - scorpio::register_variable(map_file, "col", "col", {"n_s"}, "int", int_decomp_tag); - scorpio::register_variable(map_file, "row", "row", {"n_s"}, "int", int_decomp_tag); - scorpio::register_variable(map_file, "S", "S", {"n_s"}, "real", real_decomp_tag); - - std::vector dofs_offsets(nlweights); - std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); - scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"S" ,nlweights,dofs_offsets.data()); - scorpio::set_decomp(map_file); - // Figure out if we are reading the right map, that is: // - n_a or n_b matches the fine grid ncols // - the map "direction" (fine->coarse or coarse->fine) matches m_type @@ -95,13 +74,13 @@ get_my_triplets (const std::string& map_file) const " - fine grid ncols: " + std::to_string(ncols_fine) + "\n" " - remapper type: " + std::string(type==InterpType::Refine ? "refine" : "coarsen") + "\n"); - scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),nlweights); - scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),nlweights); - scorpio::grid_read_data_array(map_file,"S" ,-1,S.data(),nlweights); + scorpio::read_var(map_file,"col",cols.data()); + scorpio::read_var(map_file,"row",rows.data()); + scorpio::read_var(map_file,"S" ,S.data()); - scorpio::eam_pio_closefile(map_file); + scorpio::release_file(map_file); - // 1.3 Dofs in grid are likely 0-based, while row/col ids in map file + // 1.2 Dofs in grid are likely 0-based, while row/col ids in map file // are likely 1-based. To match dofs, we need to offset the row/cols // ids we just read in. int map_file_min_row = std::numeric_limits::max(); diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 78d18e086a65..c899bef26dc1 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -70,7 +70,7 @@ VerticalRemapper (const grid_ptr_type& src_grid, // Add tgt pressure levels to the tgt grid tgt_grid->set_geometry_data(m_remap_pres); - scorpio::eam_pio_closefile(map_file); + scorpio::release_file(map_file); } FieldLayout VerticalRemapper:: @@ -154,13 +154,7 @@ set_pressure_levels(const std::string& map_file) auto remap_pres_scal = m_remap_pres.get_view(); - std::vector dofs_offsets(m_num_remap_levs); - std::iota(dofs_offsets.begin(),dofs_offsets.end(),0); - const std::string decomp_tag = "VR::spl,nlev=" + std::to_string(m_num_remap_levs) + ",file-idx=" + std::to_string(file2idx[map_file]); - scorpio::register_variable(map_file, "p_levs", "p_levs", {"lev"}, "real", decomp_tag); - scorpio::set_dof(map_file,"p_levs",m_num_remap_levs,dofs_offsets.data()); - scorpio::set_decomp(map_file); - scorpio::grid_read_data_array(map_file,"p_levs",-1,remap_pres_scal.data(),remap_pres_scal.size()); + scorpio::read_var(map_file,"p_levs",remap_pres_scal.data()); m_remap_pres.sync_to_dev(); } diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index 7f1a3c20c7b0..c6223b67dbff 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -236,22 +236,15 @@ void create_remap_file(const std::string& filename, const int ngdofs_tgt) scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename,"n_a", "n_a", ngdofs_src, true); - scorpio::register_dimension(filename,"n_b", "n_b", ngdofs_tgt, true); - scorpio::register_dimension(filename,"n_s", "n_s", nnz, true); + scorpio::define_dim(filename,"n_a", ngdofs_src); + scorpio::define_dim(filename,"n_b", ngdofs_tgt); + scorpio::define_dim(filename,"n_s", nnz); - scorpio::register_variable(filename,"col","col","none",{"n_s"},"int","int","int-nnz"); - scorpio::register_variable(filename,"row","row","none",{"n_s"},"int","int","int-nnz"); - scorpio::register_variable(filename,"S","S","none",{"n_s"},"double","double","Real-nnz"); + scorpio::define_var(filename,"col",{"n_s"},"int"); + scorpio::define_var(filename,"row",{"n_s"},"int"); + scorpio::define_var(filename,"S" ,{"n_s"},"double"); - std::vector dofs(nnz); - std::iota(dofs.begin(),dofs.end(),0); - - scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); - - scorpio::eam_pio_enddef(filename); + scorpio::enddef(filename); std::vector col(nnz), row(nnz); std::vector S(nnz,0.5); @@ -262,11 +255,11 @@ void create_remap_file(const std::string& filename, const int ngdofs_tgt) col[2*i+1] = i+1; } - scorpio::grid_write_data_array(filename,"row",row.data(),nnz); - scorpio::grid_write_data_array(filename,"col",col.data(),nnz); - scorpio::grid_write_data_array(filename,"S", S.data(),nnz); + scorpio::write_var(filename,"row",row.data()); + scorpio::write_var(filename,"col",col.data()); + scorpio::write_var(filename,"S", S.data()); - scorpio::eam_pio_closefile(filename); + scorpio::release_file(filename); } TEST_CASE("coarsening_remap") @@ -288,8 +281,7 @@ TEST_CASE("coarsening_remap") root_print (" | Testing coarsening remapper |\n",comm); root_print (" +---------------------------------+\n\n",comm); - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); + scorpio::init_subsystem(comm); auto engine = setup_random_test (&comm); // -------------------------------------- // @@ -507,7 +499,7 @@ TEST_CASE("coarsening_remap") } // Clean up scorpio stuff - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // namespace scream diff --git a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp index 53e423dfa4d7..f2a32818a0a1 100644 --- a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp +++ b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp @@ -139,7 +139,7 @@ TEST_CASE ("eamxx_time_interpolation_data_from_file") { // Setup basic test printf(" - Test Basics...\n"); ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); std::mt19937_64 engine(seed); const auto t0 = init_timestamp(); @@ -229,7 +229,7 @@ TEST_CASE ("eamxx_time_interpolation_data_from_file") { printf(" ... DONE\n"); // All done with IO - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); printf("TimeInterpolation - From File Case...DONE\n\n\n"); } // TEST_CASE eamxx_time_interpolation_data_from_file diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp index 75ea20f81303..56d0de8a47ae 100644 --- a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -123,21 +123,15 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); - scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); - scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); - - scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); - scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); - scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); - - std::vector dofs(nnz); - std::iota(dofs.begin(),dofs.end(),0); - scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); - - scorpio::eam_pio_enddef(filename); + scorpio::define_dim(filename, "n_a", ngdofs_src); + scorpio::define_dim(filename, "n_b", ngdofs_tgt); + scorpio::define_dim(filename, "n_s", nnz); + + scorpio::define_var(filename, "col", {"n_s"}, "int"); + scorpio::define_var(filename, "row", {"n_s"}, "int"); + scorpio::define_var(filename, "S", {"n_s"}, "double"); + + scorpio::enddef(filename); std::vector col(nnz), row(nnz); std::vector S(nnz); @@ -156,11 +150,11 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { S[ngdofs_src+2*i+1] = 0.5; } - scorpio::grid_write_data_array(filename,"row",row.data(),nnz); - scorpio::grid_write_data_array(filename,"col",col.data(),nnz); - scorpio::grid_write_data_array(filename,"S", S.data(), nnz); + scorpio::write_var(filename,"row",row.data()); + scorpio::write_var(filename,"col",col.data()); + scorpio::write_var(filename,"S", S.data()); - scorpio::eam_pio_closefile(filename); + scorpio::release_file(filename); } TEST_CASE ("refining_remapper") { @@ -172,8 +166,7 @@ TEST_CASE ("refining_remapper") { auto engine = setup_random_test (&comm); - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); + scorpio::init_subsystem(comm); // Create a map file const int ngdofs_src = 4*comm.size(); @@ -417,7 +410,7 @@ TEST_CASE ("refining_remapper") { // Clean up r = nullptr; - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // namespace scream diff --git a/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp index 53fb79321d44..b37bd2df1398 100644 --- a/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_rma_tests.cpp @@ -178,21 +178,15 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); - scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); - scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); - - scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); - scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); - scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); - - std::vector dofs(nnz); - std::iota(dofs.begin(),dofs.end(),0); - scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); - - scorpio::eam_pio_enddef(filename); + scorpio::define_dim(filename, "n_a", ngdofs_src); + scorpio::define_dim(filename, "n_b", ngdofs_tgt); + scorpio::define_dim(filename, "n_s", nnz); + + scorpio::define_var(filename, "col", {"n_s"}, "int"); + scorpio::define_var(filename, "row", {"n_s"}, "int"); + scorpio::define_var(filename, "S", {"n_s"}, "double"); + + scorpio::enddef(filename); std::vector col(nnz), row(nnz); std::vector S(nnz); @@ -211,11 +205,11 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { S[ngdofs_src+2*i+1] = 0.5; } - scorpio::grid_write_data_array(filename,"row",row.data(),nnz); - scorpio::grid_write_data_array(filename,"col",col.data(),nnz); - scorpio::grid_write_data_array(filename,"S", S.data(), nnz); + scorpio::write_var(filename,"row",row.data()); + scorpio::write_var(filename,"col",col.data()); + scorpio::write_var(filename,"S", S.data()); - scorpio::eam_pio_closefile(filename); + scorpio::release_file(filename); } TEST_CASE ("refining_remapper") { @@ -227,8 +221,7 @@ TEST_CASE ("refining_remapper") { auto engine = setup_random_test (&comm); - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); + scorpio::init_subsystem(comm); // Create a map file const int ngdofs_src = 4*comm.size(); @@ -475,7 +468,7 @@ TEST_CASE ("refining_remapper") { // Clean up r = nullptr; - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // namespace scream diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 131f6380f527..f7bd62b79989 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -96,20 +96,14 @@ Real data_func(const int col, const int vec, const Real pres) { // Helper function to create a remap file void create_remap_file(const std::string& filename, const int nlevs, const std::vector& dofs_p, const std::vector& p_tgt) { - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::define_dim(filename,"lev",nlevs); + scorpio::define_var(filename,"p_levs",{"lev"},"real"); + scorpio::enddef(filename); - scorpio::register_dimension(filename,"lev","lev",nlevs, false); - - scorpio::register_variable(filename,"p_levs","p_levs","none",{"lev"},"real","real","Real-lev"); - - scorpio::set_dof(filename,"p_levs",dofs_p.size(),dofs_p.data()); - - scorpio::eam_pio_enddef(filename); - - scorpio::grid_write_data_array(filename,"p_levs",p_tgt.data(),nlevs); + scorpio::write_var(filename,"p_levs",p_tgt.data()); - scorpio::eam_pio_closefile(filename); + scorpio::release_file(filename); } TEST_CASE ("vertical_remap") { @@ -121,8 +115,7 @@ TEST_CASE ("vertical_remap") { ekat::Comm comm(MPI_COMM_WORLD); - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); + scorpio::init_subsystem(comm); // -------------------------------------- // // Set grid/map sizes // @@ -389,7 +382,7 @@ TEST_CASE ("vertical_remap") { } // Clean up scorpio stuff - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } } // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index 3515bcdaac91..ddd9faf9e763 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -1,4 +1,5 @@ #include "share/util/eamxx_time_interpolation.hpp" +#include "share/io/scream_io_utils.hpp" namespace scream{ namespace util { @@ -164,15 +165,32 @@ void TimeInterpolation::initialize_data_from_files() m_file_data_atm_input.set_logger(m_logger); // Assign the mask value gathered from the FillValue found in the source file. // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? - float var_fill_value; for (auto& name : m_field_names) { - scorpio::get_variable_metadata(triplet_curr.filename,name,"_FillValue",var_fill_value); auto& field0 = m_fm_time0->get_field(name); - field0.get_header().set_extra_data("mask_value",var_fill_value); auto& field1 = m_fm_time1->get_field(name); - field1.get_header().set_extra_data("mask_value",var_fill_value); auto& field_out = m_interp_fields.at(name); - field_out.get_header().set_extra_data("mask_value",var_fill_value); + + auto set_fill_value = [&](const auto var_fill_value) { + field0.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + field1.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + field_out.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + }; + + const auto& pio_var = scorpio::get_var(triplet_curr.filename,name); + if (scorpio::refine_dtype(pio_var.nc_dtype)=="float") { + auto var_fill_value = scorpio::get_attribute(triplet_curr.filename,name,"_FillValue"); + set_fill_value(var_fill_value); + } else if (scorpio::refine_dtype(pio_var.nc_dtype)=="double") { + auto var_fill_value = scorpio::get_attribute(triplet_curr.filename,name,"_FillValue"); + set_fill_value(var_fill_value); + } else { + EKAT_ERROR_MSG ( + "Unrecognized/unsupported data type\n" + " - filename: " + triplet_curr.filename + "\n" + " - varname : " + name + "\n" + " - dtype : " + pio_var.dtype + "\n"); + } + } // Read first snap of data and shift to time0 read_data(); @@ -253,10 +271,10 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { for (size_t ii=0; ii(time_units_tmp); + auto time_units = scorpio::get_attribute(filename,"time","units"); int time_mult; if (time_units.find("seconds") != std::string::npos) { time_mult = 1; @@ -273,10 +291,9 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { if (ii==0) { ts_ref = ts_file_start; } - scorpio::register_file(filename,scorpio::Read); const int ntime = scorpio::get_dimlen(filename,"time"); for (int tt=0; tt0) { ts_snap += (time_snap*time_mult); @@ -292,6 +309,7 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { time_idx_tmp.push_back(tt); ++running_idx; } + scorpio::release_file(filename); } // Now that we have gathered all of the timesnaps we can arrange them in order as DataFromFileTriplet objects. // Taking advantage of maps automatically self-sorting by the first arg. @@ -323,9 +341,8 @@ void TimeInterpolation::read_data() m_file_data_atm_input.set_logger(m_logger); // Also determine the FillValue, if used // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? - float var_fill_value; for (auto& name : m_field_names) { - scorpio::get_variable_metadata(triplet_curr.filename,name,"_FillValue",var_fill_value); + auto var_fill_value = scorpio::get_attribute(triplet_curr.filename,name,"_FillValue"); auto& field = m_fm_time1->get_field(name); field.get_header().set_extra_data("mask_value",var_fill_value); } From 5307e570673322b71435602f41022617e1f44265 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 16:38:21 -0600 Subject: [PATCH 138/476] EAMxx: propagate io lib changes to the rest of eamxx --- .../eamxx/src/control/atmosphere_driver.cpp | 42 +++++++------- .../control/intensive_observation_period.cpp | 57 ++++++------------- .../src/dynamics/homme/tests/dyn_grid_io.cpp | 5 +- .../eamxx_nudging_process_interface.cpp | 2 +- .../physics/nudging/tests/create_map_file.cpp | 32 +++++------ .../nudging/tests/create_nudging_data.cpp | 4 +- .../physics/nudging/tests/nudging_tests.cpp | 4 +- .../src/physics/spa/spa_functions_impl.hpp | 2 +- .../spa/tests/spa_one_to_one_remap_test.cpp | 4 +- .../tests/spa_read_data_from_file_test.cpp | 4 +- .../create_vert_remap_and_weights.cpp | 44 ++++++-------- .../surface_coupling/surface_coupling.cpp | 4 +- 12 files changed, 84 insertions(+), 120 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index a5dacd856d75..6e941f853ecd 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -140,11 +140,10 @@ init_scorpio(const int atm_id) // Init scorpio right away, in case some class (atm procs, grids,...) // needs to source some data from NC files during construction, // before we start processing IC files. - EKAT_REQUIRE_MSG (!scorpio::is_eam_pio_subsystem_inited(), + EKAT_REQUIRE_MSG (!scorpio::is_subsystem_inited(), "Error! The PIO subsystem was alreday inited before the driver was constructed.\n" " This is an unexpected behavior. Please, contact developers.\n"); - MPI_Fint fcomm = MPI_Comm_c2f(m_atm_comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm,atm_id); + scorpio::init_subsystem(m_atm_comm,atm_id); // In CIME runs, gptl is already inited. In standalone runs, it might // not be, depending on what scorpio does. @@ -909,7 +908,7 @@ void AtmosphereDriver::restart_model () } // Restart the num steps counter in the atm time stamp - int nsteps = scorpio::get_attribute(filename,"nsteps"); + int nsteps = scorpio::get_attribute(filename,"GLOBAL","nsteps"); m_current_ts.set_num_steps(nsteps); m_run_t0.set_num_steps(nsteps); @@ -917,18 +916,22 @@ void AtmosphereDriver::restart_model () const auto& name = it.first; auto& any = it.second; - - auto data = scorpio::get_any_attribute(filename,name); - EKAT_REQUIRE_MSG (any->content().type()==data.content().type(), - "Error! Type mismatch for restart global attribute.\n" - " - file name: " + filename + "\n" - " - att name: " + name + "\n" - " - expected type: " + any->content().type().name() + "\n" - " - actual type: " + data.content().type().name() + "\n" - "NOTE: the above names use type_info::name(), which returns an implementation defined string,\n" - " with no guarantees. In particular, the string can be identical for several types,\n" - " and/or change between invocations of the same program.\n"); - *any = data; + if (any->isType()) { + ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any->isType()) { + ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any->isType()) { + ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any->isType()) { + ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any->isType()) { + ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else { + EKAT_ERROR_MSG ( + "Error! Unrecognized/unsupported concrete type for restart extra data.\n" + " - extra data name : " + name + "\n" + " - extra data typeid: " + any->content().type().name() + "\n"); + } } m_atm_logger->info(" [EAMxx] restart_model ... done!"); @@ -1681,9 +1684,10 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { finalize_gptl(); } - // Finalize scorpio - if (scorpio::is_eam_pio_subsystem_inited()) { - scorpio::eam_pio_finalize(); + // Finalize scorpio. Check, just in case we're calling finalize after + // an exception, thrown before the AD (and scorpio) was inited + if (scorpio::is_subsystem_inited()) { + scorpio::finalize_subsystem(); } m_atm_logger->info("[EAMxx] Finalize ... done!"); diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index a374c595102b..4f348747c0fd 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -39,32 +39,13 @@ void read_dimensionless_variable_from_file(const std::string& filename, const std::string& varname, T* value) { - EKAT_REQUIRE_MSG(scorpio::has_variable(filename,varname), - "Error! IOP file does not have variable "+varname+".\n"); - - int ncid, varid, err1, err2; - bool was_open = scorpio::is_file_open_c2f(filename.c_str(),-1); - if (not was_open) { - scorpio::register_file(filename,scorpio::FileMode::Read); - } - ncid = scorpio::get_file_ncid_c2f (filename.c_str()); - err1 = PIOc_inq_varid(ncid,varname.c_str(),&varid); - EKAT_REQUIRE_MSG(err1==PIO_NOERR, - "Error! Something went wrong while retrieving variable id.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - pio error: " + std::to_string(err1) + "\n"); - - err2 = PIOc_get_var(ncid, varid, value); - EKAT_REQUIRE_MSG(err2==PIO_NOERR, - "Error! Something went wrong while retrieving variable.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - pio error: " + std::to_string(err2) + "\n"); - - if (not was_open) { - scorpio::eam_pio_closefile(filename); - } + scorpio::register_file(filename,scorpio::FileMode::Read); + EKAT_REQUIRE_MSG(scorpio::has_var(filename,varname), + "Error! IOP file does not have a required variable.\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n"); + scorpio::read_var(filename,varname,value); + scorpio::release_file(filename); } // Read variable with arbitrary number of dimensions from file. @@ -76,7 +57,7 @@ void read_variable_from_file(const std::string& filename, const int time_idx, T* data) { - EKAT_REQUIRE_MSG(scorpio::has_variable(filename,varname), + EKAT_REQUIRE_MSG(scorpio::has_var(filename,varname), "Error! IOP file does not have variable "+varname+".\n"); // Compute total size of data to read @@ -88,14 +69,8 @@ void read_variable_from_file(const std::string& filename, // Read into data scorpio::register_file(filename, scorpio::FileMode::Read); - std::string io_decomp_tag = varname+","+filename; - scorpio::register_variable(filename, varname, varname, dimnames, vartype, io_decomp_tag); - std::vector dof_offsets(data_size); - std::iota(dof_offsets.begin(), dof_offsets.end(), 0); - scorpio::set_dof(filename, varname, dof_offsets.size(), dof_offsets.data()); - scorpio::set_decomp(filename); - scorpio::grid_read_data_array(filename, varname, time_idx, data, data_size); - scorpio::eam_pio_closefile(filename); + scorpio::read_var(filename, varname, data, time_idx); + scorpio::release_file(filename); } } @@ -173,7 +148,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, bool has_var = false; std::string file_varname = ""; for (auto varname : varnames) { - if (scorpio::has_variable(iop_file, varname)) { + if (scorpio::has_var(iop_file, varname)) { has_var = true; file_varname = varname; break; @@ -183,7 +158,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, // Store if iop file has a different varname than the iop field if (iop_varname != file_varname) m_iop_file_varnames.insert({iop_varname, file_varname}); // Store if variable contains a surface value in iop file - if (scorpio::has_variable(iop_file, srf_varname)) { + if (scorpio::has_var(iop_file, srf_varname)) { m_iop_field_surface_varnames.insert({iop_varname, srf_varname}); } // Store that the IOP variable is found in the IOP file @@ -291,9 +266,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, // Initialize time information int bdate; std::string bdate_name; - if (scorpio::has_variable(iop_file, "bdate")) bdate_name = "bdate"; - else if (scorpio::has_variable(iop_file, "basedate")) bdate_name = "basedate"; - else if (scorpio::has_variable(iop_file, "nbdate")) bdate_name = "nbdate"; + if (scorpio::has_var(iop_file, "bdate")) bdate_name = "bdate"; + else if (scorpio::has_var(iop_file, "basedate")) bdate_name = "basedate"; + else if (scorpio::has_var(iop_file, "nbdate")) bdate_name = "nbdate"; else EKAT_ERROR_MSG("Error! No valid name for bdate in "+iop_file+".\n"); read_dimensionless_variable_from_file(iop_file, bdate_name, &bdate); @@ -332,7 +307,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, // Store iop file pressure as helper field with dimension lev+1. // Load the first lev entries from iop file, the lev+1 entry will // be set when reading iop data. - EKAT_REQUIRE_MSG(scorpio::has_variable(iop_file, "lev"), + EKAT_REQUIRE_MSG(scorpio::has_var(iop_file, "lev"), "Error! Using IOP file requires variable \"lev\".\n"); const auto file_levs = scorpio::get_dimlen(iop_file, "lev"); FieldIdentifier fid("iop_file_pressure", diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index bb09daa1c49a..ec1e709c7cde 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -41,8 +41,7 @@ TEST_CASE("dyn_grid_io") ekat::Comm comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. // Initialize the pio_subsystem for this test: - MPI_Fint fcomm = MPI_Comm_c2f(comm.mpi_comm()); - scorpio::eam_init_pio_subsystem(fcomm); + scorpio::init_subsystem(comm); // Init homme context if (!is_parallel_inited_f90()) { @@ -179,7 +178,7 @@ TEST_CASE("dyn_grid_io") } // Cleanup everything - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); Homme::Context::finalize_singleton(); cleanup_test_f90(); } diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 40925c9edd1a..832c030e97b9 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -355,7 +355,7 @@ void Nudging::run_impl (const double dt) Real var_fill_value = constants::DefaultFillValue().value; // Query the helper field for the fill value, if not present use default if (f.get_header().has_extra_data("mask_value")) { - var_fill_value = f.get_header().get_extra_data("mask_value"); + var_fill_value = f.get_header().get_extra_data("mask_value"); } const int ncols = fl.dim(0); diff --git a/components/eamxx/src/physics/nudging/tests/create_map_file.cpp b/components/eamxx/src/physics/nudging/tests/create_map_file.cpp index 1a83abb5f7bb..d8a67d84b27d 100644 --- a/components/eamxx/src/physics/nudging/tests/create_map_file.cpp +++ b/components/eamxx/src/physics/nudging/tests/create_map_file.cpp @@ -7,7 +7,7 @@ TEST_CASE("create_map_file") using namespace scream; ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); // Add a dof in the middle of two coarse dofs const int ngdofs_src = 12; @@ -21,21 +21,15 @@ TEST_CASE("create_map_file") scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); - scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); - scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); + scorpio::define_dim(filename, "n_a", ngdofs_src); + scorpio::define_dim(filename, "n_b", ngdofs_tgt); + scorpio::define_dim(filename, "n_s", nnz); - scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); - scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); - scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); + scorpio::define_var(filename, "col", {"n_s"}, "int"); + scorpio::define_var(filename, "row", {"n_s"}, "int"); + scorpio::define_var(filename, "S", {"n_s"}, "double"); - std::vector dofs(nnz); - std::iota(dofs.begin(),dofs.end(),0); - scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); - scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); - - scorpio::eam_pio_enddef(filename); + scorpio::enddef(filename); std::vector col(nnz), row(nnz); std::vector S(nnz); @@ -54,10 +48,10 @@ TEST_CASE("create_map_file") S[ngdofs_src+2*i+1] = 0.5; } - scorpio::grid_write_data_array(filename,"row",row.data(),nnz); - scorpio::grid_write_data_array(filename,"col",col.data(),nnz); - scorpio::grid_write_data_array(filename,"S", S.data(), nnz); + scorpio::write_var(filename,"row",row.data()); + scorpio::write_var(filename,"col",col.data()); + scorpio::write_var(filename,"S", S.data()); - scorpio::eam_pio_closefile(filename); - scorpio::eam_pio_finalize(); + scorpio::release_file(filename); + scorpio::finalize_subsystem(); } diff --git a/components/eamxx/src/physics/nudging/tests/create_nudging_data.cpp b/components/eamxx/src/physics/nudging/tests/create_nudging_data.cpp index f5c8e6b4bdae..000322b23740 100644 --- a/components/eamxx/src/physics/nudging/tests/create_nudging_data.cpp +++ b/components/eamxx/src/physics/nudging/tests/create_nudging_data.cpp @@ -20,7 +20,7 @@ TEST_CASE("create_nudging_data") { const auto t0 = get_t0(); // Initialize the pio_subsystem for this test: - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); // Create a grids manager const auto gm = create_gm(comm,ngcols,nlevs); @@ -56,5 +56,5 @@ TEST_CASE("create_nudging_data") { om2->finalize(); om3->finalize(); - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index f54112056279..8870b4cce209 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -44,7 +44,7 @@ TEST_CASE("nudging_tests") { }; // Init scorpio - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); // A refined grid, with one extra node in between each of the coarse ones const int ngcols_fine = 2*ngcols_data - 1; @@ -462,5 +462,5 @@ TEST_CASE("nudging_tests") { } // Clean up scorpio - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index d9547953d1ed..287f69a98896 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -83,7 +83,7 @@ create_horiz_remapper ( const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); - scorpio::eam_pio_closefile(spa_data_file); + scorpio::release_file(spa_data_file); // We could use model_grid directly if using same num levels, // but since shallow clones are cheap, we may as well do it (less lines of code) diff --git a/components/eamxx/src/physics/spa/tests/spa_one_to_one_remap_test.cpp b/components/eamxx/src/physics/spa/tests/spa_one_to_one_remap_test.cpp index 5579c3efa7e6..71a7187ed38d 100644 --- a/components/eamxx/src/physics/spa/tests/spa_one_to_one_remap_test.cpp +++ b/components/eamxx/src/physics/spa/tests/spa_one_to_one_remap_test.cpp @@ -25,7 +25,7 @@ TEST_CASE("spa_one_to_one_remap","spa") { // Set up the mpi communicator and init the pio subsystem ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); std::string spa_data_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; @@ -107,7 +107,7 @@ TEST_CASE("spa_one_to_one_remap","spa") // All Done reader = nullptr; - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } // run_property // Some helper functions for the require statements: diff --git a/components/eamxx/src/physics/spa/tests/spa_read_data_from_file_test.cpp b/components/eamxx/src/physics/spa/tests/spa_read_data_from_file_test.cpp index 4c6533b5870f..059781f40d8e 100644 --- a/components/eamxx/src/physics/spa/tests/spa_read_data_from_file_test.cpp +++ b/components/eamxx/src/physics/spa/tests/spa_read_data_from_file_test.cpp @@ -26,7 +26,7 @@ TEST_CASE("spa_read_data","spa") // Set up the mpi communicator and init the pio subsystem ekat::Comm comm(MPI_COMM_WORLD); - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); std::string spa_data_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; std::string spa_remap_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; @@ -104,7 +104,7 @@ TEST_CASE("spa_read_data","spa") // Clean up reader = nullptr; - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); } // Some helper functions for the require statements: diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp index 702d83f16d37..6224ad0cf29d 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp @@ -8,13 +8,10 @@ using namespace scream; void create_vert_remap() { // Simple function to create a 1D remap column to test nudging w/ remapped data - ekat::Comm io_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); int nlevs = 5*SCREAM_PACK_SIZE+1; - std::vector dofs_levs(nlevs); - std::iota(dofs_levs.begin(),dofs_levs.end(),0); std::vector p_tgt; Real p_top=0, p_bot=102500; Real dp = (p_bot - p_top) / (nlevs-1); @@ -26,24 +23,20 @@ void create_vert_remap() { std::string remap_filename = "vertical_remap.nc"; scorpio::register_file(remap_filename, scorpio::FileMode::Write); - scorpio::register_dimension(remap_filename,"lev", "lev", nlevs, false); - scorpio::register_variable(remap_filename,"p_levs","p_levs","none",{"lev"},"real","real","Real-lev"); - scorpio::set_dof(remap_filename,"p_levs",dofs_levs.size(),dofs_levs.data()); - scorpio::eam_pio_enddef(remap_filename); - scorpio::grid_write_data_array(remap_filename,"p_levs",p_tgt.data(),nlevs); - scorpio::eam_pio_closefile(remap_filename); - scorpio::eam_pio_finalize(); + scorpio::define_dim(remap_filename, "lev", nlevs); + scorpio::define_var(remap_filename,"p_levs",{"lev"},"real"); + scorpio::enddef(remap_filename); + scorpio::write_var(remap_filename,"p_levs",p_tgt.data()); + scorpio::release_file(remap_filename); + scorpio::finalize_subsystem(); } void create_nudging_weights_ncfile(int ntimes, int ncols, int nlevs, const std::string& filename) { // Simple function to create a 1D remap column to test nudging w/ remapped data - ekat::Comm io_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); - std::vector dofs(ntimes*ncols*nlevs); - std::iota(dofs.begin(),dofs.end(),0); Real plev[nlevs]; Real p_top=0, p_bot=102500; Real dp = (p_bot - p_top) / (nlevs-1); @@ -65,15 +58,14 @@ void create_nudging_weights_ncfile(int ntimes, int ncols, int nlevs, const std:: } scorpio::register_file(filename, scorpio::FileMode::Write); - scorpio::register_dimension(filename,"ncol", "ncol", ncols, false); - scorpio::register_dimension(filename,"lev", "lev", nlevs, false); - scorpio::register_dimension(filename,"time","time",ntimes, false); - scorpio::register_variable(filename,"nudging_weights","nudging_weights","none",{"lev", "ncol", "time"},"real","real","Real-lev"); - scorpio::set_dof(filename,"nudging_weights",dofs.size(),dofs.data()); - scorpio::eam_pio_enddef(filename); - scorpio::grid_write_data_array(filename,"nudging_weights",&weights[0][0][0],ntimes*ncols*nlevs); - scorpio::eam_pio_closefile(filename); - scorpio::eam_pio_finalize(); + scorpio::define_dim(filename,"ncol",ncols); + scorpio::define_dim(filename,"lev", nlevs); + scorpio::define_dim(filename,"time",ntimes); + scorpio::define_var(filename,"nudging_weights",{"lev", "ncol", "time"},"real"); + scorpio::enddef(filename); + scorpio::write_var(filename,"nudging_weights",&weights[0][0][0]); + scorpio::release_file(filename); + scorpio::finalize_subsystem(); } TEST_CASE("create_vert_remap_and_weights","create_vert_remap_and_weights") diff --git a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index d34059f8559f..a4e2a18cbfb2 100644 --- a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -88,7 +88,7 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons } // Create output manager to handle the data - scorpio::eam_init_pio_subsystem(comm); + scorpio::init_subsystem(comm); ekat::ParameterList om_pl; om_pl.set("filename_prefix",std::string("surface_coupling_forcing")); om_pl.set("Field Names",fnames); @@ -119,7 +119,7 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons // Done, finalize om.finalize(); - scorpio::eam_pio_finalize(); + scorpio::finalize_subsystem(); return om.get_list_of_files(); } From c377e428e5144a0d9e7a4ae6acfc7d06a275a2a1 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 18 Apr 2024 18:33:37 -0600 Subject: [PATCH 139/476] EAMxx: add ability to "pretend" a dimension is the unlimited one --- .../eamxx/src/share/io/scorpio_input.cpp | 5 +++ .../src/share/io/scream_scorpio_interface.cpp | 37 +++++++++++++++++++ .../src/share/io/scream_scorpio_interface.hpp | 5 +++ 3 files changed, 47 insertions(+) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 50e1badb00f5..51c31f70a50a 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -355,7 +355,12 @@ void AtmosphereInput::init_scorpio_structures() scorpio::register_file(m_filename,scorpio::Read,iotype); + // Some input files have the "time" dimension as non-unlimited. This messes up our // scorpio interface. To avoid trouble, if a dim called 'time' is present we + // treat is as unlimited, even though it isn't. + if (scorpio::has_dim(m_filename,"time")) { + scorpio::pretend_dim_is_unlimited(m_filename,"time"); + } // Check variables are in the input file for (auto const& name : m_fields_names) { diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 2bd1eb999a6e..472d57874810 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -1075,6 +1075,43 @@ void define_time (const std::string& filename, const std::string& units, const s define_var(filename,time_name,units,{},"double","double",true); } +void pretend_dim_is_unlimited (const std::string& filename, const std::string& dimname) +{ + auto& f = impl::get_file(filename,"scorpio::mark_dim_as_time"); + EKAT_REQUIRE_MSG (f.mode==Read, + "Error! Cannot interpret dimension as 'time' dim. File not in Read mode.\n" + " - filename : " + filename + "\n" + " - file mode: " + e2str(f.mode) + "\n"); + + if (f.time_dim==nullptr) { + EKAT_REQUIRE_MSG (has_dim(filename,dimname), + "Error! Cannot interpret dimension as 'time' dim. Dimension not found.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n"); + + auto dim = f.dims.at(dimname); + f.time_dim = dim; + + // If a var has "time" in its dims (must be the 1st dim!), + // remove it. Recall that we only store non-time dims in + // the list of var dims. + for (auto& it : f.vars) { + auto& v = it.second; + if (v->dims.size()>0 and v->dims[0]->name==dimname) { + v->dims.erase(v->dims.begin()); + v->size = -1; + v->time_dep = true; + } + } + } else { + EKAT_REQUIRE_MSG (f.time_dim->name==dimname, + "Error! Attempt to change the time dimension.\n" + " - filenama : " + filename + "\n" + " - old time dim: " + f.time_dim->name + "\n" + " - new time dim: " + dimname + "\n"); + } +} + // Update value of time variable, increasing time dim length void update_time(const std::string &filename, const double time) { const auto& f = impl::get_file(filename,"scorpio::update_time"); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 1adfc630681c..2ef09e96188a 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -134,6 +134,11 @@ const PIOVar& get_var (const std::string& filename, void define_time (const std::string& filename, const std::string& units, const std::string& time_name = default_time_name()); +// When we read a file, and there is a "time" dimension that is FIXED rather than UNLIMITED, +// we may have some issues with our read/write logics. Hence, we can use mark a dimension +// as if it was the UNLIMITED time dim. +void pretend_dim_is_unlimited (const std::string& filename, const std::string& dimname); + // Update value of time variable, increasing time dim length void update_time(const std::string &filename, const double time); From bf647308edc91a935e82b0c48d54f501d39e4101 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 16:09:07 -0600 Subject: [PATCH 140/476] EAMxx: prevent copy of AtmosphereInput Since there are managed resources (in scorpio), copies would require correct ref counting. It's easier to use pointers... --- .../eamxx/src/share/io/scorpio_input.hpp | 5 +++++ .../share/util/eamxx_time_interpolation.cpp | 19 +++++++++---------- .../share/util/eamxx_time_interpolation.hpp | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index caa2f80aaf3b..801d224be831 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -77,8 +77,13 @@ class AtmosphereInput const std::vector& fields, const bool skip_grid_checks = false); + // Due to resource acquisition (in scorpio), avoid copies + AtmosphereInput (const AtmosphereInput&) = delete; ~AtmosphereInput (); + // Due to resource acquisition (in scorpio), avoid copies + AtmosphereInput& operator= (const AtmosphereInput&) = delete; + // --- Methods --- // // Initialize the class for reading into FieldManager-owned fields. // - params: input parameters (must contain at least "Filename") diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index ddd9faf9e763..a8b3d32d3dd8 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -31,8 +31,8 @@ TimeInterpolation::TimeInterpolation( void TimeInterpolation::finalize() { if (m_is_data_from_file) { - m_file_data_atm_input.finalize(); - m_is_data_from_file=false; + m_file_data_atm_input = nullptr; + m_is_data_from_file = false; } } /*-----------------------------------------------------------------------------------------------*/ @@ -114,7 +114,7 @@ void TimeInterpolation::shift_data() auto& field1 = m_fm_time1->get_field(name); std::swap(field0,field1); } - m_file_data_atm_input.set_field_manager(m_fm_time1); + m_file_data_atm_input->set_field_manager(m_fm_time1); } /*-----------------------------------------------------------------------------------------------*/ /* Function which will initialize the TimeStamps. @@ -161,8 +161,8 @@ void TimeInterpolation::initialize_data_from_files() ekat::ParameterList input_params; input_params.set("Field Names",m_field_names); input_params.set("Filename",triplet_curr.filename); - m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); - m_file_data_atm_input.set_logger(m_logger); + m_file_data_atm_input = std::make_shared(input_params,m_fm_time1); + m_file_data_atm_input->set_logger(m_logger); // Assign the mask value gathered from the FillValue found in the source file. // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? for (auto& name : m_field_names) { @@ -331,14 +331,13 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { void TimeInterpolation::read_data() { const auto triplet_curr = m_file_data_triplets[m_triplet_idx]; - if (triplet_curr.filename != m_file_data_atm_input.get_filename()) { + if (not m_file_data_atm_input or triplet_curr.filename != m_file_data_atm_input->get_filename()) { // Then we need to close this input stream and open a new one - m_file_data_atm_input.finalize(); ekat::ParameterList input_params; input_params.set("Field Names",m_field_names); input_params.set("Filename",triplet_curr.filename); - m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); - m_file_data_atm_input.set_logger(m_logger); + m_file_data_atm_input = std::make_shared(input_params,m_fm_time1); + m_file_data_atm_input->set_logger(m_logger); // Also determine the FillValue, if used // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? for (auto& name : m_field_names) { @@ -352,7 +351,7 @@ void TimeInterpolation::read_data() m_logger->info(m_header); m_logger->info("[EAMxx:time_interpolation] Reading data at time " + triplet_curr.timestamp.to_string()); } - m_file_data_atm_input.read_variables(triplet_curr.time_idx); + m_file_data_atm_input->read_variables(triplet_curr.time_idx); m_time1 = triplet_curr.timestamp; } /*-----------------------------------------------------------------------------------------------*/ diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp index 870dad330135..5e5fa5112f21 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp @@ -89,7 +89,7 @@ class TimeInterpolation { // Variables related to the case where we use data from file std::vector m_file_data_triplets; int m_triplet_idx; - AtmosphereInput m_file_data_atm_input; + std::shared_ptr m_file_data_atm_input; bool m_is_data_from_file=false; std::shared_ptr m_logger; From 09d6200b5f34d925d5cfa7ec95c3dbfb6c3adff5 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 23 Apr 2024 16:34:27 -0600 Subject: [PATCH 141/476] EAMxx: fix bug in time interpolation unit test The name of the field should not contain rank-local information --- .../eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp index f2a32818a0a1..c8551d081266 100644 --- a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp +++ b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp @@ -350,7 +350,9 @@ std::shared_ptr get_fm (const std::shared_ptr& const auto units = ekat::units::Units::nondimensional(); for (const auto& fl : layouts) { - FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); + int gl_size = fl.size(); + grid->get_comm().all_reduce(&gl_size,1,MPI_SUM); + FID fid("f_"+std::to_string(gl_size),fl,units,grid->name()); Field f(fid); f.allocate_view(); randomize (f,engine,my_pdf); From c782d6d026ea6fdbc7767427ce3a4a27ab43c4bd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 25 Apr 2024 17:37:27 -0600 Subject: [PATCH 142/476] EAMxx: fix handling of mask_value extra data inside fields --- .../eamxx/src/share/field/field_impl.hpp | 9 ++++- .../share/util/eamxx_time_interpolation.cpp | 37 ++++++++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 1964622d0057..f192d70bb688 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -465,10 +465,15 @@ update (const Field& x, const ST alpha, const ST beta) ST fill_val = constants::DefaultFillValue().value; if (x.get_header().has_extra_data("mask_value")) { - if (typeid(ST) == typeid(int)) { + + if (dt==DataType::IntType) { fill_val = x.get_header().get_extra_data("mask_value"); - } else { + } else if (dt==DataType::FloatType) { fill_val = x.get_header().get_extra_data("mask_value"); + } else if (dt==DataType::DoubleType) { + fill_val = x.get_header().get_extra_data("mask_value"); + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported field data type in Field::update.\n"); } } diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index a8b3d32d3dd8..e0dba0cf8382 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -87,6 +87,10 @@ void TimeInterpolation::add_field(const Field& field_in, const bool store_shallo const std::string name = field_in.name(); EKAT_REQUIRE_MSG(!m_fm_time0->has_field(name) and !m_fm_time1->has_field(name), "Error!! TimeInterpolation:add_field, field + " << name << " has already been added." << "\n"); + EKAT_REQUIRE_MSG (field_in.data_type()==DataType::FloatType or field_in.data_type()==DataType::DoubleType, + "[TimeInterpolation] Error! Input field must have floating-point data type.\n" + " - field name: " + field_in.name() + "\n" + " - data type : " + e2str(field_in.data_type()) + "\n"); // Clone the field for each field manager to get all the metadata correct. auto field0 = field_in.clone(); @@ -171,9 +175,21 @@ void TimeInterpolation::initialize_data_from_files() auto& field_out = m_interp_fields.at(name); auto set_fill_value = [&](const auto var_fill_value) { - field0.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); - field1.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); - field_out.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + const auto dt = field_out.data_type(); + if (dt==DataType::FloatType) { + field0.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + field1.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + field_out.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + } else if (dt==DataType::DoubleType) { + field0.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + field1.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + field_out.get_header().set_extra_data("mask_value",static_cast(var_fill_value)); + } else { + EKAT_ERROR_MSG ( + "[TimeInterpolation] Unexpected/unsupported field data type.\n" + " - field name: " + field_out.name() + "\n" + " - data type : " + e2str(dt) + "\n"); + } }; const auto& pio_var = scorpio::get_var(triplet_curr.filename,name); @@ -341,9 +357,20 @@ void TimeInterpolation::read_data() // Also determine the FillValue, if used // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? for (auto& name : m_field_names) { - auto var_fill_value = scorpio::get_attribute(triplet_curr.filename,name,"_FillValue"); auto& field = m_fm_time1->get_field(name); - field.get_header().set_extra_data("mask_value",var_fill_value); + const auto dt = field.data_type(); + if (dt==DataType::FloatType) { + auto var_fill_value = scorpio::get_attribute(triplet_curr.filename,name,"_FillValue"); + field.get_header().set_extra_data("mask_value",var_fill_value); + } else if (dt==DataType::DoubleType) { + auto var_fill_value = scorpio::get_attribute(triplet_curr.filename,name,"_FillValue"); + field.get_header().set_extra_data("mask_value",var_fill_value); + } else { + EKAT_ERROR_MSG ( + "[TimeInterpolation] Unexpected/unsupported field data type.\n" + " - field name: " + field.name() + "\n" + " - data type : " + e2str(dt) + "\n"); + } } } From 3043c0839b306a774f418d7b399206c7940bdf1e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 25 Apr 2024 18:50:58 -0600 Subject: [PATCH 143/476] EAMxx: we no longer need shared_ptr for restart extra data --- .../eamxx/src/control/atmosphere_driver.cpp | 24 +++++++++---------- .../homme/eamxx_homme_process_interface.cpp | 9 +++---- .../share/atm_process/atmosphere_process.hpp | 7 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 6e941f853ecd..f30742574fb7 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -712,7 +712,7 @@ void AtmosphereDriver::initialize_output_managers () { } om.set_logger(m_atm_logger); for (const auto& it : m_atm_process_group->get_restart_extra_data()) { - om.add_global(it.first,*it.second); + om.add_global(it.first,it.second); } // Store the "Output Control" pl of the model restart as the "Checkpoint Control" for all other output streams @@ -916,21 +916,21 @@ void AtmosphereDriver::restart_model () const auto& name = it.first; auto& any = it.second; - if (any->isType()) { - ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); - } else if (any->isType()) { - ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); - } else if (any->isType()) { - ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); - } else if (any->isType()) { - ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); - } else if (any->isType()) { - ekat::any_cast(*any) = scorpio::get_attribute(filename,"GLOBAL",name); + if (any.isType()) { + ekat::any_cast(any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any.isType()) { + ekat::any_cast(any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any.isType()) { + ekat::any_cast(any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any.isType()) { + ekat::any_cast(any) = scorpio::get_attribute(filename,"GLOBAL",name); + } else if (any.isType()) { + ekat::any_cast(any) = scorpio::get_attribute(filename,"GLOBAL",name); } else { EKAT_ERROR_MSG ( "Error! Unrecognized/unsupported concrete type for restart extra data.\n" " - extra data name : " + name + "\n" - " - extra data typeid: " + any->content().type().name() + "\n"); + " - extra data typeid: " + any.content().type().name() + "\n"); } } diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index b550348fa49b..dfd116b0832b 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -52,7 +52,8 @@ HommeDynamics::HommeDynamics (const ekat::Comm& comm, const ekat::ParameterList& // This class needs Homme's context, so register as a user HommeContextUser::singleton().add_user(); - auto homme_nsteps = std::make_shared(-1); + ekat::any homme_nsteps; + homme_nsteps.reset(-1); m_restart_extra_data["homme_nsteps"] = homme_nsteps; if (!is_parallel_inited_f90()) { @@ -516,7 +517,7 @@ void HommeDynamics::run_impl (const double dt) // Update nstep in the restart extra data, so it can be written to restart if needed. const auto& tl = c.get(); - auto& nstep = ekat::any_cast(*m_restart_extra_data["homme_nsteps"]); + auto& nstep = ekat::any_cast(m_restart_extra_data["homme_nsteps"]); nstep = tl.nstep; // Post process Homme's output, to produce what the rest of Atm expects @@ -955,7 +956,7 @@ void HommeDynamics::restart_homme_state () { auto& tl = c.get(); // For BFB restarts, set nstep counter in Homme's TimeLevel to match the restarted value. - const auto& nstep = ekat::any_ptr_cast(*m_restart_extra_data["homme_nsteps"]); + const auto& nstep = ekat::any_ptr_cast(m_restart_extra_data["homme_nsteps"]); tl.nstep = *nstep; set_homme_param("num_steps",*nstep); @@ -1135,7 +1136,7 @@ void HommeDynamics::initialize_homme_state () { const int n0 = tl.n0; const int n0_qdp = tl.n0_qdp; - ekat::any_cast(*m_restart_extra_data["homme_nsteps"]) = tl.nstep; + ekat::any_cast(m_restart_extra_data["homme_nsteps"]) = tl.nstep; const auto phis_dyn_view = m_helper_fields.at("phis_dyn").get_view(); const auto phi_int_view = m_helper_fields.at("phi_int_dyn").get_view(); diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.hpp b/components/eamxx/src/share/atm_process/atmosphere_process.hpp index da8f7c09debf..071ef906a751 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.hpp @@ -78,7 +78,6 @@ class AtmosphereProcess : public ekat::enable_shared_from_this; template using strmap_t = std::map; @@ -268,8 +267,8 @@ class AtmosphereProcess : public ekat::enable_shared_from_this ekat::any // - the data_name is unique across the whole atm // The AD will take care of ensuring these are written/read to/from restart files. - const strmap_t& get_restart_extra_data () const { return m_restart_extra_data; } - strmap_t& get_restart_extra_data () { return m_restart_extra_data; } + const strmap_t& get_restart_extra_data () const { return m_restart_extra_data; } + strmap_t& get_restart_extra_data () { return m_restart_extra_data; } // Boolean that dictates whether or not the conservation checks are run for this process bool has_column_conservation_check () { return m_column_conservation_check_data.has_check; } @@ -477,7 +476,7 @@ class AtmosphereProcess : public ekat::enable_shared_from_this m_atm_logger; // Extra data needed for restart - strmap_t m_restart_extra_data; + strmap_t m_restart_extra_data; // Use at your own risk. Motivation: Free up device memory for a field that is // no longer used, such as a field read in the ICs used only to initialize From 4c6c85294e3928b3c64b1832eacea0e46dfccef0 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 11:44:32 -0600 Subject: [PATCH 144/476] EAMxx: fix handling of special tag names from grids We cannot pass the layout anymore, since every layout may be different. Instead, simply query the grid to see if there is a special tag name, and if so retrieve it. Then, handle the situation from outside the grid. --- .../eamxx/src/share/grid/abstract_grid.hpp | 6 +- .../eamxx/src/share/io/scorpio_input.cpp | 19 +++++-- .../eamxx/src/share/io/scorpio_output.cpp | 57 +++++++++---------- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 5da06c1edff9..2098fd49e946 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -188,10 +188,8 @@ class AbstractGrid : public ekat::enable_shared_from_this virtual bool check_valid_lid_to_idx () const { return true; } void reset_field_tag_name (const FieldTag t, const std::string& s) { m_special_tag_names[t] = s; } - std::string get_dim_name (const FieldLayout& lt, const int idim) const { - const auto t = lt.tag(idim); - return m_special_tag_names.count(t)==1 ? m_special_tag_names.at(t) : lt.names()[idim]; - } + bool has_special_tag_name (const FieldTag t) const { return m_special_tag_names.count(t)==1; } + std::string get_special_tag_name (const FieldTag t) const { return m_special_tag_names.at(t); } // This member is used mostly by IO: if a field exists on multiple grids // with the same name, IO can use this as a suffix to diambiguate the fields in diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 51c31f70a50a..24fe37a65715 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -399,10 +399,16 @@ AtmosphereInput::get_vec_of_dims(const FieldLayout& layout) std::vector dims_names; dims_names.reserve(layout.rank()); for (int i=0; iget_dim_name(layout,i)); - if (dims_names.back()=="dim") { - dims_names.back() += std::to_string(layout.dim(i)); - } + const auto t = layout.tag(i); + std::string n = m_io_grid->has_special_tag_name(t) + ? m_io_grid->get_special_tag_name(t) + : layout.names()[i]; + + // If t==CMP, and the name stored in the layout is the default ("dim"), + // we append also the extent, to allow different vector dims in the file + n += n=="dim" ? std::to_string(layout.dim(i)) : ""; + + dims_names.push_back(n); } return dims_names; @@ -431,7 +437,10 @@ void AtmosphereInput::set_decompositions() // Set the decomposition for the partitioned dimension const int local_dim = m_io_grid->get_partitioned_dim_local_size(); - auto decomp_dim = m_io_grid->get_dim_name(decomp_tag); + std::string decomp_dim = m_io_grid->has_special_tag_name(decomp_tag) + ? m_io_grid->get_special_tag_name(decomp_tag) + : e2str(decomp_tag); + auto gids_f = m_io_grid->get_partitioned_dim_gids(); auto gids_h = gids_f.get_view(); auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index e215d0bcc7aa..f167e1699654 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -754,14 +754,18 @@ void AtmosphereOutput::register_dimensions(const std::string& name) m_layouts.emplace(fid.name(),layout); // Now check taht all the dims of this field are already set to be registered. + const auto& tags = layout.tags(); + const auto& dims = layout.dims(); for (int i=0; iget_dim_name(layout,i); - if (tag_name=="dim") { - tag_name += std::to_string(dims[i]); - } + std::string tag_name = m_io_grid->has_special_tag_name(tags[i]) + ? m_io_grid->get_special_tag_name(tags[i]) + : layout.names()[i]; + + // If t==CMP, and the name stored in the layout is the default ("dim"), + // we append also the extent, to allow different vector dims in the file + tag_name += tag_name=="dim" ? std::to_string(layout.dim(i)) : ""; + auto is_partitioned = m_io_grid->get_partitioned_dim_tag()==tags[i]; int dim_len = is_partitioned ? m_io_grid->get_partitioned_dim_global_size() @@ -848,8 +852,16 @@ void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const Field const auto tags = layout.tags(); if (m_track_avg_cnt) { std::string avg_cnt_name = "avg_count" + avg_cnt_suffix; - for (int ii=0; iiget_dim_name(layout,ii); + for (int i=0; ihas_special_tag_name(t) + ? m_io_grid->get_special_tag_name(t) + : layout.names()[i]; + + // If t==CMP, and the name stored in the layout is the default ("dim"), + // we append also the extent, to allow different vector dims in the file + tag_name += tag_name=="dim" ? std::to_string(layout.dim(i)) : ""; + avg_cnt_name += "_" + tag_name; } if (std::find(m_avg_cnt_names.begin(),m_avg_cnt_names.end(),avg_cnt_name)==m_avg_cnt_names.end()) { @@ -906,28 +918,13 @@ register_variables(const std::string& filename, " - supported values: float, single, double, real\n"); // Helper lambdas - auto set_decomp_tag = [&](const FieldLayout& layout) { - std::string decomp_tag = "dt=real,grid-idx=" + std::to_string(m_io_grid->get_unique_grid_id()) + ",layout="; - - std::vector range(layout.rank()); - std::iota(range.begin(),range.end(),0); - auto tag_and_dim = [&](int i) { - return m_io_grid->get_dim_name(layout,i) + - std::to_string(layout.dim(i)); - }; - - decomp_tag += ekat::join (range, tag_and_dim,"-"); - - if (m_add_time_dim) { - decomp_tag += "-time"; - } - return decomp_tag; - }; - auto set_vec_of_dims = [&](const FieldLayout& layout) { std::vector vec_of_dims; for (int i=0; iget_dim_name(layout,i); + const auto t = layout.tag(i); + auto tag_name = m_io_grid->has_special_tag_name(t) + ? m_io_grid->get_special_tag_name(t) + : layout.names()[i]; if (tag_name=="dim") { tag_name += std::to_string(layout.dim(i)); } @@ -947,7 +944,6 @@ register_variables(const std::string& filename, // We use real here because the data type for the decomp is the one used // in the simulation and not the one used in the output file. const auto& layout = fid.get_layout(); - const auto& io_decomp_tag = set_decomp_tag(layout); auto vec_of_dims = set_vec_of_dims(layout); std::string units = fid.get_units().get_string(); @@ -1035,7 +1031,6 @@ register_variables(const std::string& filename, if (m_track_avg_cnt) { for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); - auto io_decomp_tag = set_decomp_tag(layout); auto vec_of_dims = set_vec_of_dims(layout); scorpio::define_var(filename, name, "unitless", vec_of_dims, "real",fp_precision, m_add_time_dim); @@ -1148,7 +1143,9 @@ void AtmosphereOutput::set_decompositions(const std::string& filename) // Set the decomposition for the partitioned dimension const int local_dim = m_io_grid->get_partitioned_dim_local_size(); - auto decomp_dim = m_io_grid->get_dim_name(decomp_tag); + std::string decomp_dim = m_io_grid->has_special_tag_name(decomp_tag) + ? m_io_grid->get_special_tag_name(decomp_tag) + : e2str(decomp_tag); auto gids_f = m_io_grid->get_partitioned_dim_gids(); auto gids_h = gids_f.get_view(); auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); From 2d6212326d7a4cf97c7df3b886aef5a294e3f301 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 11:46:21 -0600 Subject: [PATCH 145/476] EAMxx: removed unused commented code from early iterations --- .../eamxx/src/share/io/scorpio_input.cpp | 2 +- .../src/share/io/scream_output_manager.cpp | 2 +- .../src/share/io/scream_scorpio_types.hpp | 19 ------------------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 24fe37a65715..c847fe386fbf 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -357,7 +357,7 @@ void AtmosphereInput::init_scorpio_structures() // Some input files have the "time" dimension as non-unlimited. This messes up our // scorpio interface. To avoid trouble, if a dim called 'time' is present we - // treat is as unlimited, even though it isn't. + // treat it as unlimited, even though it isn't. if (scorpio::has_dim(m_filename,"time")) { scorpio::pretend_dim_is_unlimited(m_filename,"time"); } diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 33e40978e2d4..35a830465bb0 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -810,7 +810,7 @@ void OutputManager::set_file_header(const IOFileSpecs& file_specs) set_str_att("atm_initial_conditions_file",p.get("initial_conditions_file","NONE")); set_str_att("topography_file",p.get("topography_file","NONE")); set_str_att("contact","e3sm-data-support@llnl.gov"); - set_str_att("institution_id","E3SM-Projet"); + set_str_att("institution_id","E3SM-Project"); set_str_att("realm","atmos"); set_str_att("history",ts_str); set_str_att("Conventions","CF-1.8"); diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp index 4781f7ad62ef..645b1b26937b 100644 --- a/components/eamxx/src/share/io/scream_scorpio_types.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -59,11 +59,6 @@ struct PIOFileEntity : public PIOEntity { int fid = -1; // Allow retrieving file from the entity }; -// struct PIOAtt : public PIOFileEntity { -// int varid = -1; -// ekat::any val; -// }; - // A dimension struct PIODim : public PIOFileEntity { int length = -1; @@ -88,24 +83,11 @@ struct PIODecomp : public PIOEntity { std::shared_ptr dim; }; -// // A decomposition of a single dimension -// struct PIODimDecomp : public PIOEntity { -// std::shared_ptr dim; // The dim being decomposed -// std::vector offsets; // Owned offsets along decomposed dimension -// }; - -// // A decomposition of a full variable layout -// struct PIOVarDecomp : public PIOEntity { -// std::shared_ptr dim_decomp; // The decomposition along the dimension -// std::vector offsets; // Owned offsets along the full var layout -// }; - // A variable struct PIOVar : public PIOFileEntity { // Note: if time_dep=true, we will add it to the list of dims passed // to scorpio, but the time dim will not appear in this list. std::vector> dims; - // std::map> atts; std::vector dim_names () const { std::vector n; @@ -136,7 +118,6 @@ struct PIOVar : public PIOFileEntity { struct PIOFile : public PIOEntity { std::map> dims; std::map> vars; - // std::map> atts; std::shared_ptr time_dim; FileMode mode; From 44c0ec0f2d1ec25211ae8548bcef40dcc69ae012 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 12:04:23 -0600 Subject: [PATCH 146/476] EAMxx: fix remap weights creation in shoc_p3_nudged test --- .../shoc_p3_nudging/create_vert_remap_and_weights.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp index 6224ad0cf29d..8cedad048a64 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp @@ -57,11 +57,14 @@ void create_nudging_weights_ncfile(int ntimes, int ncols, int nlevs, const std:: } } + // NOTE: we are generating a file with "time" dimension that is not unlimited. + // This should stress test AtmosphereInput, which will 'pretend' that + // a dimension called 'time' is unlimited, even though it isn't in the file scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::define_dim(filename,"ncol",ncols); scorpio::define_dim(filename,"lev", nlevs); scorpio::define_dim(filename,"time",ntimes); - scorpio::define_var(filename,"nudging_weights",{"lev", "ncol", "time"},"real"); + scorpio::define_var(filename,"nudging_weights",{"time","ncol","lev"},"real"); scorpio::enddef(filename); scorpio::write_var(filename,"nudging_weights",&weights[0][0][0]); scorpio::release_file(filename); From 080fa4235c7af86dcf4bedaea7db63d57b0b229f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 12:04:32 -0600 Subject: [PATCH 147/476] EAMxx: fix warning --- components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp | 2 +- components/eamxx/src/share/io/scorpio_output.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp index 44b4cd851b9a..b7867c3ac726 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_level_tests.cpp @@ -74,7 +74,7 @@ TEST_CASE("field_at_level") using IPDF = std::uniform_int_distribution; IPDF ipdf (1,nlevs-2); - for (const std::string& lev_loc : {"model_top", "model_bot", "rand"}) { + for (const std::string lev_loc : {"model_top", "model_bot", "rand"}) { int lev; std::string lev_str; if (lev_loc=="bot") { diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index f167e1699654..b1ce7d3f3266 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -764,7 +764,7 @@ void AtmosphereOutput::register_dimensions(const std::string& name) // If t==CMP, and the name stored in the layout is the default ("dim"), // we append also the extent, to allow different vector dims in the file - tag_name += tag_name=="dim" ? std::to_string(layout.dim(i)) : ""; + tag_name += tag_name=="dim" ? std::to_string(dims[i]) : ""; auto is_partitioned = m_io_grid->get_partitioned_dim_tag()==tags[i]; int dim_len = is_partitioned From 4fcf2fa73d378a26f33c0829f3261e2b9cdff24b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 13:01:28 -0600 Subject: [PATCH 148/476] EAMxx: expose utility to get datatype string from type in scorpio interface --- .../src/share/io/scream_scorpio_interface.cpp | 20 ---------------- .../src/share/io/scream_scorpio_interface.hpp | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 472d57874810..95c4a29070fa 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -192,26 +192,6 @@ std::string refine_dtype (const std::string& dtype) { } } -template -std::string get_dtype () { - using raw_t = typename std::remove_cv::type; - std::string s; - if (std::is_same::value) { - s = "int"; - } else if (std::is_same::value) { - s = "float"; - } else if (std::is_same::value) { - s = "double"; - } else if (std::is_integral::value && - std::is_signed::value && - sizeof(raw_t)==sizeof(long long)) { - s = "int64"; - } else { - EKAT_ERROR_MSG ("Error! Invalid/unsupported data type.\n"); - } - return s; -} - size_t dtype_size (const std::string& dtype) { if (dtype=="int") { return sizeof(int); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 2ef09e96188a..e45e4f310339 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -4,6 +4,7 @@ #include "scream_scorpio_types.hpp" #include +#include #include #include @@ -33,6 +34,28 @@ inline std::string default_time_name () { return "time"; } // real -> float or double (depending on SCREAM_DOUBLE_PRECISION) std::string refine_dtype (const std::string& dtype); +template +std::string get_dtype () { + using raw_t = typename std::remove_cv::type; + std::string s; + if (std::is_same::value) { + s = "int"; + } else if (std::is_same::value) { + s = "float"; + } else if (std::is_same::value) { + s = "double"; + } else if (std::is_integral::value && + std::is_signed::value && + sizeof(raw_t)==sizeof(long long)) { + s = "int64"; + } else if (std::is_same::value) { + s = "char"; + } else { + EKAT_ERROR_MSG ("Error! Invalid/unsupported data type.\n"); + } + return s; +} + // =================== Global operations ================= // void init_subsystem(const ekat::Comm& comm, const int atm_id = 0); From 2813091051c5e7648a6c311e4253dd699f827fe1 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 13:03:16 -0600 Subject: [PATCH 149/476] EAMxx: improve PeekFile utility inside scorpio interface, and use it more --- .../src/share/io/scream_scorpio_interface.cpp | 69 ++++++++----------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 95c4a29070fa..579f6c09f250 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -222,6 +222,8 @@ struct PeekFile { if (not was_open) { register_file(filename,Read); } + auto& s = ScorpioSession::instance(); + file = &s.files[filename]; } ~PeekFile () { @@ -234,9 +236,10 @@ struct PeekFile { release_file(filename); } } -private: - std::string filename; - bool was_open; + + const PIOFile* file; + std::string filename; + bool was_open; }; PIOFile& get_file (const std::string& filename, @@ -625,10 +628,8 @@ bool has_dim (const std::string& filename, // If file wasn't open, open it on the fly. See comment in PeekFile class above. impl::PeekFile pf(filename); - const auto& f = impl::get_file(filename,"scorpio::has_dim"); - - auto it = f.dims.find(dimname); - if (it==f.dims.end()) { + auto it = pf.file->dims.find(dimname); + if (it==pf.file->dims.end()) { return false; } else if (length==0) { return it->second->unlimited; @@ -648,10 +649,7 @@ int get_dimlen (const std::string& filename, const std::string& dimname) " - filename: " + filename + "\n" " - dimname : " + dimname + "\n"); - auto& s = ScorpioSession::instance(); - auto& f = s.files[filename]; - - return f.dims.at(dimname)->length; + return pf.file->dims.at(dimname)->length; } int get_dimlen_local (const std::string& filename, const std::string& dimname) @@ -664,10 +662,7 @@ int get_dimlen_local (const std::string& filename, const std::string& dimname) " - filename: " + filename + "\n" " - dimname : " + dimname + "\n"); - auto& s = ScorpioSession::instance(); - auto& f = s.files[filename]; - - const auto& dim = f.dims.at(dimname); + const auto& dim = pf.file->dims.at(dimname); return dim->offsets==nullptr ? dim->length : dim->offsets->size(); } @@ -676,14 +671,12 @@ int get_time_len (const std::string& filename) // If file wasn't open, open it on the fly. See comment in PeekFile class above. impl::PeekFile pf(filename); - auto& s = ScorpioSession::instance(); - auto& f = s.files[filename]; - EKAT_REQUIRE_MSG (f.time_dim, + EKAT_REQUIRE_MSG (pf.file->time_dim!=nullptr, "Error! Could not inquire time dimension length. The time dimension is not in the file.\n" " - filename: " + filename + "\n"); - return f.time_dim->length; + return pf.file->time_dim->length; } std::string get_time_name (const std::string& filename) @@ -691,14 +684,11 @@ std::string get_time_name (const std::string& filename) // If file wasn't open, open it on the fly. See comment in PeekFile class above. impl::PeekFile pf(filename); - auto& s = ScorpioSession::instance(); - auto& f = s.files[filename]; - - EKAT_REQUIRE_MSG (f.time_dim, + EKAT_REQUIRE_MSG (pf.file->time_dim!=nullptr, "Error! Could not inquire time dimension name. The time dimension is not in the file.\n" " - filename: " + filename + "\n"); - return f.time_dim->name; + return pf.file->time_dim->name; } // =================== Decompositions operations ==================== // @@ -1031,9 +1021,7 @@ bool has_var (const std::string& filename, const std::string& varname) // If file wasn't open, open it on the fly. See comment in PeekFile class above. impl::PeekFile pf(filename); - const auto& f = impl::get_file(filename,"scorpio::has_var"); - - return f.vars.count(varname)==1; + return pf.file->vars.count(varname)==1; } const PIOVar& get_var (const std::string& filename, @@ -1108,8 +1096,7 @@ double get_time (const std::string& filename, const int time_index) { impl::PeekFile pf (filename); - const auto& f = impl::get_file(filename,"scorpio::get_time"); - const auto& time_name = f.time_dim->name; + const auto& time_name = pf.file->time_dim->name; double t; read_var(filename,time_name,&t,time_index); return t; @@ -1117,8 +1104,8 @@ double get_time (const std::string& filename, const int time_index) std::vector get_all_times (const std::string& filename) { - const auto& f = impl::get_file(filename,"scorpio::get_all_times"); - const auto& dim = *f.time_dim; + impl::PeekFile pf (filename); + const auto& dim = *pf.file->time_dim; std::vector times (dim.length); for (int i=0; incid; // Get var id int varid; @@ -1367,8 +1356,6 @@ bool has_attribute (const std::string& filename, const std::string& varname, con " - attname : " + attname + "\n" " - pio error: " + std::to_string(err) + "\n"); - release_file (filename); - return true; } @@ -1377,7 +1364,8 @@ T get_attribute (const std::string& filename, const std::string& varname, const std::string& attname) { - const auto& f = impl::get_file (filename,"scorpio::set_any_attribute"); + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); int varid; if (varname=="GLOBAL") { @@ -1387,7 +1375,7 @@ T get_attribute (const std::string& filename, } T val; - int err = PIOc_get_att(f.ncid,varid,attname.c_str(),reinterpret_cast(&val)); + int err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&val)); check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); return val; @@ -1413,7 +1401,8 @@ std::string get_attribute (const std::string& filename, const std::string& varname, const std::string& attname) { - const auto& f = impl::get_file (filename,"scorpio::set_any_attribute"); + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); int varid; if (varname=="GLOBAL") { @@ -1424,12 +1413,12 @@ std::string get_attribute (const std::string& filename, int err; PIO_Offset len; - err = PIOc_inq_attlen(f.ncid,varid,attname.c_str(),&len); + err = PIOc_inq_attlen(pf.file->ncid,varid,attname.c_str(),&len); check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","inq_attlen"); std::string val(len,'\0'); - err = PIOc_get_att(f.ncid,varid,attname.c_str(),val.data()); + err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),val.data()); check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); return val; From d1ba1b9d6e65554bed46eee0f35cc9d89469eac3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 13:04:03 -0600 Subject: [PATCH 150/476] EAMxx: add scorpio interface to query if a dimension is unlimited --- components/eamxx/src/share/io/scorpio_input.cpp | 2 +- .../src/share/io/scream_scorpio_interface.cpp | 15 ++++++++++++++- .../src/share/io/scream_scorpio_interface.hpp | 13 +++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index c847fe386fbf..13cabb3d6d52 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -358,7 +358,7 @@ void AtmosphereInput::init_scorpio_structures() // Some input files have the "time" dimension as non-unlimited. This messes up our // scorpio interface. To avoid trouble, if a dim called 'time' is present we // treat it as unlimited, even though it isn't. - if (scorpio::has_dim(m_filename,"time")) { + if (scorpio::has_dim(m_filename,"time") and not scorpio::is_dim_unlimited(m_filename,"time")) { scorpio::pretend_dim_is_unlimited(m_filename,"time"); } diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 579f6c09f250..ae4f93b023a9 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -666,11 +666,24 @@ int get_dimlen_local (const std::string& filename, const std::string& dimname) return dim->offsets==nullptr ? dim->length : dim->offsets->size(); } -int get_time_len (const std::string& filename) +bool is_dim_unlimited (const std::string& filename, + const std::string& dimname) { // If file wasn't open, open it on the fly. See comment in PeekFile class above. impl::PeekFile pf(filename); + EKAT_REQUIRE_MSG (has_dim(filename,dimname), + "Error! Could not inquire if dimension is unlimited. The dimension is not in the file.\n" + " - filename: " + filename + "\n" + " - dimname : " + dimname + "\n"); + + return pf.file->dims.at(dimname)->unlimited; +} + +int get_time_len (const std::string& filename) +{ + // If file wasn't open, open it on the fly. See comment in PeekFile class above. + impl::PeekFile pf(filename); EKAT_REQUIRE_MSG (pf.file->time_dim!=nullptr, "Error! Could not inquire time dimension length. The time dimension is not in the file.\n" diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index e45e4f310339..e5ecee90c6fc 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -20,6 +20,13 @@ * we throw any time scorpio returns something different from PIO_NOERR. * - allow using simpler interfaces: by using string/vector and templates, * the host code can have more compact interfaces. + * + * Most of the "get" interfaces can open a file in read mode on the fly. + * This allows easier usage from the user point of view, but has some drawbacks: + * - the file is open/closed every time you query it + * - the file is open with default IOType + * If you need to do several queries, you should consider registering the file manually + * before doing any query, and release it once you are done. */ namespace scream { @@ -93,6 +100,12 @@ bool has_dim (const std::string& filename, int get_dimlen (const std::string& filename, const std::string& dimname); int get_dimlen_local (const std::string& filename, const std::string& dimname); +// Checks if the dimension is unlimited +bool is_dim_unlimited (const std::string& filename, + const std::string& dimname); + +// Get len/name of the time dimension (i.e., the unlimited one) +// NOTE: these throw if time dim is not present. Use has_dim to check first. int get_time_len (const std::string& filename); std::string get_time_name (const std::string& filename); From d2c56132239dad087454e359bc5c8932186aa6ee Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 13:04:21 -0600 Subject: [PATCH 151/476] EAMxx: add checks on dims lengths in AtmosphereInput --- .../eamxx/src/share/io/scorpio_input.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 13cabb3d6d52..e8a510130e03 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -364,8 +364,10 @@ void AtmosphereInput::init_scorpio_structures() // Check variables are in the input file for (auto const& name : m_fields_names) { + const auto& layout = m_layouts.at(name); + // Determine the IO-decomp and construct a vector of dimension ids for this variable: - auto vec_of_dims = get_vec_of_dims(m_layouts.at(name)); + auto vec_of_dims = get_vec_of_dims(layout); // Check that the variable is in the file. EKAT_REQUIRE_MSG (scorpio::has_var(m_filename,name), @@ -375,12 +377,28 @@ void AtmosphereInput::init_scorpio_structures() const auto& var = scorpio::get_var(m_filename,name); EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, - "Error! Dimensions mismatch for input file variable.\n" + "Error! Layout mismatch for input file variable.\n" " - filename: " + m_filename + "\n" " - varname : " + name + "\n" " - expected dims : " + ekat::join(vec_of_dims,",") + "\n" " - dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + // Check that all dims for this var match the ones on file + for (int i=0; iget_partitioned_dim_tag()==layout.tag(i); + const int eamxx_len = partitioned ? m_io_grid->get_partitioned_dim_global_size() + : layout.dim(i); + EKAT_REQUIRE_MSG (eamxx_len==file_len, + "Error! Dimension mismatch for input file variable.\n" + " - filename : " + m_filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - dim name : " + vec_of_dims[i] + "\n" + " - expected extent : " + std::to_string(eamxx_len) + "\n" + " - extent from file: " + std::to_string(file_len) + "\n"); + } + // Ensure that we can read the var using Real data type scorpio::change_var_dtype (m_filename,name,"real"); } From e135e1c0d458ad7a7cd1fe1ac1a0b52ed57d36a6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 13:04:46 -0600 Subject: [PATCH 152/476] EAMxx: some fixes to IOP usage of new scorpio interfaces --- .../eamxx/src/control/intensive_observation_period.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 4f348747c0fd..46c02b8435df 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -44,6 +44,7 @@ void read_dimensionless_variable_from_file(const std::string& filename, "Error! IOP file does not have a required variable.\n" " - filename: " + filename + "\n" " - varname : " + varname + "\n"); + scorpio::change_var_dtype(filename,varname,scorpio::get_dtype()); scorpio::read_var(filename,varname,value); scorpio::release_file(filename); } @@ -69,6 +70,8 @@ void read_variable_from_file(const std::string& filename, // Read into data scorpio::register_file(filename, scorpio::FileMode::Read); + // This allows, e.g., to use double* to read a var with float dtype + scorpio::change_var_dtype(filename, varname, scorpio::get_dtype()); scorpio::read_var(filename, varname, data, time_idx); scorpio::release_file(filename); } @@ -128,6 +131,11 @@ initialize_iop_file(const util::TimeStamp& run_t0, const auto iop_file = m_params.get("iop_file"); + // All the scorpio::has_var call can open the file on the fly, but since there + // are a lot of those calls, for performance reasons we just open it now. + // All the calls to register_file made on-the-fly inside has_var will be no-op. + scorpio::register_file(iop_file,scorpio::FileMode::Read); + // Lambda for allocating space and storing information for potential iop fields. // Inputs: // - varnames: Vector of possible variable names in the iop file. @@ -333,6 +341,8 @@ initialize_iop_file(const util::TimeStamp& run_t0, model_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); model_pressure.allocate_view(); m_helper_fields.insert({"model_pressure", model_pressure}); + + scorpio::release_file(iop_file); } void IntensiveObservationPeriod:: From c58423008fc6791b4943ef78732aecadb549afa8 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 13:25:02 -0600 Subject: [PATCH 153/476] EAMxx: make IOP use directly the scorpio interfaces --- .../control/intensive_observation_period.cpp | 110 ++++++++---------- .../control/intensive_observation_period.hpp | 2 +- .../src/share/io/scream_scorpio_interface.cpp | 1 + 3 files changed, 48 insertions(+), 65 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 46c02b8435df..0b20a3ca9ffe 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -30,53 +30,6 @@ namespace ekat { namespace scream { namespace control { -// Helper functions for reading data from .nc file to support -// cases not currently supported in EAMxx scorpio interface. -namespace { -// Read the value of a dimensionless variable from file. -template -void read_dimensionless_variable_from_file(const std::string& filename, - const std::string& varname, - T* value) -{ - scorpio::register_file(filename,scorpio::FileMode::Read); - EKAT_REQUIRE_MSG(scorpio::has_var(filename,varname), - "Error! IOP file does not have a required variable.\n" - " - filename: " + filename + "\n" - " - varname : " + varname + "\n"); - scorpio::change_var_dtype(filename,varname,scorpio::get_dtype()); - scorpio::read_var(filename,varname,value); - scorpio::release_file(filename); -} - -// Read variable with arbitrary number of dimensions from file. -template -void read_variable_from_file(const std::string& filename, - const std::string& varname, - const std::string& vartype, - const std::vector& dimnames, - const int time_idx, - T* data) -{ - EKAT_REQUIRE_MSG(scorpio::has_var(filename,varname), - "Error! IOP file does not have variable "+varname+".\n"); - - // Compute total size of data to read - int data_size = 1; - for (auto dim : dimnames) { - const auto dim_len = scorpio::get_dimlen(filename, dim); - data_size *= dim_len; - } - - // Read into data - scorpio::register_file(filename, scorpio::FileMode::Read); - // This allows, e.g., to use double* to read a var with float dtype - scorpio::change_var_dtype(filename, varname, scorpio::get_dtype()); - scorpio::read_var(filename, varname, data, time_idx); - scorpio::release_file(filename); -} -} - IntensiveObservationPeriod:: IntensiveObservationPeriod(const ekat::Comm& comm, const ekat::ParameterList& params, @@ -122,6 +75,13 @@ IntensiveObservationPeriod(const ekat::Comm& comm, initialize_iop_file(run_t0, model_nlevs); } +IntensiveObservationPeriod:: +~IntensiveObservationPeriod () +{ + const auto iop_file = m_params.get("iop_file"); + scorpio::release_file(iop_file); +} + void IntensiveObservationPeriod:: initialize_iop_file(const util::TimeStamp& run_t0, int model_nlevs) @@ -278,7 +238,10 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_var(iop_file, "basedate")) bdate_name = "basedate"; else if (scorpio::has_var(iop_file, "nbdate")) bdate_name = "nbdate"; else EKAT_ERROR_MSG("Error! No valid name for bdate in "+iop_file+".\n"); - read_dimensionless_variable_from_file(iop_file, bdate_name, &bdate); + + // Just in case bdate_name is stored with a type that is not "int" (e.g., it's int64) + scorpio::change_var_dtype(iop_file,bdate_name,"int"); + scorpio::read_var(iop_file, bdate_name, &bdate); int yr=bdate/10000; int mo=(bdate/100) - yr*100; @@ -289,11 +252,17 @@ initialize_iop_file(const util::TimeStamp& run_t0, if (scorpio::has_dim(iop_file, "time")) time_dimname = "time"; else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); - m_time_info.iop_file_times_in_sec = - decltype(m_time_info.iop_file_times_in_sec)("iop_file_times", ntimes); - read_variable_from_file(iop_file, "tsec", "int", {time_dimname}, -1, - m_time_info.iop_file_times_in_sec.data()); + m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); + // Just in case "tsec" is stored with a different data type + scorpio::change_var_dtype(iop_file,"tsec","int"); + scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); + + // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues + if (not scorpio::is_dim_unlimited(iop_file,time_dimname)) { + scorpio::pretend_dim_is_unlimited(iop_file,time_dimname); + } // Check that lat/lon from iop file match the targets in parameters. Note that // longitude may be negtive in the iop file, we convert to positive before checking. @@ -301,8 +270,13 @@ initialize_iop_file(const util::TimeStamp& run_t0, const auto nlons = scorpio::get_dimlen(iop_file, "lon"); EKAT_REQUIRE_MSG(nlats==1 and nlons==1, "Error! IOP data file requires a single lat/lon pair.\n"); Real iop_file_lat, iop_file_lon; - read_variable_from_file(iop_file, "lat", "real", {"lat"}, -1, &iop_file_lat); - read_variable_from_file(iop_file, "lon", "real", {"lon"}, -1, &iop_file_lon); + + // Just in case "lat/lon" are stored with a different data type (e.g., float instead of double) + scorpio::change_var_dtype(iop_file,"lat","real"); + scorpio::change_var_dtype(iop_file,"lon","real"); + scorpio::read_var(iop_file,"lat",&iop_file_lat); + scorpio::read_var(iop_file,"lon",&iop_file_lon); + const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ m_params.get("target_latitude"); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ @@ -326,7 +300,10 @@ initialize_iop_file(const util::TimeStamp& run_t0, iop_file_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); iop_file_pressure.allocate_view(); auto data = iop_file_pressure.get_view().data(); - read_variable_from_file(iop_file, "lev", "real", {"lev"}, -1, data); + // Just in case "lev" is stored with a different data type (e.g., float instead of double) + scorpio::change_var_dtype(iop_file,"lev","real"); + scorpio::read_var(iop_file,"lev",data); + // Convert to pressure to millibar (file gives pressure in Pa) for (int ilev=0; ilev().data(); - read_variable_from_file(iop_file, "Ps", "real", {"lon","lat"}, iop_file_time_idx, ps_data); + + // Just in case "Ps" is stored with a different data type (e.g., float instead of double) + scorpio::change_var_dtype(iop_file,"Ps","real"); + scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); // Pre-process file pressures, store number of file levels @@ -661,7 +639,9 @@ read_iop_file_data (const util::TimeStamp& current_ts) if (field.rank()==0) { // For scalar data, read iop file variable directly into field data auto data = field.get_view().data(); - read_variable_from_file(iop_file, file_varname, "real", {"lon","lat"}, iop_file_time_idx, data); + // Just in case the var is stored with a different data type (e.g., float instead of double) + scorpio::change_var_dtype(iop_file,file_varname,"real"); + scorpio::read_var(iop_file,file_varname,data,iop_file_time_idx); field.sync_to_dev(); } else if (field.rank()==1) { // Create temporary fields for reading iop file variables. We use @@ -678,7 +658,9 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Read data from iop file. std::vector data(file_levs); - read_variable_from_file(iop_file, file_varname, "real", {"lon","lat","lev"}, iop_file_time_idx, data.data()); + // Just in case the var is stored with a different data type (e.g., float instead of double) + scorpio::change_var_dtype(iop_file,file_varname,"real"); + scorpio::read_var(iop_file,file_varname,data.data(),iop_file_time_idx); // Copy first adjusted_file_levs-1 values to field auto iop_file_v_h = iop_file_field.get_view(); @@ -688,7 +670,9 @@ read_iop_file_data (const util::TimeStamp& current_ts) const auto has_srf = m_iop_field_surface_varnames.count(fname)>0; if (has_srf) { const auto srf_varname = m_iop_field_surface_varnames[fname]; - read_variable_from_file(iop_file, srf_varname, "real", {"lon","lat"}, iop_file_time_idx, &iop_file_v_h(adjusted_file_levs-1)); + // Just in case the var is stored with a different data type (e.g., float instead of double) + scorpio::change_var_dtype(iop_file,srf_varname,"real"); + scorpio::read_var(iop_file,srf_varname,&iop_file_v_h(adjusted_file_levs-1),iop_file_time_idx); } else { // No surface value exists, compute surface value const auto dx = iop_file_v_h(adjusted_file_levs-2) - iop_file_v_h(adjusted_file_levs-3); @@ -739,7 +723,7 @@ read_iop_file_data (const util::TimeStamp& current_ts) KOKKOS_LAMBDA (const int ilev) { iop_field_v(ilev) = iop_file_v(0); }); - Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end-1, total_nlevs), + Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end-1, total_nlevs), KOKKOS_LAMBDA (const int ilev) { iop_field_v(ilev) = iop_file_v(adjusted_file_levs-1); }); @@ -915,5 +899,3 @@ correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr) } // namespace control } // namespace scream - - diff --git a/components/eamxx/src/control/intensive_observation_period.hpp b/components/eamxx/src/control/intensive_observation_period.hpp index 6c0b3640ffb2..be3d389257bd 100644 --- a/components/eamxx/src/control/intensive_observation_period.hpp +++ b/components/eamxx/src/control/intensive_observation_period.hpp @@ -55,7 +55,7 @@ class IntensiveObservationPeriod const Field& hybm); // Default destructor - ~IntensiveObservationPeriod() = default; + ~IntensiveObservationPeriod(); // Read data from IOP file and store internally. void read_iop_file_data(const util::TimeStamp& current_ts); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index ae4f93b023a9..3a19ff4a0108 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -1018,6 +1018,7 @@ void change_var_dtype (const std::string& filename, const std::string& varname, auto& var = impl::get_var(filename,varname,"scorpio::change_var_dtype"); if (refine_dtype(dtype)==refine_dtype(var.dtype)) { + // The type is not changing, nothing to do return; } From a638b70fea731b09b257a3a284f0598b6a61c5c3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 26 Apr 2024 16:34:08 -0600 Subject: [PATCH 154/476] EAMxx: manually delete iop in AD finalize, to avoid hanging files --- components/eamxx/src/control/atmosphere_driver.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index f30742574fb7..4f2c0fc21b4c 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1663,6 +1663,9 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { m_atm_process_group = nullptr; } + // Destroy iop + m_iop = nullptr; + // Destroy the buffer manager m_memory_buffer = nullptr; From bcdc820eac12a7ef1ed939caa8bc9456b4232626 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 29 Apr 2024 15:37:24 -0600 Subject: [PATCH 155/476] EAMxx: use 'if constexpr' in a scorpio utility --- .../eamxx/src/share/io/scream_scorpio_interface.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index e5ecee90c6fc..6af6f7ea7c7e 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -45,17 +45,17 @@ template std::string get_dtype () { using raw_t = typename std::remove_cv::type; std::string s; - if (std::is_same::value) { + if constexpr (std::is_same::value) { s = "int"; - } else if (std::is_same::value) { + } else if constexpr (std::is_same::value) { s = "float"; - } else if (std::is_same::value) { + } else if constexpr (std::is_same::value) { s = "double"; - } else if (std::is_integral::value && + } else if constexpr (std::is_integral::value && std::is_signed::value && sizeof(raw_t)==sizeof(long long)) { s = "int64"; - } else if (std::is_same::value) { + } else if constexpr (std::is_same::value) { s = "char"; } else { EKAT_ERROR_MSG ("Error! Invalid/unsupported data type.\n"); From 29dd63b8e0eb13e7153f5f5805c0ae3ff7661570 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 29 Apr 2024 15:37:38 -0600 Subject: [PATCH 156/476] EAMxx: automatically change var dtype inside io interfaces --- .../control/intensive_observation_period.cpp | 17 ------- .../src/share/io/scream_scorpio_interface.cpp | 47 ++++++++----------- .../src/share/io/scream_scorpio_interface.hpp | 6 +++ 3 files changed, 26 insertions(+), 44 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index 0b20a3ca9ffe..ab01bfc7e3ed 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -239,8 +239,6 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_var(iop_file, "nbdate")) bdate_name = "nbdate"; else EKAT_ERROR_MSG("Error! No valid name for bdate in "+iop_file+".\n"); - // Just in case bdate_name is stored with a type that is not "int" (e.g., it's int64) - scorpio::change_var_dtype(iop_file,bdate_name,"int"); scorpio::read_var(iop_file, bdate_name, &bdate); int yr=bdate/10000; @@ -255,8 +253,6 @@ initialize_iop_file(const util::TimeStamp& run_t0, const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); - // Just in case "tsec" is stored with a different data type - scorpio::change_var_dtype(iop_file,"tsec","int"); scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues @@ -271,9 +267,6 @@ initialize_iop_file(const util::TimeStamp& run_t0, EKAT_REQUIRE_MSG(nlats==1 and nlons==1, "Error! IOP data file requires a single lat/lon pair.\n"); Real iop_file_lat, iop_file_lon; - // Just in case "lat/lon" are stored with a different data type (e.g., float instead of double) - scorpio::change_var_dtype(iop_file,"lat","real"); - scorpio::change_var_dtype(iop_file,"lon","real"); scorpio::read_var(iop_file,"lat",&iop_file_lat); scorpio::read_var(iop_file,"lon",&iop_file_lon); @@ -300,8 +293,6 @@ initialize_iop_file(const util::TimeStamp& run_t0, iop_file_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); iop_file_pressure.allocate_view(); auto data = iop_file_pressure.get_view().data(); - // Just in case "lev" is stored with a different data type (e.g., float instead of double) - scorpio::change_var_dtype(iop_file,"lev","real"); scorpio::read_var(iop_file,"lev",data); // Convert to pressure to millibar (file gives pressure in Pa) @@ -550,8 +541,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Load surface pressure (Ps) from iop file auto ps_data = surface_pressure.get_view().data(); - // Just in case "Ps" is stored with a different data type (e.g., float instead of double) - scorpio::change_var_dtype(iop_file,"Ps","real"); scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); @@ -639,8 +628,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) if (field.rank()==0) { // For scalar data, read iop file variable directly into field data auto data = field.get_view().data(); - // Just in case the var is stored with a different data type (e.g., float instead of double) - scorpio::change_var_dtype(iop_file,file_varname,"real"); scorpio::read_var(iop_file,file_varname,data,iop_file_time_idx); field.sync_to_dev(); } else if (field.rank()==1) { @@ -658,8 +645,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) // Read data from iop file. std::vector data(file_levs); - // Just in case the var is stored with a different data type (e.g., float instead of double) - scorpio::change_var_dtype(iop_file,file_varname,"real"); scorpio::read_var(iop_file,file_varname,data.data(),iop_file_time_idx); // Copy first adjusted_file_levs-1 values to field @@ -670,8 +655,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) const auto has_srf = m_iop_field_surface_varnames.count(fname)>0; if (has_srf) { const auto srf_varname = m_iop_field_surface_varnames[fname]; - // Just in case the var is stored with a different data type (e.g., float instead of double) - scorpio::change_var_dtype(iop_file,srf_varname,"real"); scorpio::read_var(iop_file,srf_varname,&iop_file_v_h(adjusted_file_levs-1),iop_file_time_idx); } else { // No surface value exists, compute surface value diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 3a19ff4a0108..0d4918ad230b 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -1012,11 +1012,12 @@ void define_var (const std::string& filename, const std::string& varname, define_var(filename,varname,"",dimensions,dtype,dtype,time_dependent); } -void change_var_dtype (const std::string& filename, const std::string& varname, - const std::string& dtype) +// This overload is not exposed externally. Also, filename is only +// used to print it in case there are errors +void change_var_dtype (PIOVar& var, + const std::string& dtype, + const std::string& filename) { - auto& var = impl::get_var(filename,varname,"scorpio::change_var_dtype"); - if (refine_dtype(dtype)==refine_dtype(var.dtype)) { // The type is not changing, nothing to do return; @@ -1030,6 +1031,14 @@ void change_var_dtype (const std::string& filename, const std::string& varname, } } +void change_var_dtype (const std::string& filename, + const std::string& varname, + const std::string& dtype) +{ + auto& var = impl::get_var(filename,varname,"scorpio::change_var_dtype"); + change_var_dtype(var,dtype,filename); +} + bool has_var (const std::string& filename, const std::string& varname) { // If file wasn't open, open it on the fly. See comment in PeekFile class above. @@ -1142,6 +1151,9 @@ void read_var (const std::string &filename, const std::string &varname, T* buf, const auto& f = impl::get_file(filename,"scorpio::read_var"); auto& var = impl::get_var(filename,varname,"scorpio::read_var"); + // If the input pointer type already matches var.dtype, this is a no-op + change_var_dtype(var,get_dtype(),filename); + int err; int frame = -1; if (var.time_dep) { @@ -1169,18 +1181,6 @@ void read_var (const std::string &filename, const std::string &varname, T* buf, " - first dim len: " + std::to_string(var.dims[0]->length) + "\n"); } - // The user may be using a different data type than the one that is in the file - // If that's the case, they should call `change_var_dtype` *before* trying to read it - if (get_dtype()!=var.dtype) { - EKAT_ERROR_MSG ( - "Error! You're attempting to read a variable with the wrong data type.\n" - "If you want to use a different data type, call change_var_dtype *before* attempting a read.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - var dtype: " + var.dtype + "\n" - " - ptr dtype: " + get_dtype() + "\n"); - } - std::string pioc_func; if (var.decomp) { // A decomposed variable, requires read_darray @@ -1245,6 +1245,10 @@ void write_var (const std::string &filename, const std::string &varname, const T const auto& f = impl::get_file(filename,"scorpio::write_var"); auto& var = impl::get_var(filename,varname,"scorpio::write_var"); + + // If the input pointer type already matches var.dtype, this is a no-op + change_var_dtype(var,get_dtype(),filename); + int err; if (var.time_dep) { @@ -1259,17 +1263,6 @@ void write_var (const std::string &filename, const std::string &varname, const T check_scorpio_noerr (err,f.name,"variable",varname,"write_var","setframe"); } - // The user may be using a different data type than the one that is in the file - if (get_dtype()!=var.dtype) { - EKAT_ERROR_MSG ( - "Error! You're attempting to write a variable with the wrong data type.\n" - "If you want to use a different data type, call change_var_dtype *before* attempting a write.\n" - " - filename : " + filename + "\n" - " - varname : " + varname + "\n" - " - var dtype: " + var.dtype + "\n" - " - ptr dtype: " + get_dtype() + "\n"); - } - std::string pioc_func; if (var.decomp) { // A decomposed variable, requires write_darray diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 6af6f7ea7c7e..549de680fe45 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -155,6 +155,12 @@ void define_var (const std::string& filename, const std::string& varname, // This is useful when reading data sets. E.g., if the pio file is storing // a var as float, but we need to read it as double, we need to call this. +// NOTE: read_var/write_var automatically change the dtype if the input +// pointer type does not match the var dtype. However, changing dtype +// forces a rebuild of the var decomp (if any). Hence, if you know +// the var WILL be read/written as decomposed, you should call this method +// BEFORE calling set_dim_decomp, so that the decomp is built directly +// with the correct data type (PIO decomps depend on var dtype). void change_var_dtype (const std::string& filename, const std::string& varname, const std::string& dtype); From d4bb336d4eb3b5a7f95b4c1ead965fccea490431 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 6 May 2024 18:48:00 -0600 Subject: [PATCH 157/476] EAMxx: small changes to model input docs Use subsections to get index on side of page --- components/eamxx/docs/user/model_input.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index fb25422846ec..a0076c02f89e 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -1,5 +1,4 @@ -Model inputs -===================================== +# Model inputs This section explains how input parameters are passed to EAMxx, and how the user can change their value. The full list of the currently configuraable runtime parameters for @@ -35,7 +34,7 @@ The infrastructure for reading inputs into EAMxx involves a few scripts/files: Since these files are automatically generated when `buildnml` runs, users should not manually modify them. Any manual modification will be lost the next time `buildnml` runs (e.g., at `case.submit` time). -# Querying parameters with `atmquery` +## Querying model inputs: atmquery This script is the simplest way for the user to check the value and properties of EAMxx input parameters. A basic usage of the script is @@ -131,7 +130,7 @@ $ ./atmquery --listall ... ``` -# Changing model inputs via `atmchange` +## Changing model inputs: atmchange When `buildnml` runs, the model inputs are deduced from the case configuration settings (e.g., the grid, the compset, etc.) and the `namelist_scream_defaults.xml` file, located in the eamxx source tree. @@ -238,7 +237,7 @@ $ ./atmquery homme::compute_tendencies --full valid values: [] ``` -# Modifying the list of atmosphere processes +### Modifying the list of atmosphere processes The `atmchange` script can be used to change any of the runtime parameters of EAMxx. In particular, it can be used to add, remove, or reorder atmosphere processes. When adding an atmosphere process, we must first From 65c04960d9fa26b7e5f8f1c95cba15e887dffb79 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 6 May 2024 18:48:10 -0600 Subject: [PATCH 158/476] EAMxx: rework model output docs page --- components/eamxx/docs/user/model_output.md | 56 ++++++++++------------ 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/components/eamxx/docs/user/model_output.md b/components/eamxx/docs/user/model_output.md index 1ef0c8b26761..84d351c0302f 100644 --- a/components/eamxx/docs/user/model_output.md +++ b/components/eamxx/docs/user/model_output.md @@ -1,11 +1,20 @@ # Model output EAMxx allows the user to configure the desired model output via [YAML](https://yaml.org/) files, -with each YAML file associated to a different output file. +with each YAML file associated to a different output file. In order to add an output stream, +one needs to run `atmchange output_yaml_files+=/path/to/my/output/yaml` (more information on how +to use `atmchange` can be found [here](./model_input.md#changing-model-inputs-atmchange)). +During the `buildnml` phase of the case management system, a copy of these YAML files will be copied +into the RUNDIR/data folder. During this process, the files will be parsed, and any CIME-related +variable will be resolved accordingly. Therefore, it is not advised to put the original YAML files in RUNDIR/data, +since upon `buildnml` execution, all the CIME vars will no longer be available in the YAML file, +making it harder to tweak it, and even harder to share with other users/cases. Another consequence +of this is that the user should not modify the yaml files in RUNDIR/data, since any modification will +be lost on the next run of `buildnml`. -## Basic output YAML file syntax +## Basic output -The following is an example of a simple output request. +The following is a basic example of an output request. ```yaml %YAML 1.1 @@ -30,7 +39,7 @@ output_control: Notice that lists can be equivalently specified in YAML as `Field Names: [f1, f2, f3]`. The user can specify fields to be outputted from any of the grids used in the simulation. In the example above, we requested fields from both the Physics and Dynamics grid. -The other parameters are +The meaning of the other parameters is as follows: - `Averaging Type`: how the fields are integrated in time before being saved. Valid options are @@ -61,7 +70,7 @@ request to output derived quantities, which will be computed on the fly by the I/O interface of EAMxx. There are two types of diagnostic outputs: - quantities computed as a function of EAMxx fields. These are simply physical quantities - that EAMxx does not keep in persistent storage. As of August 2023, the available + that EAMxx does not keep in persistent storage. As of May 2024, the available derived quantities are (case sensitive): - `PotentialTemperature` @@ -89,6 +98,12 @@ I/O interface of EAMxx. There are two types of diagnostic outputs: - `precip_ice_surf_mass_flux` - `precip_total_surf_mass_flux` - `surface_upward_latent_heat_flux` + - `wind_speed` + - `AerosolOpticalDepth550nm` + - `NumberPath` + - `AeroComCld` + + TODO: add some information about what each diagnostic is, perhaps a formula - lower-dimensional slices of a field. These are hyperslices of an existing field or of another diagnostic output. As of August 2023, given a field X, the available options @@ -120,34 +135,11 @@ they are computed on. where `ngp` is the number of Gauss points along each axis in the 2d spectral element. Note: this feature cannot be used along with the horizontal/vertical remapper. -## Add output stream to a CIME case - -In order to tell EAMxx that a new output stream is needed, one must add the name of -the yaml file to be used to the list of yaml files that EAMxx will process. From the -case folder, after `case.setup` has run, one can do +## Tendencies output -```shell -./atmchange output_yaml_files=/path/to/my/yaml/file -``` +EXPLAIN how to request tendencies on a per-process basis -to specify a single yaml file, or +## Additional options -```shell -./atmchange output_yaml_files+=/path/to/my/yaml/file -``` +LIST ADDITIONAL OPTIONS, like flush_frequency, file_max_storage_type, etc. -to append to the list of yaml files. - -### Important notes - -- The user should not specify a path to a file in `$RUNDIR/data`. EAMxx will -put a copy of the specified yaml files in that directory, pruning any existing copy -of that file. This happens every time that `buildnml` runs; in particular, it happens -during `case.submit`. -- As a consequence of the above, the user should not modify the generated yaml files - that are in `$RUNDIR/data`, since any modification will be lost on the next run - of `buildnml`. To modify output parmeters, the user should modify the yaml file - that was specified with the `atmchange` command. -- EAMxx will parse the yaml file and expand any string of the form $VAR, by looking - for the value of the variable VAR in the CIME case. If VAR is not a valid CIME - variable, an error will be raised. From 9918cc228ad179065eebdb3088d7e52c306d377d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 7 May 2024 07:19:04 -0700 Subject: [PATCH 159/476] correct comment string, use function call in shoc --- .../src/diagnostics/potential_temperature.cpp | 1 + .../tests/potential_temperature_test.cpp | 1 - .../physics/shoc/eamxx_shoc_process_interface.hpp | 8 ++------ .../share/util/scream_common_physics_functions.hpp | 14 ++++---------- .../util/scream_common_physics_functions_impl.hpp | 14 ++++++-------- 5 files changed, 13 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index 602f3ffc218a..988bb5552240 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -50,6 +50,7 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, ps); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); // Only needed for LiqPotentialTemperature, but put it here for ease + // TODO: only request it if it is needed add_field("qc", scalar3d_layout_mid, Q, grid_name, ps); // Construct and allocate the diagnostic field diff --git a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp index d82f85fa8ce6..1f584ec7e71c 100644 --- a/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp +++ b/components/eamxx/src/diagnostics/tests/potential_temperature_test.cpp @@ -91,7 +91,6 @@ void run(std::mt19937_64& engine, int int_ptype) auto diag = diag_factory.create("PotentialTemperature",comm,params); diag->set_grids(gm); - // Set the required fields for the diagnostic. std::map input_fields; for (const auto& req : diag->get_required_field_requests()) { diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 8d9feb496b82..d3f5b05f88da 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -75,7 +75,6 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const int i = team.league_rank(); const Real zvir = C::ZVIR; - const Real latvap = C::LatVap; const Real cpair = C::Cpair; const Real ggr = C::gravit; const Real inv_ggr = 1/ggr; @@ -107,12 +106,9 @@ class SHOCMacrophysics : public scream::AtmosphereProcess qw(i,k) = qv(i,k) + qc(i,k); // Temperature - // NOTE: vtheta (thv) is inconsistent with one in HOMME - // SEE gh issue #2813 and gh pr #2812 + // NOTE: theta_v (thv) is intentionally different from one in HOMME const auto theta_zt = PF::calculate_theta_from_T(T_mid(i,k),p_mid(i,k)); - // TODO? Can use PF::calculate_thetal_from_theta instead? (added in gh pr #2812) - thlm(i,k) = theta_zt-(theta_zt/T_mid(i,k))*(latvap/cpair)*qc(i,k); - // TODO? Can use PF::calculate_virtual_temperature instead? (also be consistent with HOMME) + thlm(i,k) = PF::calculate_thetal_from_theta(theta_zt,T_mid(i,k),qc(i,k)); thv(i,k) = theta_zt*(1 + zvir*qv(i,k) - qc(i,k)); // Vertical layer thickness diff --git a/components/eamxx/src/share/util/scream_common_physics_functions.hpp b/components/eamxx/src/share/util/scream_common_physics_functions.hpp index b926a7b7bc8e..87ae6604d999 100644 --- a/components/eamxx/src/share/util/scream_common_physics_functions.hpp +++ b/components/eamxx/src/share/util/scream_common_physics_functions.hpp @@ -133,14 +133,8 @@ struct PhysicsFunctions static ScalarT calculate_temperature_from_virtual_temperature(const ScalarT& T_virtual, const ScalarT& qv); //-----------------------------------------------------------------------------------------------// - // FIXME: this comment is different from what's actually happening in the code: - // FIXME: T_virtual = temperature * ( one + c1*qv ) where c1 = - one + one/ep_2 - // FIXME: The code seems to be correct, and the comment is wrong. - // FIXME: ep_2 = 0.622, so the comment produces vpt = pt * (0.75) for qv = 0.5 - // FIXME: but the code produces vpt = pt * (1.30) for qv = 0.5 - // XREFS: pr #1059, issue #2812 // Compute virtual temperature - // T_virtual = temperature * (qv+ep_2)/(qv+1) + // T_virtual = temperature * (1+(-1+1/ep_2)*qv) // where // ep_2 is ratio of molecular mass of water to the molecular mass of dry air // temperature is the atmospheric temperature. Units in [K]. @@ -411,12 +405,12 @@ struct PhysicsFunctions const InputProviderP& pressure, const view_1d& theta); - template + template KOKKOS_INLINE_FUNCTION static void calculate_thetal_from_theta(const MemberType& team, - const InputProviderT& theta, + const InputProviderTheta& theta, const InputProviderT& temperature, - const InputProviderP& qc, + const InputProviderQ& qc, const view_1d& thetal); template KOKKOS_INLINE_FUNCTION diff --git a/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp b/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp index 769622a30adb..f41e389bf67b 100644 --- a/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp +++ b/components/eamxx/src/share/util/scream_common_physics_functions_impl.hpp @@ -123,10 +123,8 @@ KOKKOS_INLINE_FUNCTION ScalarT PhysicsFunctions::calculate_thetal_from_theta(const ScalarT& theta, const ScalarT& temperature, const ScalarT& qc) { using C = scream::physics::Constants; - static constexpr auto cpair = C::Cpair; - static constexpr auto latvap = C::LatVap; - return theta - (theta / temperature) * (latvap/cpair) * qc; + return theta - (theta / temperature) * (C::LatVap/C::Cpair) * qc; } template @@ -144,13 +142,13 @@ void PhysicsFunctions::calculate_theta_from_T(const MemberType& team, } template -template +template KOKKOS_INLINE_FUNCTION void PhysicsFunctions::calculate_thetal_from_theta(const MemberType& team, - const InputProviderT& theta, - const InputProviderT& temperature, - const InputProviderP& qc, - const view_1d& thetal) + const InputProviderTheta& theta, + const InputProviderT& temperature, + const InputProviderQ& qc, + const view_1d& thetal) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team,thetal.extent(0)), [&] (const int k) { From 99a8b23b9d61a948bbbf56b099f3604c85e0cd87 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 7 May 2024 11:40:59 -0600 Subject: [PATCH 160/476] mam4xx - Replace KOKKOS_INLINE_FUNCTION with inline for routines that are only invoked by the host. --- .../eamxx/src/physics/mam/mam_coupling.hpp | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 69fccddccdbf..01dd80ae7770 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -76,7 +76,7 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. -KOKKOS_INLINE_FUNCTION +inline const char* aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { "1", @@ -89,7 +89,7 @@ const char* aero_mode_name(const int mode) { // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. -KOKKOS_INLINE_FUNCTION +inline const char* gas_species_name(const int gas_id) { static const char *species_names[num_aero_gases()] = { "O3", @@ -110,14 +110,14 @@ constexpr int max_field_name_len() { return 128; } -KOKKOS_INLINE_FUNCTION +inline size_t gpu_strlen(const char* s) { size_t l = 0; while (s[l]) ++l; return l; } -KOKKOS_INLINE_FUNCTION +inline void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); for (size_t i = 0; i < len1; ++i) @@ -128,7 +128,7 @@ void concat_2_strings(const char *s1, const char *s2, char *concatted) { concatted[len1+len2] = 0; } -KOKKOS_INLINE_FUNCTION +inline void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { size_t len1 = gpu_strlen(s1); for (size_t i = 0; i < len1; ++i) @@ -142,31 +142,31 @@ void concat_3_strings(const char *s1, const char *s2, const char *s3, char *conc concatted[len1+len2+len3] = 0; } -KOKKOS_INLINE_FUNCTION +inline char* int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } -KOKKOS_INLINE_FUNCTION +inline char* cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } -KOKKOS_INLINE_FUNCTION +inline char* int_aero_mmr_names(int mode, int species) { static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } -KOKKOS_INLINE_FUNCTION +inline char* cld_aero_mmr_names(int mode, int species) { static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } -KOKKOS_INLINE_FUNCTION +inline char* gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; @@ -176,7 +176,7 @@ char* gas_mmr_names(int gas_id) { // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") -KOKKOS_INLINE_FUNCTION +inline const char* int_aero_nmr_field_name(const int mode) { if (!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); @@ -186,7 +186,7 @@ const char* int_aero_nmr_field_name(const int mode) { // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") -KOKKOS_INLINE_FUNCTION +inline const char* cld_aero_nmr_field_name(const int mode) { if (!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); @@ -199,7 +199,7 @@ const char* cld_aero_nmr_field_name(const int mode) { // field in EAMxx. The form of the field name is "_a<1-based-mode-index>". // If the desired species is not present within the desire mode, returns a blank // string (""). -KOKKOS_INLINE_FUNCTION +inline const char* int_aero_mmr_field_name(const int mode, const int species) { if (!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); @@ -218,7 +218,7 @@ const char* int_aero_mmr_field_name(const int mode, const int species) { // field in EAMxx. The form of the field name is "_c<1-based-mode-index>". // If the desired species is not present within the desire mode, returns a blank // string (""). -KOKKOS_INLINE_FUNCTION +inline const char* cld_aero_mmr_field_name(const int mode, const int species) { if (!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); @@ -234,7 +234,7 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { // Given a MAM aerosol-related gas identifier, returns the name of its mass // mixing ratio field in EAMxx -KOKKOS_INLINE_FUNCTION +inline const char* gas_mmr_field_name(const int gas) { return const_cast(gas_species_name(gas)); } From 1f4d1f1881fcdb29769087752174fa3db0b98bb0 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 7 May 2024 12:26:55 -0600 Subject: [PATCH 161/476] EAMxx: add more beef to model output docs --- components/eamxx/docs/user/model_output.md | 72 ++++++++++++++++++++-- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/components/eamxx/docs/user/model_output.md b/components/eamxx/docs/user/model_output.md index 84d351c0302f..6490ff4ea338 100644 --- a/components/eamxx/docs/user/model_output.md +++ b/components/eamxx/docs/user/model_output.md @@ -9,7 +9,7 @@ into the RUNDIR/data folder. During this process, the files will be parsed, and variable will be resolved accordingly. Therefore, it is not advised to put the original YAML files in RUNDIR/data, since upon `buildnml` execution, all the CIME vars will no longer be available in the YAML file, making it harder to tweak it, and even harder to share with other users/cases. Another consequence -of this is that the user should not modify the yaml files in RUNDIR/data, since any modification will +of this is that the user should not modify the YAML files in RUNDIR/data, since any modification will be lost on the next run of `buildnml`. ## Basic output @@ -131,15 +131,75 @@ they are computed on. denote the grid (which must exist in the simulation) where the fields must be remapped before being saved to file. This feature is really only used to save fields on the dynamics grid without saving twice the DOFs at the interface of two spectral elements. - In fact, native output from the Dynamics grid would produce `6*num_elems*ngp*ngp`, - where `ngp` is the number of Gauss points along each axis in the 2d spectral element. - Note: this feature cannot be used along with the horizontal/vertical remapper. + E.g., for a scalar quantity defined only in the horizontal direction, native output + from the Dynamics grid would produce arrays of length `nelems*ngp*ngp`, where `ngp` + is the number of Gauss points along each axis in the 2d spectral element, and `nelems` + is the number of horizontal elements. However, due to continuity, the values on the + Gauss points on the element boundary must match the values on the neighboring element, + resulting in duplicated data. By remapping to a "unique" version of the dynamics grid + (which in EAMxx is referred to as "Physics GLL"), we can save roughly 45% of storage. + Note: this feature cannot be used along with the horizontal/vertical remap. ## Tendencies output -EXPLAIN how to request tendencies on a per-process basis +It is also possible to request tendencies of fields that are computed by atmosphere processes, +on a per-process basis. Since the term "tendency" can be used with slightly different connotations, +we clarify what we mean by that when it comes to model output: if process P updates field A, +by the tendency of A from process P we mean `(A_after_P - A_before_P) / dt`, where `dt` is +the atmosphere timestep. + +As of today, the user needs two things in order to get tendencies from a process. E.g., to get +the tendencies of `T_mid` and `horiz_winds` from the process `shoc`, one needs: + +- `atmchange shoc::compute_tendencies=T_mid,horiz_winds` +- add `shoc_T_mid_tend` and `shoc_horiz_winds_tend` to the list of fields in the output YAML file ## Additional options -LIST ADDITIONAL OPTIONS, like flush_frequency, file_max_storage_type, etc. +There YAML file shown at the top of this section, together with the remap options in the following +section, covers most of the options used in a typical run. There are however particular use cases +that require some less common options, which we list here (in parentheses, the location in the YAML +file and the type of the parameter value). + +- `flush_frequency` (toplevel list, integer): this parameter can be used to specify how often the IO + library should sync the in-memory data to file. If not specified, the IO library is free to decide + when it should flush the data. This option can be helpful for debugging, in case a crash is occurring + after a certain number of steps, but before the IO library would automatically flush to file. +- `Floating Point Precision` (toplevel list, string): this parameter specifies the precision to be used for floating + point variables in the output file. By default, EAMxx uses single precision. Valid values are + `single`, `float`, `double`, and `real`. The first two are synonyms, while the latter resolves + to `single` or `double` depending on EAMxx cmake configuration parameter `SCREAM_DOUBLE_PRECISION`. +- `file_max_storage_type` (toplevel list, string): this parameter determines how the capacity of the file is specified. + By default, it is set to `num_snapshots`, which makes EAMxx read `Max Snapshots Per File` (explained + in the first section). However, the user can specify `one_year` or `one_month`, which will make + EAMxx create one output file per year/month of simulation, fitting however many snapshots are needed + in each file (depending on the output frequency). If `one_year` or `one_month` are used, the option + `Max Snapshots Per File` is ignored. +- `MPI Ranks in Filename` (toplevel list, boolean): this option specifies whether the number of MPI ranks in the atm + communicator should be put inside the output file name. By default, this is `false`, since it is + usually not important. This option is mostly important for standalone unit testing, where several + versions of the same test (corresponding to different numbers of MPI ranks) are running concurrently, + so that different file names are needed to avoid resource contention. +- `save_grid_data` (`output_control` sublist, boolean): this option allows to specify whether grid data + (such as `lat`/`lon`) should be added to the output stream. By default, it is `true`. +- `iotype` (toplevel list, string): this option allows the user to request a particular format for the output + file. The possible values are `default`, `netcdf`, `pnetcdf, `adios`, `hdf5`, where `default` means + "whatever is the PIO type from the case settings". +- `skip_t0_output` (`output_control` sublist, boolean): this option is relevant only for `Instant` output, + where fields are also outputed at the case start time (i.e., after initialization but before the beginning + of the first timestep). By default it is set to `false`. +- restart options: when performing a restart, EAMxx attempts to restart every output stream listed in + the `output_yaml_files` atm option (which can be queried via `atmquery output_yaml_files`). The user + can specify a few options, in order to tweak the restart behavior: + - `Perform Restart` (`Restart` sublist, boolean): this parameter is `true` by default, but can be set + to `false` to force the model to ignore any history restart files, and start the output stream from + scratch, as if this was an initial run. + - `filename_prefix` (`Restart` sublist, string): by default, this parameter is set to match the value + of `filename_prefix` from the toplevel list. It can be set to something else in case we want to + restart a previous simulation that was using a different filename prefix. + - `force_new_file` (`Restart` sublist, boolean): this parameter allows to start a fresh new output file + upon restarts. By default, is is set to `false`, so that EAMxx will attempt to resume filling the + last produced output file (if any, and if it can accommodate more snapshots). + + From 889da0462b3c44a295a158a41cf220c0a66a6e82 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 7 May 2024 12:37:20 -0600 Subject: [PATCH 162/476] EAMxx: minor fixes/clarifications in the model output docs --- components/eamxx/docs/user/model_output.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/components/eamxx/docs/user/model_output.md b/components/eamxx/docs/user/model_output.md index 6490ff4ea338..65b8c9e7d940 100644 --- a/components/eamxx/docs/user/model_output.md +++ b/components/eamxx/docs/user/model_output.md @@ -114,7 +114,8 @@ I/O interface of EAMxx. There are two types of diagnostic outputs: - `X_at_model_bot`, `X_at_model_top`: special case for top and bottom of the model. - `X_at_Ymb`, `X_at_YPa`, `X_at_YhPa`: interpolates the field `X` at a vertical position specified by the give pressure `Y`. Available units are `mb` (millibar), `Pa`, and `hPa`. - - `X_at_Ym`: interpolates the field `X` at a vertical height of `Y` meters. + - `X_at_Ym_above_Z`: interpolates the field `X` at a vertical height of `Y` meters above + `Z`, with `Z=surface` or `Z=sealevel`. ## Remapped output @@ -142,21 +143,22 @@ they are computed on. ## Tendencies output -It is also possible to request tendencies of fields that are computed by atmosphere processes, -on a per-process basis. Since the term "tendency" can be used with slightly different connotations, +It is also possible to request tendencies of fields that are updated by atmosphere processes, +on a per-process basis (here, "updated" means that the field is both an input as well as an output +of the atmosphere process). Since the term "tendency" can be used with slightly different connotations, we clarify what we mean by that when it comes to model output: if process P updates field A, by the tendency of A from process P we mean `(A_after_P - A_before_P) / dt`, where `dt` is the atmosphere timestep. -As of today, the user needs two things in order to get tendencies from a process. E.g., to get +As of May 2024, the user needs two things in order to get tendencies from a process. E.g., to get the tendencies of `T_mid` and `horiz_winds` from the process `shoc`, one needs: -- `atmchange shoc::compute_tendencies=T_mid,horiz_winds` -- add `shoc_T_mid_tend` and `shoc_horiz_winds_tend` to the list of fields in the output YAML file +- `atmchange shoc::compute_tendencies=T_mid,horiz_winds`; +- add `shoc_T_mid_tend` and `shoc_horiz_winds_tend` to the list of fields in the desired output YAML file. ## Additional options -There YAML file shown at the top of this section, together with the remap options in the following +The YAML file shown at the top of this section, together with the remap options in the following section, covers most of the options used in a typical run. There are however particular use cases that require some less common options, which we list here (in parentheses, the location in the YAML file and the type of the parameter value). From c531a4d9366537825dc13e39766de7206534e826 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 8 May 2024 13:48:50 -0400 Subject: [PATCH 163/476] edit banner message --- components/eamxx/docs/overrides/main.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/docs/overrides/main.html b/components/eamxx/docs/overrides/main.html index 7c535238088d..f888486682d7 100644 --- a/components/eamxx/docs/overrides/main.html +++ b/components/eamxx/docs/overrides/main.html @@ -1,8 +1,8 @@ {% extends "base.html" %} {% block announce %} -EAMxx SCREAM is not supported yet. Read more about this here. +EAMxx is not supported yet. - In the meanwhile, go to docs.e3sm.org instead! + Pleaae visit docs.e3sm.org instead! {% endblock %} From 8f670e9dde7a3304597db7e71fb962690109c691 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 8 May 2024 13:45:09 -0600 Subject: [PATCH 164/476] Fixes to eamxx_rrtmgp_process_interface --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 112e52b98313..9df15ecd0f8c 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -365,6 +365,10 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_lw_bnd.totElems(); #endif + // During the transition to kokkos, the buffer views/arrays will point to the same memory, + // so care is needed to avoid repeating calculations in such a way that answers change. + // Example: buff_view(x) += foo; + // Stuff like this cannot be done twice when both kokkos and yakl are enabled #ifdef RRTMGP_ENABLE_KOKKOS mem = reinterpret_cast(buffer_manager.get_memory()); @@ -870,14 +874,22 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_gpt = subview_3d(m_buffer.cld_tau_lw_gpt); #endif #ifdef RRTMGP_ENABLE_KOKKOS + // For now, we need to duplicate this data. In the future, these should be + // unmanaged views. auto subview_1dk = [&](const real1dk v) -> real1dk { - return real1dk(v.data(),ncol); + real1dk rv(v.label(),ncol); + Kokkos::deep_copy(rv, v); + return rv; }; auto subview_2dk = [&](const real2dk v) -> real2dk { - return real2dk(v.data(),ncol,v.extent(1)); + real2dk rv(v.label(),ncol,v.extent(1)); + Kokkos::deep_copy(rv, v); + return rv; }; auto subview_3dk = [&](const real3dk v) -> real3dk { - return real3dk(v.data(),ncol,v.extent(1),v.extent(2)); + real3dk rv(v.label(),ncol,v.extent(1),v.extent(2)); + Kokkos::deep_copy(rv, v); + return rv; }; auto p_lay_k = subview_2dk(m_buffer.p_lay_k); @@ -1254,8 +1266,11 @@ void RRTMGPRadiation::run_impl (const double dt) { iwp(i+1,k+1) *= 1e3; #endif #ifdef RRTMGP_ENABLE_KOKKOS +#ifndef RRTMGP_ENABLE_YAKL + // lwp and lwp_k point to the same memory lwp_k(i,k) *= 1e3; iwp_k(i,k) *= 1e3; +#endif #endif }); }); From c4cad12d086484c51b477e63fb8f12ecb6d77a9b Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 8 May 2024 13:51:18 -0600 Subject: [PATCH 165/476] Fixes for scream_rrtmgp_interface --- .../rrtmgp/scream_rrtmgp_interface.cpp | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 438b65d7cb0c..b2341498756b 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -225,6 +225,7 @@ OpticalProps2str get_subsampled_clouds( seeds(icol) = 1e9 * (p_lay(icol,nlay) - int(p_lay(icol,nlay))); }); auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); + // Assign optical properties to subcolumns (note this implements MCICA) auto gpoint_bands = kdist.get_gpoint_bands(); parallel_for(SimpleBounds<3>(ngpt,nlay,ncol), YAKL_LAMBDA(int igpt, int ilay, int icol) { @@ -273,7 +274,7 @@ OpticalProps2strK get_subsampled_clouds( // use decimal part of pressure for this, consistent with the implementation in EAM auto seeds = int1dk("seeds", ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay) - int(p_lay(icol,nlay))); + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); // Assign optical properties to subcolumns (note this implements MCICA) @@ -367,7 +368,7 @@ OpticalProps1sclK get_subsampled_clouds( // seed values for longwave and shortwave auto seeds = int1dk("seeds", ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); // Assign optical properties to subcolumns (note this implements MCICA) @@ -1115,16 +1116,25 @@ int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) // // First, fill cldx with random numbers. Need to use a unique seed for each column! - auto seeds_host = Kokkos::create_mirror_view(seeds); - Kokkos::deep_copy(seeds_host, seeds); - for (int icol = 0; icol < ncol; ++icol) { - Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { - auto generator = random_pool.get_state(); - cldx(icol,ilay,igpt) = generator.drand(0., 1.); - random_pool.free_state(generator); - }); - } + // auto seeds_host = Kokkos::create_mirror_view(seeds); + // Kokkos::deep_copy(seeds_host, seeds); + // for (int icol = 0; icol < ncol; ++icol) { + // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // auto generator = random_pool.get_state(); + // cldx(icol,ilay,igpt) = generator.drand(0., 1.); + // random_pool.free_state(generator); + // }); + // } + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + yakl::Random rand(seeds(icol)); + for (int igpt = 0; igpt < ngpt; igpt++) { + for (int ilay = 0; ilay < nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + // Step down columns and apply algorithm from eq (14) Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { for (int ilay = 1; ilay < nlay; ilay++) { @@ -1227,6 +1237,7 @@ void rrtmgp_sw( dayIndices_h(nday) = icol; } } + // Copy data back to the device dayIndices_h.deep_copy_to(dayIndices); if (nday == 0) { @@ -1338,7 +1349,6 @@ void rrtmgp_sw( k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, optics_no_aerosols, toa_flux); } - #ifdef SCREAM_RRTMGP_DEBUG // Check gas optics check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); @@ -1369,6 +1379,7 @@ void rrtmgp_sw( // Compute clearsky (gas + aerosol) fluxes on daytime columns rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns parallel_for(SimpleBounds<2>(nlay+1,nday), YAKL_LAMBDA(int ilev, int iday) { int icol = dayIndices(iday); @@ -1478,9 +1489,8 @@ void rrtmgp_sw( Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { for (int icol = 0; icol < ncol; ++icol) { if (mu0(icol) > 0) { - ++nday_inner; + dayIndices(nday_inner++) = icol; } - dayIndices(nday_inner) = icol; } }, Kokkos::Sum(nday)); @@ -1627,6 +1637,7 @@ void rrtmgp_sw( // Compute clearsky (gas + aerosol) fluxes on daytime columns rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { const int icol = dayIndices(iday); From 3fc0a7187ea93d5fd27f2f5536bd132ac99e9b10 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Wed, 8 May 2024 13:12:54 -0700 Subject: [PATCH 166/476] Add a precipitation adjustment to the ml-correction process. This commit adds a precipitation adjustment in cases where the ml-correction is applied to qv. We interpret the hard adjustment of qv in the column as an attempt by ML to represent the missing precipitation in the coarse resolution run vs. the fine resolution run. In order to have balance with the land models we need to add this adjustment to the precipitation, otherwise we risk drying out the land. --- .../eamxx_ml_correction_process_interface.cpp | 75 ++++++++++++++++++- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 42a54250fe55..1d1ef8822e41 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -2,6 +2,7 @@ #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" #include "share/field/field_utils.hpp" +#include "physics/share/physics_constants.hpp" namespace scream { // ========================================================================================= @@ -39,14 +40,14 @@ void MLCorrection::set_grids( FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); FieldLayout vector3d_mid = m_grid->get_3d_vector_layout(true,2); + const auto m2 = m*m; if (not m_ML_correction_unit_test) { - const auto m2 = m*m; const auto s2 = s*s; auto Wm2 = W / m / m; auto nondim = m/m; add_field("phis", scalar2d, m2/s2, grid_name); - add_field("SW_flux_dn", scalar3d_int, Wm2, grid_name, ps); add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); + add_field("SW_flux_dn", scalar3d_int, Wm2, grid_name, ps); add_field("sfc_flux_sw_net", scalar2d, Wm2, grid_name); add_field("sfc_flux_lw_dn", scalar2d, Wm2, grid_name); m_lat = m_grid->get_geometry_data("lat"); @@ -62,6 +63,10 @@ void MLCorrection::set_grids( add_field("T_mid", scalar3d_mid, K, grid_name, ps); add_field("qv", scalar3d_mid, Q, grid_name, "tracers", ps); add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); + /* Note: we also need to update the precipitation after ML commits any column drying */ + add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); + add_field("precip_liq_surf_mass", scalar2d, kg/m2, grid_name); + add_field("precip_ice_surf_mass", scalar2d, kg/m2, grid_name); /* ----------------------- WARNING --------------------------------*/ add_group("tracers", grid_name, 1, Bundling::Required); } @@ -86,16 +91,23 @@ void MLCorrection::run_impl(const double dt) { // use model time to infer solar zenith angle for the ML prediction auto current_ts = timestamp(); std::string datetime_str = current_ts.get_date_string() + " " + current_ts.get_time_string(); + + const auto &phis = get_field_in("phis").get_view(); + const auto &sfc_alb_dif_vis = get_field_in("sfc_alb_dif_vis").get_view(); + const auto &qv = get_field_out("qv").get_view(); const auto &T_mid = get_field_out("T_mid").get_view(); - const auto &phis = get_field_in("phis").get_view(); const auto &SW_flux_dn = get_field_out("SW_flux_dn").get_view(); - const auto &sfc_alb_dif_vis = get_field_in("sfc_alb_dif_vis").get_view(); const auto &sfc_flux_sw_net = get_field_out("sfc_flux_sw_net").get_view(); const auto &sfc_flux_lw_dn = get_field_out("sfc_flux_lw_dn").get_view(); const auto &u = get_field_out("horiz_winds").get_component(0).get_view(); const auto &v = get_field_out("horiz_winds").get_component(1).get_view(); + // For precipitation adjustment we need to track the change in column integrated 'qv' + auto qv_told = Kokkos::create_mirror_view(qv); + Kokkos::deep_copy(qv_told,qv); + + auto h_lat = m_lat.get_view(); auto h_lon = m_lon.get_view(); @@ -135,6 +147,61 @@ void MLCorrection::run_impl(const double dt) { ML_model_tq, ML_model_uv, ML_model_sfc_fluxes, datetime_str); pybind11::gil_scoped_release no_gil; ekat::enable_fpes(fpe_mask); + + // Now back out the qv change abd apply it to precipitation, only if Tq ML is turned on + if (not (m_ML_model_path_tq == "None")) { + using PC = scream::physics::Constants; + using KT = KokkosTypes; + using MT = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + const auto &pseudo_density = get_field_in("pseudo_density").get_view(); + const auto &precip_liq_surf_mass = get_field_out("precip_liq_surf_mass").get_view(); + const auto &precip_ice_surf_mass = get_field_out("precip_ice_surf_mass").get_view(); + constexpr Real g = PC::gravit; + const auto num_levs = m_num_levs; + const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); + + const auto &qv_tnew = get_field_in("qv").get_view(); + Kokkos::parallel_for("Compute WVP diff", policy, + KOKKOS_LAMBDA(const MT& team) { + const int icol = team.league_rank(); + auto qold_icol = ekat::subview(qv_told,icol); + auto qnew_icol = ekat::subview(qv_tnew,icol); + auto rho_icol = ekat::subview(pseudo_density,icol); + Real wvp = 0; + // Compute WaterVaporPath Difference + Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), + [&] (const int& ilev, Real& lsum) { + lsum += (qnew_icol(ilev)-qold_icol(ilev)) * rho_icol(ilev) / g; + },wvp); + team.team_barrier(); + // Adjust Precipitation + // - Note, we subtract the water vapor path because positive precip represents + // a descrease in qv. + auto tot_precip = precip_liq_surf_mass(icol)+precip_ice_surf_mass(icol); + if (tot_precip>0) { + // adjust precip by weighted avg of both phases + auto liq_w = precip_liq_surf_mass(icol)/tot_precip; + auto ice_w = precip_ice_surf_mass(icol)/tot_precip; + precip_liq_surf_mass(icol) -= liq_w*wvp; + precip_ice_surf_mass(icol) -= ice_w*wvp; + } else { + // Apply all the adjustment to a single phase based on surface temperature + auto T_icol = ekat::subview(T_mid,icol); + if (T_icol(m_num_levs-1)>273.15) { + precip_liq_surf_mass(icol) -= wvp; + } else { + precip_ice_surf_mass(icol) -= wvp; + } + } + if (precip_liq_surf_mass(icol)<0) { + precip_liq_surf_mass(icol) = 0.0; + } + if (precip_ice_surf_mass(icol)<0) { + precip_ice_surf_mass(icol) = 0.0; + } + }); + } } // ========================================================================================= From ce81ea0e4e38f0bb7f31de8de20af2f8bb2214d3 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 3 May 2024 10:30:48 -0600 Subject: [PATCH 167/476] git Call setup_io_info correctly if PG2 fields exist Only current case is TMS where SGH30 needs to be loaded on PG2 grid --- components/eamxx/src/control/atmosphere_driver.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 4f2c0fc21b4c..df1aa2c384f8 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1144,9 +1144,14 @@ void AtmosphereDriver::set_initial_conditions () if (m_iop) { // For runs with IOP, call to setup io grids and lat // lon information needed for reading from file + // We use a single topo file for both GLL and PG2 runs. All + // lat/lon data in topo file is defined in terms of PG2 grid, + // so if we have a topo field on GLL grid, we need to setup + // io info using the IC file (which is always GLL). for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (ic_fields_names[grid_name].size() > 0) { + if (ic_fields_names[grid_name].size() > 0 or + topography_eamxx_fields_names[grid_name].size() > 0) { const auto& file_name = grid_name == "Physics GLL" ? ic_pl.get("Filename") From 4061c80fae8108c0ae7bce064ac3f575fef9cfea Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 3 May 2024 10:33:58 -0600 Subject: [PATCH 168/476] Do not load PG2 phis dynamics will always overwrite this --- components/eamxx/src/control/atmosphere_driver.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index df1aa2c384f8..b8a55db946b5 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1034,19 +1034,19 @@ void AtmosphereDriver::set_initial_conditions () auto& this_grid_topo_eamxx_fnames = topography_eamxx_fields_names[grid_name]; if (fname == "phis") { - // The eamxx field "phis" corresponds to the name - // "PHIS_d" on the GLL and Point grids and "PHIS" - // on the PG2 grid in the topography file. + // For GLL points, phis corresponds to "PHIS_d" in the + // topography file. On PG2 grid, dynamics will take care + // of computing phis, so do not add to initialized fields. if (grid_name == "Physics PG2") { - this_grid_topo_file_fnames.push_back("PHIS"); + // Skip } else if (grid_name == "Physics GLL" || grid_name == "Point Grid") { this_grid_topo_file_fnames.push_back("PHIS_d"); + this_grid_topo_eamxx_fnames.push_back(fname); + fields_inited[grid_name].push_back(fname); } else { EKAT_ERROR_MSG ("Error! Requesting phis on an unknown grid: " + grid_name + ".\n"); } - this_grid_topo_eamxx_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); } else if (fname == "sgh30") { // The eamxx field "sgh30" is called "SGH30" in the // topography file and is only available on the PG2 grid. From 8ee5f3402a593c1dff06313192eefc8c7db8d663 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Fri, 3 May 2024 11:04:33 -0600 Subject: [PATCH 169/476] Change DP unit test - use PG2 (CIME test still uses GLL, so both are covered) - Use 72 levels (nothing in DP is specific to 128, so save compute time) - Use TMS to test loading SGH30 on PG2 grid --- .../dynamics_physics/CMakeLists.txt | 2 +- .../CMakeLists.txt | 18 +++++++++--------- .../homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp} | 0 .../input.yaml | 6 +++--- .../output.yaml | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) rename components/eamxx/tests/multi-process/dynamics_physics/{homme_shoc_cld_spa_p3_rrtmgp_dp => homme_shoc_cld_spa_p3_rrtmgp_pg2_dp}/CMakeLists.txt (83%) rename components/eamxx/tests/multi-process/dynamics_physics/{homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp => homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp} (100%) rename components/eamxx/tests/multi-process/dynamics_physics/{homme_shoc_cld_spa_p3_rrtmgp_dp => homme_shoc_cld_spa_p3_rrtmgp_pg2_dp}/input.yaml (94%) rename components/eamxx/tests/multi-process/dynamics_physics/{homme_shoc_cld_spa_p3_rrtmgp_dp => homme_shoc_cld_spa_p3_rrtmgp_pg2_dp}/output.yaml (96%) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 5b263cbd66e5..64ac2187c280 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -6,7 +6,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(model_restart) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_128levels) - add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_dp) + add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_pg2_dp) if (SCREAM_ENABLE_MAM) # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the # corresponding test here needs to be reworked with valid aerosol diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt similarity index 83% rename from components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt index c499ccf21875..f56d8beefa60 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/CMakeLists.txt @@ -1,17 +1,17 @@ include (ScreamUtils) -set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_dp) +set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_pg2_dp) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Get or create the dynamics lib # HOMME_TARGET NP PLEV QSIZE_D -CreateDynamicsLib("theta-l_kokkos" 4 128 10) +CreateDynamicsLib("theta-l_kokkos" 4 72 10) # Create the test -set (TEST_LABELS "dynamics;driver;shoc;cld;spa;p3;rrtmgp;physics;dp") -CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_dp "homme_shoc_cld_spa_p3_rrtmgp_dp.cpp" +set (TEST_LABELS "dynamics;driver;tms;shoc;cld;spa;p3;rrtmgp;physics;dp") +CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_pg2_dp "homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp" LABELS ${TEST_LABELS} - LIBS cld_fraction shoc spa p3 scream_rrtmgp ${dynLibName} scream_control diagnostics + LIBS cld_fraction tms shoc spa p3 scream_rrtmgp ${dynLibName} scream_control diagnostics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -61,8 +61,8 @@ set (HOMME_TEST_MOISTURE notdry) set (HOMME_THETA_HY_MODE false) # Vert coord settings -set (HOMME_TEST_VCOORD_INT_FILE acme-128i.ascii) -set (HOMME_TEST_VCOORD_MID_FILE acme-128m.ascii) +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) # Configure the namelist into the test directory configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta-dp.nl @@ -70,7 +70,7 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta-dp.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/init/${EAMxx_tests_IC_FILE_128lev} + scream/init/${EAMxx_tests_IC_FILE_72lev} cam/topo/${EAMxx_tests_TOPO_FILE} cam/scam/iop/CASS_iopfile_4scam.nc ) @@ -85,7 +85,7 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS dynamics physics shoc cld spa p3 rrtmgp dp + LABELS dynamics physics tms shoc cld spa p3 rrtmgp dp pg2 META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp similarity index 100% rename from components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml similarity index 94% rename from components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml index 387f0670784b..0950f6bfdbc2 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/input.yaml @@ -21,7 +21,7 @@ time_stepping: number_of_steps: ${NUM_STEPS} initial_conditions: - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_128lev} + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} surf_evap: 0.0 surf_sens_flux: 0.0 @@ -44,7 +44,7 @@ atmosphere_processes: schedule_type: Sequential Type: Group mac_aero_mic: - atm_procs_list: [shoc,CldFraction,spa,p3] + atm_procs_list: [tms,shoc,CldFraction,spa,p3] Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} @@ -78,7 +78,7 @@ atmosphere_processes: grids_manager: Type: Homme - physics_grid_type: GLL + physics_grid_type: PG2 dynamics_namelist_file_name: namelist.nl vertical_coordinate_filename: IC_FILE diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/output.yaml similarity index 96% rename from components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/output.yaml index ddec1a3b9fa6..4440aa856bcd 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/output.yaml @@ -1,10 +1,10 @@ %YAML 1.1 --- -filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_dp_output +filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_pg2_dp_output Averaging Type: Instant Max Snapshots Per File: 1 Fields: - Physics GLL: + Physics PG2: Field Names: # HOMME - ps @@ -105,4 +105,4 @@ output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps ... - + From 3510a9c755e6ee20a83be0849d3cfc9294f04385 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 9 May 2024 06:14:11 -0600 Subject: [PATCH 170/476] mam4xx- updation mam4xx to get fix for optics-mam4xx CIME case in chrysalis. --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 42ea4d6dd65d..39e2d886b84d 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 42ea4d6dd65d560f79554b2a3c24d18c4ad9dfa7 +Subproject commit 39e2d886b84d08996b739fd1a1f8cc2deccb758a From 1fd4a66d839cfa96850047e686e3b394f2421e0b Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 9 May 2024 11:13:29 -0600 Subject: [PATCH 171/476] mam4xx- using mam4xx's main branch. --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 39e2d886b84d..fd26c9a33569 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 39e2d886b84d08996b739fd1a1f8cc2deccb758a +Subproject commit fd26c9a33569f8797a257194ee1499042a871554 From 9ad56f44ba535e5d62e09bbf4ca846cf9eeedc17 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Wed, 8 May 2024 14:30:02 -0600 Subject: [PATCH 172/476] Remove old dx_short code --- components/eamxx/src/control/atmosphere_driver.cpp | 3 --- .../eamxx/src/control/intensive_observation_period.hpp | 7 ------- 2 files changed, 10 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index b8a55db946b5..90bc12f8e85a 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -199,9 +199,6 @@ setup_iop () hyam, hybm); - auto dx_short_f = phys_grid->get_geometry_data("dx_short"); - m_iop->set_grid_spacing(dx_short_f.get_view()()); - // Set IOP object in atm processes m_atm_process_group->set_iop(m_iop); } diff --git a/components/eamxx/src/control/intensive_observation_period.hpp b/components/eamxx/src/control/intensive_observation_period.hpp index be3d389257bd..860eb082c427 100644 --- a/components/eamxx/src/control/intensive_observation_period.hpp +++ b/components/eamxx/src/control/intensive_observation_period.hpp @@ -115,13 +115,6 @@ class IntensiveObservationPeriod // data. void correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr); - // Store grid spacing for use in SHOC ad interface - void set_grid_spacing (const Real dx_short) { - m_dynamics_dx_size = dx_short*1000; - } - - Real get_dynamics_dx_size () { return m_dynamics_dx_size; } - ekat::ParameterList& get_params() { return m_params; } bool has_iop_field(const std::string& fname) { From 2faeb0731d78a7712fea6439bdc45064693d9441 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 9 May 2024 13:45:40 -0600 Subject: [PATCH 173/476] Use IOP lat/lon in rrtmgp interface --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 256174baef63..08fd833d1f68 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -58,8 +58,19 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ const auto& grid_name = m_grid->name(); m_ncol = m_grid->get_num_local_dofs(); m_nlay = m_grid->get_num_vertical_levels(); - m_lat = m_grid->get_geometry_data("lat"); - m_lon = m_grid->get_geometry_data("lon"); + + if (m_iop) { + // For IOP runs, we need to use the lat/lon from the + // IOP files instead of the geometry data. + m_lat = m_grid->get_geometry_data("lat").clone(); + m_lat.deep_copy(m_iop->get_params().get("target_latitude")); + + m_lon = m_grid->get_geometry_data("lon").clone(); + m_lon.deep_copy(m_iop->get_params().get("target_longitude")); + } else { + m_lat = m_grid->get_geometry_data("lat"); + m_lon = m_grid->get_geometry_data("lon"); + } // Figure out radiation column chunks stats m_col_chunk_size = std::min(m_params.get("column_chunk_size", m_ncol),m_ncol); From 5131951cc12ca3732b02c8d15d3a5ddf0d6be954 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 9 May 2024 14:59:37 -0600 Subject: [PATCH 174/476] More fixes --- components/eam/src/physics/rrtmgp/external | 2 +- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 18 +++++++++++++----- .../physics/rrtmgp/scream_rrtmgp_interface.cpp | 10 +++++++++- .../physics/rrtmgp/scream_rrtmgp_interface.hpp | 6 ++++-- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index fff1f14603e9..17cf33c1f099 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit fff1f14603e9f7aea84bdf03e9abc0d5189b39a0 +Subproject commit 17cf33c1f099aff7c6ecd9ced1bd353724d50fd1 diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 9df15ecd0f8c..638e209a7de1 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -804,6 +804,8 @@ void RRTMGPRadiation::run_impl (const double dt) { // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. + // JGF: this doesn't appear to be copying the data, just returning a new array + // pointing to the same memory. #ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); @@ -1360,6 +1362,7 @@ void RRTMGPRadiation::run_impl (const double dt) { COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); + VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif // Update heating tendency @@ -1492,13 +1495,13 @@ void RRTMGPRadiation::run_impl (const double dt) { std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif + // Compute cloud-top diagnostics following AeroCOM recommendation +#ifdef RRTMGP_ENABLE_YAKL // Get visible 0.67 micron band for COSP auto idx_067 = rrtmgp::get_wavelength_index_sw(0.67e-6); // Get IR 10.5 micron band for COSP auto idx_105 = rrtmgp::get_wavelength_index_lw(10.5e-6); - // Compute cloud-top diagnostics following AeroCOM recommendation -#ifdef RRTMGP_ENABLE_YAKL real1d T_mid_at_cldtop ("T_mid_at_cldtop", d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1d p_mid_at_cldtop ("p_mid_at_cldtop", d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1d cldfrac_ice_at_cldtop ("cldfrac_ice_at_cldtop", d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); @@ -1515,6 +1518,11 @@ void RRTMGPRadiation::run_impl (const double dt) { eff_radius_qc_at_cldtop, eff_radius_qi_at_cldtop); #endif #ifdef RRTMGP_ENABLE_KOKKOS + // Get visible 0.67 micron band for COSP + auto idx_067_k = rrtmgp::get_wavelength_index_sw_k(0.67e-6); + // Get IR 10.5 micron band for COSP + auto idx_105_k = rrtmgp::get_wavelength_index_lw_k(10.5e-6); + real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1dk cldfrac_ice_at_cldtop_k (d_cldfrac_ice_at_cldtop.data() + m_col_chunk_beg[ic], ncol); @@ -1619,8 +1627,8 @@ void RRTMGPRadiation::run_impl (const double dt) { }); // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { - d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067); - d_dtau105(icol,k) = cld_tau_lw_bnd_k(i,k,idx_105); + d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); + d_dtau105(icol,k) = cld_tau_lw_bnd_k(i,k,idx_105_k); }); if (d_sw_clrsky_flux_dn(icol,0) > 0) { d_sunlit(icol) = 1.0; @@ -1636,7 +1644,7 @@ void RRTMGPRadiation::run_impl (const double dt) { m_gas_concs.concs = gas_concs; #endif #ifdef RRTMGP_ENABLE_KOKKOS - m_gas_concs_k.concs = gas_concs_k; + VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif } // update_rad diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index b2341498756b..b06c6908928a 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1127,7 +1127,7 @@ int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d // }); // } Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - yakl::Random rand(seeds(icol)); + conv::Random rand(seeds(icol)); for (int igpt = 0; igpt < ngpt; igpt++) { for (int ilay = 0; ilay < nlay; ilay++) { cldx(icol,ilay,igpt) = rand.genFP(); @@ -2008,9 +2008,17 @@ void compute_cloud_area( } #endif +#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } +#endif + +#ifdef RRTMGP_ENABLE_KOKKOS +int get_wavelength_index_sw_k(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } + +int get_wavelength_index_lw_k(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } +#endif #ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 4096041b6645..b67933b1ef3a 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -356,13 +356,15 @@ void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) #ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength); +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); #endif #ifdef RRTMGP_ENABLE_KOKKOS int get_wavelength_index(OpticalPropsK &kdist, double wavelength); +int get_wavelength_index_sw_k(double wavelength); +int get_wavelength_index_lw_k(double wavelength); #endif -int get_wavelength_index_sw(double wavelength); -int get_wavelength_index_lw(double wavelength); } // namespace rrtmgp } // namespace scream From aa8c4ad6aab9a65daffc0e91647c0d6a335e6390 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 9 May 2024 16:38:54 -0600 Subject: [PATCH 175/476] Set rrtmgp submodule to kokkos-conversion-branch --- components/eam/src/physics/rrtmgp/external | 2 +- .../eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 17cf33c1f099..59e6f72cc922 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 17cf33c1f099aff7c6ecd9ced1bd353724d50fd1 +Subproject commit 59e6f72cc92268c69d1bd6ede570a959c1ebc28a diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 638e209a7de1..3d82e58e7b55 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -962,6 +962,7 @@ void RRTMGPRadiation::run_impl (const double dt) { m_gas_concs.concs = subview_3d(gas_concs); #endif #ifdef RRTMGP_ENABLE_KOKKOS + assert(m_gas_concs.ncol == ncol); // I think a lot of this code only works if chunk_size==ncol m_gas_concs_k.ncol = ncol; m_gas_concs_k.concs = subview_3dk(gas_concs_k); #endif From d6c8f4cec3b14d4ea860b8b01093bf55875bf0b8 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 6 May 2024 11:23:40 -0600 Subject: [PATCH 176/476] EAMxx: allow diags in IO to perform some ops at beginning of a timestep --- .../eamxx/src/control/atmosphere_driver.cpp | 10 +++ .../atm_process/atmosphere_diagnostic.hpp | 2 + .../eamxx/src/share/io/scorpio_output.cpp | 8 +++ .../eamxx/src/share/io/scorpio_output.hpp | 2 + .../src/share/io/scream_output_manager.cpp | 19 +++++ .../src/share/io/scream_output_manager.hpp | 2 + .../eamxx/src/share/io/tests/io_diags.cpp | 72 +++++++++++++------ 7 files changed, 93 insertions(+), 22 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index b8a55db946b5..745369bed0e2 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1605,6 +1605,16 @@ void AtmosphereDriver::run (const int dt) { // nano-opt of removing the call for the 1st timestep. reset_accumulated_fields(); + // Tell the output managers that we're starting a timestep. This is usually + // a no-op, but some diags *may* require to do something. E.g., a diag that + // computes tendency of an arbitrary quantity may want to store a copy of + // that quantity at the beginning of the timestep. Or they may need to store + // the timestamp at the beginning of the timestep, so that we can compute + // dt at the end. + for (auto it : m_output_managers) { + it.init_timestep(m_current_ts); + } + // The class AtmosphereProcessGroup will take care of dispatching arguments to // the individual processes, which will be called in the correct order. m_atm_process_group->run(dt); diff --git a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp index 2bd7fc4ddf26..0d7b89a28746 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp @@ -51,6 +51,8 @@ class AtmosphereDiagnostic : public AtmosphereProcess // Getting the diagnostic output Field get_diagnostic () const; + virtual void init_timestep (const util::TimeStamp& /* start_of_step */) {} + void compute_diagnostic (const double dt = 0); protected: diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index f35c7fb0c567..3db201f424ba 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -361,6 +361,14 @@ void AtmosphereOutput::init() register_views(); } +void AtmosphereOutput:: +init_timestep (const util::TimeStamp& start_of_step) +{ + for (auto& it : m_diagnostics) { + it.second->init_timestep(start_of_step); + } +} + void AtmosphereOutput:: run (const std::string& filename, const bool output_step, const bool checkpoint_step, diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index a4468123d1f5..2cb45f28a322 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -139,6 +139,8 @@ class AtmosphereOutput void reset_dev_views(); void update_avg_cnt_view(const Field&, view_1d_dev& dev_view); void setup_output_file (const std::string& filename, const std::string& fp_precision, const scorpio::FileMode mode); + + void init_timestep (const util::TimeStamp& start_of_step); void run (const std::string& filename, const bool output_step, const bool checkpoint_step, const int nsteps_since_last_output, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 35a830465bb0..4c064ee0dcea 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -264,6 +264,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, { // In order to trigger a t0 write, we need to have next_write_ts matching run_t0 m_output_control.next_write_ts = m_run_t0; + // This is in case some diags need to init the timestep. Most likely, their output + // is meaningless at t0, but they may still require the start-of-step timestamp to be valid + init_timestep(m_run_t0); this->run(m_run_t0); } @@ -281,6 +284,22 @@ add_global (const std::string& name, const ekat::any& global) { } /*===============================================================================================*/ +void OutputManager::init_timestep (const util::TimeStamp& start_of_step) +{ + // In case output is disabled, no point in doing anything else + if (not m_output_control.output_enabled()) { + return; + } + + if (m_atm_logger) { + m_atm_logger->debug("[OutputManager::init_timestep] filename_prefix: " + m_filename_prefix + "\n"); + } + + for (auto s : m_output_streams) { + s->init_timestep(start_of_step); + } +} + void OutputManager::run(const util::TimeStamp& timestamp) { // In case output is disabled, no point in doing anything else diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index b707c61ee4c0..df54f614970d 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -112,6 +112,8 @@ class OutputManager m_atm_logger = atm_logger; } void add_global (const std::string& name, const ekat::any& global); + + void init_timestep (const util::TimeStamp& start_of_step); void run (const util::TimeStamp& current_ts); void finalize(); diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 7daa13c1904e..98af5429e6dd 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -26,15 +26,6 @@ namespace scream { -void multiply (const Field& f, const double v) { - auto data = f.get_internal_view_data(); - auto nscalars = f.get_header().get_alloc_properties().get_num_scalars(); - for (int i=0; i(f_in); - multiply(m_diagnostic_output,2.0); - m_diagnostic_output.sync_to_dev(); + const auto& t = f_in.get_header().get_tracking().get_time_stamp(); + const double dt = t - m_t_beg; + + m_diagnostic_output.deep_copy(f_in); + m_diagnostic_output.update(m_one,dt,2.0); } void initialize_impl (const RunType /* run_type */ ) { @@ -92,12 +91,19 @@ class MyDiag : public AtmosphereDiagnostic int m_num_cols, m_num_levs; std::string m_f_in; + + util::TimeStamp m_t_beg; + Field m_one; }; util::TimeStamp get_t0 () { return util::TimeStamp({2023,2,17},{0,0,0}); } +constexpr double get_dt () { + return 10; +}; + std::shared_ptr get_gm (const ekat::Comm& comm) { @@ -167,6 +173,7 @@ void write (const int seed, const ekat::Comm& comm) // Time advance parameters auto t0 = get_t0(); + auto dt = get_dt(); // Create some fields auto fm = get_fm(grid,t0,seed); @@ -193,7 +200,15 @@ void write (const int seed, const ekat::Comm& comm) om.setup(comm,om_pl,fm,gm,t0,t0,false); // Run output manager - om.run (t0); + for (auto it : *fm) { + auto& f = *it.second; + Field one = f.clone("one"); + one.deep_copy(1.0); + f.get_header().get_tracking().update_time_stamp(t0+dt); + f.update(one,1.0,1.0); + } + om.init_timestep(t0); + om.run (t0+dt); // Close file and cleanup om.finalize(); @@ -211,6 +226,7 @@ void read (const int seed, const ekat::Comm& comm) // Get initial fields auto fm0 = get_fm(grid,t0,seed); auto fm = get_fm(grid,t0,seed,true); + std::vector fnames; std::string f_name; for (auto it : *fm) { @@ -220,6 +236,10 @@ void read (const int seed, const ekat::Comm& comm) f_name = fn; } } + + auto f0 = fm0->get_field(f_name).clone(); + auto f = fm->get_field(f_name); + // Sanity check REQUIRE (f_name!=""); @@ -235,17 +255,25 @@ void read (const int seed, const ekat::Comm& comm) reader_pl.set("Field Names",fnames); AtmosphereInput reader(reader_pl,fm); - reader.read_variables(); + Field one = f0.clone("one"); + one.deep_copy(1.0); + for (int i=0; i<2; ++i) { + reader.read_variables(i); - // Check regular field is correct - auto f0 = fm0->get_field(f_name).clone(); - auto f = fm->get_field(f_name); - REQUIRE (views_are_equal(f,f0)); + // Check regular field is correct + REQUIRE (views_are_equal(f,f0)); - // Check diag field is correct - multiply (f0,2.0); - auto d = fm->get_field("MyDiag"); - REQUIRE (views_are_equal(d,f0)); + // Check diag field is correct + const auto t = t0+i*get_dt(); + const double dt = t-t0; + auto d = fm->get_field("MyDiag"); + auto d0 = f0.clone(); + d0.update(one,dt,2.0); + REQUIRE (views_are_equal(d,d0)); + + // Update f0 + f0.update(one,1.0,1.0); + } } TEST_CASE ("io_diags") { From ea683540382556864038fa20c674d814c78ccbb7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 9 May 2024 18:17:53 -0600 Subject: [PATCH 177/476] EAMxx: only init timestep in IO if this will be an output step --- .../eamxx/src/control/atmosphere_driver.cpp | 2 +- .../src/share/io/scream_output_manager.cpp | 19 +++++++++++++------ .../src/share/io/scream_output_manager.hpp | 2 +- .../eamxx/src/share/io/tests/io_diags.cpp | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 745369bed0e2..7efa53d81248 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1612,7 +1612,7 @@ void AtmosphereDriver::run (const int dt) { // the timestamp at the beginning of the timestep, so that we can compute // dt at the end. for (auto it : m_output_managers) { - it.init_timestep(m_current_ts); + it.init_timestep(m_current_ts,dt); } // The class AtmosphereProcessGroup will take care of dispatching arguments to diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 4c064ee0dcea..87c9a46785a6 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -266,7 +266,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_output_control.next_write_ts = m_run_t0; // This is in case some diags need to init the timestep. Most likely, their output // is meaningless at t0, but they may still require the start-of-step timestamp to be valid - init_timestep(m_run_t0); + init_timestep(m_run_t0,0); this->run(m_run_t0); } @@ -284,13 +284,24 @@ add_global (const std::string& name, const ekat::any& global) { } /*===============================================================================================*/ -void OutputManager::init_timestep (const util::TimeStamp& start_of_step) +void OutputManager::init_timestep (const util::TimeStamp& start_of_step, const Real dt) { // In case output is disabled, no point in doing anything else if (not m_output_control.output_enabled()) { return; } + // Update counters + ++m_output_control.nsamples_since_last_write; + ++m_checkpoint_control.nsamples_since_last_write; + + // Check if the end of this timestep will correspond to an output step. If not, there's nothing to do + const auto& end_of_step = start_of_step+dt; + const bool will_be_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; + if (not will_be_output_step) { + return; + } + if (m_atm_logger) { m_atm_logger->debug("[OutputManager::init_timestep] filename_prefix: " + m_filename_prefix + "\n"); } @@ -316,10 +327,6 @@ void OutputManager::run(const util::TimeStamp& timestamp) std::string timer_root = m_is_model_restart_output ? "EAMxx::IO::restart" : "EAMxx::IO::standard"; start_timer(timer_root); - // Check if we need to open a new file - ++m_output_control.nsamples_since_last_write; - ++m_checkpoint_control.nsamples_since_last_write; - // Check if this is a write step (and what kind) // Note: a full checkpoint not only writes globals in the restart file, but also all the history variables. // Since we *always* write a history restart file, we can have a non-full checkpoint, if the average diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index df54f614970d..e1f58d93fede 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -113,7 +113,7 @@ class OutputManager } void add_global (const std::string& name, const ekat::any& global); - void init_timestep (const util::TimeStamp& start_of_step); + void init_timestep (const util::TimeStamp& start_of_step, const Real dt); void run (const util::TimeStamp& current_ts); void finalize(); diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 98af5429e6dd..01da7f18d1a5 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -207,7 +207,7 @@ void write (const int seed, const ekat::Comm& comm) f.get_header().get_tracking().update_time_stamp(t0+dt); f.update(one,1.0,1.0); } - om.init_timestep(t0); + om.init_timestep(t0,dt); om.run (t0+dt); // Close file and cleanup From 606b146ed13572b55ec919fe1d978aa2f3894421 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 9 May 2024 18:22:39 -0600 Subject: [PATCH 178/476] EAMxx: rephrased comment in output manager --- components/eamxx/src/share/io/scream_output_manager.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 87c9a46785a6..b1415772b42b 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -264,8 +264,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, { // In order to trigger a t0 write, we need to have next_write_ts matching run_t0 m_output_control.next_write_ts = m_run_t0; - // This is in case some diags need to init the timestep. Most likely, their output - // is meaningless at t0, but they may still require the start-of-step timestamp to be valid + // This is in case some diags need to init the timestep. Their output may be meaningless + // at t0 (e.g., if their input fields are not in the initial condition fields set, + // and have yet to be computed), but they may still require the start-of-step timestamp to be valid init_timestep(m_run_t0,0); this->run(m_run_t0); } From 9f15359c4b2c3cbb245e1330d9a4e3891829b522 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 9 May 2024 18:55:47 -0600 Subject: [PATCH 179/476] EAMxx: add calls to init_timestep to IO unit tests --- components/eamxx/src/share/io/tests/io_basic.cpp | 1 + components/eamxx/src/share/io/tests/io_filled.cpp | 1 + components/eamxx/src/share/io/tests/io_monthly.cpp | 1 + components/eamxx/src/share/io/tests/io_packed.cpp | 1 + .../eamxx/src/share/io/tests/io_remap_test.cpp | 14 ++++++++++---- components/eamxx/src/share/io/tests/io_se_grid.cpp | 4 +++- .../eamxx/src/share/io/tests/output_restart.cpp | 1 + 7 files changed, 18 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 11905fd8480e..033705a4dd4c 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -167,6 +167,7 @@ void write (const std::string& avg_type, const std::string& freq_units, const int nsteps = num_output_steps*freq; auto t = t0; for (int n=0; n source data ... done\n",io_comm); @@ -242,7 +245,8 @@ TEST_CASE("io_remap_test","io_remap_test") auto vert_remap_control = set_output_params("remap_vertical",remap_filename,p_ref,true,false); om_vert.setup(io_comm,vert_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); - om_vert.run(t0); + om_vert.init_timestep(t0,dt); + om_vert.run(t0+dt); om_vert.finalize(); print (" -> vertical remap ... done\n",io_comm); @@ -250,7 +254,8 @@ TEST_CASE("io_remap_test","io_remap_test") auto horiz_remap_control = set_output_params("remap_horizontal",remap_filename,p_ref,false,true); om_horiz.setup(io_comm,horiz_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); - om_horiz.run(t0); + om_horiz.init_timestep(t0,dt); + om_horiz.run(t0+dt); om_horiz.finalize(); print (" -> horizontal remap ... done\n",io_comm); @@ -258,7 +263,8 @@ TEST_CASE("io_remap_test","io_remap_test") auto vert_horiz_remap_control = set_output_params("remap_vertical_horizontal",remap_filename,p_ref,true,true); om_vert_horiz.setup(io_comm,vert_horiz_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); - om_vert_horiz.run(t0); + om_vert_horiz.init_timestep(t0,dt); + om_vert_horiz.run(t0+dt); om_vert_horiz.finalize(); print (" -> vertical-horizontal remap ... done\n",io_comm); print (" -> Create output ... done\n",io_comm); diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index b610002c8aca..36acf2a833cd 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -45,6 +45,7 @@ TEST_CASE("se_grid_io") int num_my_elems = 2; int np = 4; int num_levs = 2 + SCREAM_PACK_SIZE; + int dt = 10; // Initialize the pio_subsystem for this test: scorpio::init_subsystem(io_comm); @@ -70,7 +71,8 @@ TEST_CASE("se_grid_io") OutputManager om; om.setup(io_comm,params,fm0,gm,t0,t0,false); - om.run(t0); + om.init_timestep(t0,dt); + om.run(t0+dt); om.finalize(); // Get a fresh new field manager, and set fields to NaN diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 40c8ffee528e..8eea8c97f93c 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -99,6 +99,7 @@ TEST_CASE("output_restart","io") // The output restart data is written every 5 time steps, while the output freq is 10. auto time = run_t0; for (int i=0; i Date: Fri, 10 May 2024 09:23:50 -0600 Subject: [PATCH 180/476] EAMxx: minor fixes to output manager * Move IOControl counters update back to run method * Also init timestep if this is a checkpoint step --- .../src/share/io/scream_output_manager.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index b1415772b42b..9e25c0aa7483 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -292,14 +292,17 @@ void OutputManager::init_timestep (const util::TimeStamp& start_of_step, const R return; } - // Update counters - ++m_output_control.nsamples_since_last_write; - ++m_checkpoint_control.nsamples_since_last_write; - // Check if the end of this timestep will correspond to an output step. If not, there's nothing to do const auto& end_of_step = start_of_step+dt; - const bool will_be_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; - if (not will_be_output_step) { + + // Note: a full checkpoint not only writes globals in the restart file, but also all the history variables. + // Since we *always* write a history restart file, we can have a non-full checkpoint, if the average + // type is Instant and/or the frequency is every step. A non-full checkpoint will simply write some + // global attribute, such as the time of last write. + const bool is_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; + const bool is_checkpoint_step = m_checkpoint_control.is_write_step(end_of_step); + const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant; + if (not is_output_step and not (is_checkpoint_step and has_checkpoint_data) ) { return; } @@ -319,6 +322,10 @@ void OutputManager::run(const util::TimeStamp& timestamp) return; } + // Update counters + ++m_output_control.nsamples_since_last_write; + ++m_checkpoint_control.nsamples_since_last_write; + if (m_atm_logger) { m_atm_logger->debug("[OutputManager::run] filename_prefix: " + m_filename_prefix + "\n"); } From cf1e12892177513266ca3c622d9e78fe1c3c4d2d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 10 May 2024 09:45:51 -0600 Subject: [PATCH 181/476] EAMxx: added comment, and use override keyword in io diags unit test classes --- .../src/share/atm_process/atmosphere_diagnostic.hpp | 2 ++ components/eamxx/src/share/io/tests/io_diags.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp index 0d7b89a28746..d57a1477d741 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_diagnostic.hpp @@ -51,6 +51,8 @@ class AtmosphereDiagnostic : public AtmosphereProcess // Getting the diagnostic output Field get_diagnostic () const; + // Allows the diagnostic to save some start-of-step quantity (e.g., in case + // we need to compute tendencies, or accumulated stuff) virtual void init_timestep (const util::TimeStamp& /* start_of_step */) {} void compute_diagnostic (const double dt = 0); diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 01da7f18d1a5..f4658c8fdfa3 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -35,9 +35,9 @@ class MyDiag : public AtmosphereDiagnostic //Do nothing } - std::string name() const { return "MyDiag"; } + std::string name() const override { return "MyDiag"; } - void set_grids (const std::shared_ptr gm) { + void set_grids (const std::shared_ptr gm) override { using namespace ekat::units; using namespace ShortFieldTagsNames; using FL = FieldLayout; @@ -64,13 +64,13 @@ class MyDiag : public AtmosphereDiagnostic m_one.deep_copy(1.0); } - void init_timestep (const util::TimeStamp& start_of_step) { + void init_timestep (const util::TimeStamp& start_of_step) override { m_t_beg = start_of_step; } protected: - void compute_diagnostic_impl () { + void compute_diagnostic_impl () override { const auto& f_in = get_field_in(m_f_in); const auto& t = f_in.get_header().get_tracking().get_time_stamp(); @@ -80,12 +80,12 @@ class MyDiag : public AtmosphereDiagnostic m_diagnostic_output.update(m_one,dt,2.0); } - void initialize_impl (const RunType /* run_type */ ) { + void initialize_impl (const RunType /* run_type */ ) override { m_diagnostic_output.get_header().get_tracking().update_time_stamp(timestamp()); } // Clean up - void finalize_impl ( /* inputs */ ) {} + void finalize_impl ( /* inputs */ ) override {} // Internal variables int m_num_cols, m_num_levs; From ac01c257e1efcd82fcff278d71775dfec96a70c7 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Fri, 10 May 2024 14:06:52 -0700 Subject: [PATCH 182/476] Make changes in ml_correction thread safe, address review comments. This commit adds Kokkos::single(Kokkos::PerTeam... to the conditionals in the new precip adjustment. This commit fixes an error in how qv_told was defined, which was originally just an alias but should now be a truly separate copy of the initial qv state. This commit also addresses review comments about naming of new variables. --- .../eamxx_ml_correction_process_interface.cpp | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 1d1ef8822e41..171e3486ba4f 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -104,7 +104,7 @@ void MLCorrection::run_impl(const double dt) { const auto &v = get_field_out("horiz_winds").get_component(1).get_view(); // For precipitation adjustment we need to track the change in column integrated 'qv' - auto qv_told = Kokkos::create_mirror_view(qv); + host_view2d_type qv_told("",qv.extent(0),qv.extent(1)); Kokkos::deep_copy(qv_told,qv); @@ -149,7 +149,7 @@ void MLCorrection::run_impl(const double dt) { ekat::enable_fpes(fpe_mask); // Now back out the qv change abd apply it to precipitation, only if Tq ML is turned on - if (not (m_ML_model_path_tq == "None")) { + if (m_ML_model_path_tq != "None") { using PC = scream::physics::Constants; using KT = KokkosTypes; using MT = typename KT::MemberType; @@ -168,12 +168,12 @@ void MLCorrection::run_impl(const double dt) { auto qold_icol = ekat::subview(qv_told,icol); auto qnew_icol = ekat::subview(qv_tnew,icol); auto rho_icol = ekat::subview(pseudo_density,icol); - Real wvp = 0; + Real net_column_moistening = 0; // Compute WaterVaporPath Difference Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const int& ilev, Real& lsum) { lsum += (qnew_icol(ilev)-qold_icol(ilev)) * rho_icol(ilev) / g; - },wvp); + },net_column_moistening); team.team_barrier(); // Adjust Precipitation // - Note, we subtract the water vapor path because positive precip represents @@ -181,24 +181,32 @@ void MLCorrection::run_impl(const double dt) { auto tot_precip = precip_liq_surf_mass(icol)+precip_ice_surf_mass(icol); if (tot_precip>0) { // adjust precip by weighted avg of both phases - auto liq_w = precip_liq_surf_mass(icol)/tot_precip; - auto ice_w = precip_ice_surf_mass(icol)/tot_precip; - precip_liq_surf_mass(icol) -= liq_w*wvp; - precip_ice_surf_mass(icol) -= ice_w*wvp; + Kokkos::single(Kokkos::PerTeam(team), [&] { + auto liq_frac = precip_liq_surf_mass(icol)/tot_precip; + auto ice_frac = precip_ice_surf_mass(icol)/tot_precip; + precip_liq_surf_mass(icol) -= liq_frac*net_column_moistening; + precip_ice_surf_mass(icol) -= ice_frac*net_column_moistening; + }); } else { // Apply all the adjustment to a single phase based on surface temperature - auto T_icol = ekat::subview(T_mid,icol); - if (T_icol(m_num_levs-1)>273.15) { - precip_liq_surf_mass(icol) -= wvp; - } else { - precip_ice_surf_mass(icol) -= wvp; - } + Kokkos::single(Kokkos::PerTeam(team), [&] { + auto T_icol = ekat::subview(T_mid,icol); + if (T_icol(m_num_levs-1)>273.15) { + precip_liq_surf_mass(icol) -= net_column_moistening; + } else { + precip_ice_surf_mass(icol) -= net_column_moistening; + } + }); } if (precip_liq_surf_mass(icol)<0) { - precip_liq_surf_mass(icol) = 0.0; + Kokkos::single(Kokkos::PerTeam(team), [&] { + precip_liq_surf_mass(icol) = 0.0; + }); } if (precip_ice_surf_mass(icol)<0) { - precip_ice_surf_mass(icol) = 0.0; + Kokkos::single(Kokkos::PerTeam(team), [&] { + precip_ice_surf_mass(icol) = 0.0; + }); } }); } From abbe380996658db0443dda758c09260e8c8dc9dc Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 10 May 2024 16:18:39 -0600 Subject: [PATCH 183/476] More fixes --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 53 ++++++++++++++----- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 52 +++++++++--------- 2 files changed, 65 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 3d82e58e7b55..c8bec7e14d57 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -720,9 +720,11 @@ void RRTMGPRadiation::run_impl (const double dt) { // array, to restore at the end inside the m_gast_concs object. #ifdef RRTMGP_ENABLE_YAKL auto gas_concs = m_gas_concs.concs; + auto orig_ncol = m_gas_concs.ncol; #endif #ifdef RRTMGP_ENABLE_KOKKOS auto gas_concs_k = m_gas_concs_k.concs; + auto orig_ncol_k = m_gas_concs_k.ncol; #endif // Compute orbital parameters; these are used both for computing @@ -876,22 +878,37 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_gpt = subview_3d(m_buffer.cld_tau_lw_gpt); #endif #ifdef RRTMGP_ENABLE_KOKKOS - // For now, we need to duplicate this data. In the future, these should be - // unmanaged views. + // If YAKL is on, we don't want aliased memory in both the yakl and kokos + // subviews. auto subview_1dk = [&](const real1dk v) -> real1dk { - real1dk rv(v.label(),ncol); - Kokkos::deep_copy(rv, v); + real1dk subv(v, std::make_pair(0, ncol)); +#ifdef RRTMGP_ENABLE_YAKL + real1dk rv(v.label(), ncol); + Kokkos::deep_copy(rv, subv); return rv; +#else + return subv; +#endif }; auto subview_2dk = [&](const real2dk v) -> real2dk { - real2dk rv(v.label(),ncol,v.extent(1)); - Kokkos::deep_copy(rv, v); + real2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); +#ifdef RRTMGP_ENABLE_YAKL + real2dk rv(v.label(), ncol, v.extent(1)); + Kokkos::deep_copy(rv, subv); return rv; +#else + return subv; +#endif }; auto subview_3dk = [&](const real3dk v) -> real3dk { - real3dk rv(v.label(),ncol,v.extent(1),v.extent(2)); - Kokkos::deep_copy(rv, v); + real3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); +#ifdef RRTMGP_ENABLE_YAKL + real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); + Kokkos::deep_copy(rv, subv); return rv; +#else + return subv; +#endif }; auto p_lay_k = subview_2dk(m_buffer.p_lay_k); @@ -962,7 +979,6 @@ void RRTMGPRadiation::run_impl (const double dt) { m_gas_concs.concs = subview_3d(gas_concs); #endif #ifdef RRTMGP_ENABLE_KOKKOS - assert(m_gas_concs.ncol == ncol); // I think a lot of this code only works if chunk_size==ncol m_gas_concs_k.ncol = ncol; m_gas_concs_k.concs = subview_3dk(gas_concs_k); #endif @@ -1133,8 +1149,8 @@ void RRTMGPRadiation::run_impl (const double dt) { } Kokkos::fence(); #ifdef RRTMGP_ENABLE_KOKKOS - COMPARE_ALL_WRAP(std::vector({aero_tau_sw}), //, aero_ssa_sw, aero_g_sw, aero_tau_lw}), - std::vector({aero_tau_sw_k}));//, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); + COMPARE_ALL_WRAP(std::vector({aero_tau_sw, aero_ssa_sw, aero_g_sw, aero_tau_lw}), + std::vector({aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k})); #endif @@ -1177,8 +1193,9 @@ void RRTMGPRadiation::run_impl (const double dt) { m_gas_concs.set_vmr(name, tmp2d); #endif #ifdef RRTMGP_ENABLE_KOKKOS + COMPARE_WRAP(tmp2d, tmp2d_k); m_gas_concs_k.set_vmr(name, tmp2d_k); - VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); + //VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); broken for some reason #endif } @@ -1363,7 +1380,7 @@ void RRTMGPRadiation::run_impl (const double dt) { COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); - VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); + //VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif // Update heating tendency @@ -1637,15 +1654,23 @@ void RRTMGPRadiation::run_impl (const double dt) { d_sunlit(icol) = 0.0; } }); +#ifdef RRTMGP_ENABLE_YAKL + // Sync back to gas_concs_k + real3dk temp(gas_concs_k, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + Kokkos::deep_copy(temp, m_gas_concs_k.concs); +#endif #endif } // loop over chunk // Restore the refCounted array. #ifdef RRTMGP_ENABLE_YAKL m_gas_concs.concs = gas_concs; + m_gas_concs.ncol = orig_ncol; #endif #ifdef RRTMGP_ENABLE_KOKKOS - VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); + m_gas_concs_k.concs = gas_concs_k; + m_gas_concs_k.ncol = orig_ncol_k; + //VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif } // update_rad diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 400e3aa19329..5b560456183a 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -1093,10 +1093,10 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { - if (ibnd < 10) { + if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; - } else if (ibnd == 10) { + } else if (ibnd == 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; } else { @@ -1115,20 +1115,20 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Check computed surface fluxes logger->info("Check computed fluxes...\n"); const double tol = 1e-10; // tolerance on floating point inequality for assertions - REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 0.5) < tol); - REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 0.5) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 0.5) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(0) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(0) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(0) - 0.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(0) - 0.5) < tol); // --------------------------------- // --------------------------------- // Test case, only flux in NIR bands logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { - if (ibnd < 10) { + if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; - } else if (ibnd == 10) { + } else if (ibnd == 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; } else { @@ -1146,20 +1146,20 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { ); // Check computed surface fluxes logger->info("Check computed fluxes...\n"); - REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 9.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 0.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 9.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(0) - 9.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(0) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(0) - 9.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(0) - 0.0) < tol); // --------------------------------- // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { - if (ibnd < 10) { + if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; - } else if (ibnd == 10) { + } else if (ibnd == 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; } else { @@ -1177,20 +1177,20 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { ); // Check computed surface fluxes logger->info("Check computed fluxes...\n"); - REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 0.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 4.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 0.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 4.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(0) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(0) - 4.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(0) - 0.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(0) - 4.0) < tol); // --------------------------------- // --------------------------------- // Test case, only flux in all bands logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { - if (ibnd < 10) { + if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; - } else if (ibnd == 10) { + } else if (ibnd == 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 3.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 4.0; } else { @@ -1208,10 +1208,10 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { ); // Check computed surface fluxes logger->info("Check computed fluxes...\n"); - REQUIRE(std::abs(chc(sfc_flux_dir_nir)(1) - 10.5) < tol); - REQUIRE(std::abs(chc(sfc_flux_dir_vis)(1) - 21.5) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_nir)(1) - 20.0) < tol); - REQUIRE(std::abs(chc(sfc_flux_dif_vis)(1) - 26.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_nir)(0) - 10.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dir_vis)(0) - 21.5) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_nir)(0) - 20.0) < tol); + REQUIRE(std::abs(chc(sfc_flux_dif_vis)(0) - 26.0) < tol); // --------------------------------- // Finalize YAKL @@ -1252,10 +1252,10 @@ TEST_CASE("rrtmgp_test_check_range_k") { Kokkos::deep_copy(dummy, 0.1); REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == true); // At least one value below lower bound - Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 1) = -0.1;}); + Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = -0.1;}); REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); // At least one value above upper bound - Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 1) = 1.1;}); + Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = 1.1;}); REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); scream::finalize_kls(); } From b49667c39f88ddd3b472814275ea143616a26544 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 10 May 2024 16:34:01 -0600 Subject: [PATCH 184/476] Fix merge mistake --- .../rrtmgp/scream_rrtmgp_interface.cpp | 102 +----------------- 1 file changed, 1 insertion(+), 101 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 6e87fe472506..203336f395dd 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -2178,7 +2178,7 @@ void compute_aerocom_cloudtop( real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCOM recommendation. See reference for routine + * based on the AeroCom recommendation. See reference for routine * get_subcolumn_mask above, where equation 14 is used for the * maximum-random overlap assumption for subcolumn generation. We use * equation 13, the column counterpart. @@ -2270,105 +2270,5 @@ void compute_aerocom_cloudtop( } #endif -<<<<<<< HEAD } // namespace rrtmgp -======= - void compute_aerocom_cloudtop( - int ncol, int nlay, const real2d &tmid, const real2d &pmid, - const real2d &p_del, const real2d &z_del, const real2d &qc, - const real2d &qi, const real2d &rel, const real2d &rei, - const real2d &cldfrac_tot, const real2d &nc, - real1d &T_mid_at_cldtop, real1d &p_mid_at_cldtop, - real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, - real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, - real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCom recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - memset(T_mid_at_cldtop, 0.0); - memset(p_mid_at_cldtop, 0.0); - memset(cldfrac_ice_at_cldtop, 0.0); - memset(cldfrac_liq_at_cldtop, 0.0); - memset(cldfrac_tot_at_cldtop, 0.0); - memset(cdnc_at_cldtop, 0.0); - memset(eff_radius_qc_at_cldtop, 0.0); - memset(eff_radius_qi_at_cldtop, 0.0); - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1d("aerocom_clr", ncol); - memset(aerocom_clr, 1.0); - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - // Loop over all columns in parallel - yakl::fortran::parallel_for( - SimpleBounds<1>(ncol), YAKL_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 2; ilay <= nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); - } - } // namespace rrtmgp ->>>>>>> origin/master } // namespace scream From 18bbfcaaa7ee43ae77873dfcd5fbe62475a851d5 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Fri, 10 May 2024 17:37:53 -0600 Subject: [PATCH 185/476] ma4xx - upgrading mam4xx. --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index fd26c9a33569..7b0be56f4c3f 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit fd26c9a33569f8797a257194ee1499042a871554 +Subproject commit 7b0be56f4c3f164259c6a9d8a0d16d6f5b72f3ae From c9331650db897c30331037e044ef9ee4a4229e5d Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 13 May 2024 22:09:13 -0400 Subject: [PATCH 186/476] bind gpus to srun on pm-gpu --- components/eamxx/cmake/machine-files/pm-gpu.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/cmake/machine-files/pm-gpu.cmake b/components/eamxx/cmake/machine-files/pm-gpu.cmake index 7654d75138f8..5756aa9e9589 100644 --- a/components/eamxx/cmake/machine-files/pm-gpu.cmake +++ b/components/eamxx/cmake/machine-files/pm-gpu.cmake @@ -18,6 +18,8 @@ endif() include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) +set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpu-per-task 1") + #option(Kokkos_ARCH_AMPERE80 "" ON) set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) From 61d830596ba5e30ce93a6838d1656e83f0219de6 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 14 May 2024 14:54:22 -0400 Subject: [PATCH 187/476] change extra directives --- components/eamxx/cmake/machine-files/pm-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cmake/machine-files/pm-gpu.cmake b/components/eamxx/cmake/machine-files/pm-gpu.cmake index 5756aa9e9589..4c448eb76262 100644 --- a/components/eamxx/cmake/machine-files/pm-gpu.cmake +++ b/components/eamxx/cmake/machine-files/pm-gpu.cmake @@ -18,7 +18,7 @@ endif() include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpu-per-task 1") +set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpu-per-node 4 --gpu-bind=none") #option(Kokkos_ARCH_AMPERE80 "" ON) set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) From 3b07fa759664baecabfeb119acd319ef889cb0a1 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 15 May 2024 01:52:27 -0400 Subject: [PATCH 188/476] go back to --gpus-per-task=1 and force it (otherwise won't work) --- components/eamxx/cmake/machine-files/pm-gpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cmake/machine-files/pm-gpu.cmake b/components/eamxx/cmake/machine-files/pm-gpu.cmake index 4c448eb76262..5ca181653ab6 100644 --- a/components/eamxx/cmake/machine-files/pm-gpu.cmake +++ b/components/eamxx/cmake/machine-files/pm-gpu.cmake @@ -18,7 +18,7 @@ endif() include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpu-per-node 4 --gpu-bind=none") +set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpus-per-task=1" CACHE STRING "" FORCE) #option(Kokkos_ARCH_AMPERE80 "" ON) set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) From 49856acccbe4c5cc1fc62a36f19e3dfe95600fc0 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Wed, 15 May 2024 16:29:58 -0700 Subject: [PATCH 189/476] This commit addresses two fixes in Corrective-ML 1. Fixes precipitation adjustment in Corrective-ML, and it should now do the adjustment on Device when possible. 2. Fixes the interface to the TQ trained model to include phis and lat. --- .../eamxx_ml_correction_process_interface.cpp | 7 +++---- .../eamxx/src/physics/ml_correction/ml_correction.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 171e3486ba4f..82029abbc6a6 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -104,10 +104,9 @@ void MLCorrection::run_impl(const double dt) { const auto &v = get_field_out("horiz_winds").get_component(1).get_view(); // For precipitation adjustment we need to track the change in column integrated 'qv' - host_view2d_type qv_told("",qv.extent(0),qv.extent(1)); + decltype(qv) qv_told("", qv.extent(0), qv.extent(1)); Kokkos::deep_copy(qv_told,qv); - auto h_lat = m_lat.get_view(); auto h_lon = m_lon.get_view(); @@ -155,8 +154,8 @@ void MLCorrection::run_impl(const double dt) { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; const auto &pseudo_density = get_field_in("pseudo_density").get_view(); - const auto &precip_liq_surf_mass = get_field_out("precip_liq_surf_mass").get_view(); - const auto &precip_ice_surf_mass = get_field_out("precip_ice_surf_mass").get_view(); + const auto &precip_liq_surf_mass = get_field_out("precip_liq_surf_mass").get_view(); + const auto &precip_ice_surf_mass = get_field_out("precip_ice_surf_mass").get_view(); constexpr Real g = PC::gravit; const auto num_levs = m_num_levs; const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); diff --git a/components/eamxx/src/physics/ml_correction/ml_correction.py b/components/eamxx/src/physics/ml_correction/ml_correction.py index 4e22f711c5ae..acecdd95b467 100644 --- a/components/eamxx/src/physics/ml_correction/ml_correction.py +++ b/components/eamxx/src/physics/ml_correction/ml_correction.py @@ -25,7 +25,7 @@ def ensure_correction_ordering(correction): return correction -def get_ML_correction_dQ1_dQ2(model, T_mid, qv, cos_zenith, dt): +def get_ML_correction_dQ1_dQ2(model, T_mid, qv, cos_zenith, lat, phis, dt): """Get ML correction for air temperature (dQ1) and specific humidity (dQ2) Args: @@ -40,6 +40,8 @@ def get_ML_correction_dQ1_dQ2(model, T_mid, qv, cos_zenith, dt): T_mid=(["ncol", "z"], T_mid), qv=(["ncol", "z"], qv), cos_zenith_angle=(["ncol"], cos_zenith), + lat=(["ncol"], lat), + surface_geopotential=(["ncol"], phis), ) ) return ensure_correction_ordering(predict(model, ds, dt)) @@ -180,7 +182,13 @@ def update_fields( ) if model_tq is not None: correction_tq = get_ML_correction_dQ1_dQ2( - model_tq, T_mid, qv[:, 0, :], cos_zenith, dt + model_tq, + T_mid, + qv[:, 0, :], + cos_zenith, + lat, + phis, + dt ) T_mid[:, :] += correction_tq["dQ1"].values * dt qv[:, 0, :] += correction_tq["dQ2"].values * dt From 2f5893824aacd1e301fb6b2da2857c681f2e66b9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 15 May 2024 20:31:24 -0600 Subject: [PATCH 190/476] Update ekat submodule --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index 0811e6cad49b..eed9ddbefe68 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 0811e6cad49b0f10d6c343bb6fed97e6e98fb794 +Subproject commit eed9ddbefe685ea084853fa9c9262c861a5a7f39 From 1f470ad3ac33c6b48315eb1948507ad6cbdc19ed Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 15 May 2024 17:50:59 -0600 Subject: [PATCH 191/476] EAMxx: fix usage of ekat's Units --- .../atmosphere_surface_coupling_exporter.cpp | 25 +++---- .../atmosphere_surface_coupling_importer.cpp | 8 +- .../eamxx/src/diagnostics/atm_density.cpp | 11 +-- .../src/diagnostics/dry_static_energy.cpp | 16 ++-- .../diagnostics/longwave_cloud_forcing.cpp | 6 +- .../eamxx/src/diagnostics/number_path.cpp | 5 +- .../src/diagnostics/potential_temperature.cpp | 5 +- .../src/diagnostics/relative_humidity.cpp | 12 ++- .../src/diagnostics/sea_level_pressure.cpp | 6 +- .../diagnostics/shortwave_cloud_forcing.cpp | 13 ++-- .../surf_upward_latent_heat_flux.cpp | 8 +- .../eamxx/src/diagnostics/vapor_flux.cpp | 12 +-- .../eamxx/src/diagnostics/vertical_layer.cpp | 8 +- .../src/diagnostics/virtual_temperature.cpp | 7 +- .../eamxx/src/diagnostics/water_path.cpp | 8 +- .../dynamics/homme/eamxx_homme_fv_phys.cpp | 6 +- .../homme/eamxx_homme_process_interface.cpp | 10 +-- .../dynamics/homme/homme_grids_manager.cpp | 14 ++-- .../eamxx_cld_fraction_process_interface.cpp | 4 +- .../eamxx/src/physics/cosp/eamxx_cosp.cpp | 16 ++-- ...mxx_mam_microphysics_process_interface.cpp | 21 +++--- .../eamxx_mam_optics_process_interface.cpp | 21 +++--- .../eamxx_ml_correction_process_interface.cpp | 22 +++--- .../eamxx_nudging_process_interface.cpp | 4 +- .../physics/p3/eamxx_p3_process_interface.cpp | 43 ++++++----- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 75 +++++++++---------- .../shoc/eamxx_shoc_process_interface.cpp | 27 +++---- .../spa/eamxx_spa_process_interface.cpp | 4 - .../tms/eamxx_tms_process_interface.cpp | 10 +-- .../physics/zm/eamxx_zm_process_interface.cpp | 11 +-- .../atm_process/atmosphere_process_dag.cpp | 2 +- .../src/share/field/field_identifier.cpp | 2 +- .../eamxx/src/share/field/field_manager.cpp | 4 +- .../share/grid/mesh_free_grids_manager.cpp | 13 ++-- .../eamxx/src/share/io/scorpio_output.cpp | 2 +- 35 files changed, 189 insertions(+), 272 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp index dfdea16ae93f..7a0253ec908a 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp @@ -25,15 +25,8 @@ void SurfaceCouplingExporter::set_grids(const std::shared_ptrget_num_local_dofs(); // Number of columns on this rank m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - const auto m2 = m*m; - const auto s2 = s*s; - auto Wm2 = W/m2; - Wm2.set_string("W/m2"); - - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Qunit = kg/kg; - Qunit.set_string("kg/kg"); + Units m2 (m*m,"m2"); + Units s2 (s*s,"s2"); // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -50,16 +43,16 @@ void SurfaceCouplingExporter::set_grids(const std::shared_ptr("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); add_field("phis", scalar2d_layout, m2/s2, grid_name); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); // TODO: Switch horiz_winds to using U and V, note right now there is an issue with when the subfields are created, so can't switch yet. add_field("horiz_winds", vector3d_layout, m/s, grid_name); - add_field("sfc_flux_dir_nir", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_dir_vis", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_dif_nir", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_dif_vis", scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_sw_net" , scalar2d_layout, Wm2, grid_name); - add_field("sfc_flux_lw_dn" , scalar2d_layout, Wm2, grid_name); + add_field("sfc_flux_dir_nir", scalar2d_layout, W/m2, grid_name); + add_field("sfc_flux_dir_vis", scalar2d_layout, W/m2, grid_name); + add_field("sfc_flux_dif_nir", scalar2d_layout, W/m2, grid_name); + add_field("sfc_flux_dif_vis", scalar2d_layout, W/m2, grid_name); + add_field("sfc_flux_sw_net" , scalar2d_layout, W/m2, grid_name); + add_field("sfc_flux_lw_dn" , scalar2d_layout, W/m2, grid_name); add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name); add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name); diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index e816801fa618..52deba16e8d3 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -28,12 +28,8 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("surf_mom_flux", vector2d_layout, N/m2, grid_name); add_field("surf_radiative_T", scalar2d_layout, K, grid_name); add_field("T_2m", scalar2d_layout, K, grid_name); - add_field("qv_2m", scalar2d_layout, Qunit, grid_name); + add_field("qv_2m", scalar2d_layout, kg/kg, grid_name); add_field("wind_speed_10m", scalar2d_layout, m/s, grid_name); add_field("snow_depth_land", scalar2d_layout, m, grid_name); add_field("ocnfrac", scalar2d_layout, nondim, grid_name); diff --git a/components/eamxx/src/diagnostics/atm_density.cpp b/components/eamxx/src/diagnostics/atm_density.cpp index 24749ba1a1ef..8f1e8b77b54a 100644 --- a/components/eamxx/src/diagnostics/atm_density.cpp +++ b/components/eamxx/src/diagnostics/atm_density.cpp @@ -16,9 +16,6 @@ void AtmDensityDiagnostic::set_grids(const std::shared_ptr g using namespace ekat::units; using namespace ShortFieldTagsNames; - auto Q = kg/kg; - Q.set_string("kg/kg"); - // Boiler Plate auto grid = grids_manager->get_grid("Physics"); const auto& grid_name = grid->name(); @@ -29,10 +26,10 @@ void AtmDensityDiagnostic::set_grids(const std::shared_ptr g FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols,m_num_levs} }; // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, SCREAM_PACK_SIZE); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); - add_field("qv", scalar3d_layout_mid, Q, grid_name, SCREAM_PACK_SIZE); + add_field("T_mid", scalar3d_layout_mid, K, grid_name, SCREAM_PACK_SIZE); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, SCREAM_PACK_SIZE); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, kg/(m*m*m), grid_name); diff --git a/components/eamxx/src/diagnostics/dry_static_energy.cpp b/components/eamxx/src/diagnostics/dry_static_energy.cpp index f4ed33ca56b6..8b0264dd0532 100644 --- a/components/eamxx/src/diagnostics/dry_static_energy.cpp +++ b/components/eamxx/src/diagnostics/dry_static_energy.cpp @@ -20,10 +20,8 @@ void DryStaticEnergyDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -35,11 +33,11 @@ void DryStaticEnergyDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, ps); - add_field("phis", scalar2d_layout_col, m2/s2, grid_name, ps); + add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, ps); + add_field("phis", scalar2d_layout_col, m2/s2, grid_name, ps); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, m2/s2, grid_name); diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp index 4e44dff9dc76..49109b64ef7c 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp @@ -18,9 +18,7 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -35,7 +33,7 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("LW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_col, radflux_units, grid_name); + FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(); diff --git a/components/eamxx/src/diagnostics/number_path.cpp b/components/eamxx/src/diagnostics/number_path.cpp index 6ce0e773b684..d4df2f1bc22f 100644 --- a/components/eamxx/src/diagnostics/number_path.cpp +++ b/components/eamxx/src/diagnostics/number_path.cpp @@ -39,8 +39,7 @@ void NumberPathDiagnostic::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - auto out_units = kg / (kg * m * m); - out_units.set_string("kg/(kg m2)"); + auto m2 = pow(m,2); auto grid = grids_manager->get_grid("Physics"); const auto &grid_name = grid->name(); @@ -56,7 +55,7 @@ void NumberPathDiagnostic::set_grids( add_field(m_nname, scalar3d, 1 / kg, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d, out_units, grid_name); + FieldIdentifier fid(name(), scalar2d, kg/(kg*m2), grid_name); m_diagnostic_output = Field(fid); m_diagnostic_output.allocate_view(); } diff --git a/components/eamxx/src/diagnostics/potential_temperature.cpp b/components/eamxx/src/diagnostics/potential_temperature.cpp index 988bb5552240..67260e647f6f 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.cpp +++ b/components/eamxx/src/diagnostics/potential_temperature.cpp @@ -35,9 +35,6 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank @@ -51,7 +48,7 @@ void PotentialTemperatureDiagnostic::set_grids(const std::shared_ptr("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); // Only needed for LiqPotentialTemperature, but put it here for ease // TODO: only request it if it is needed - add_field("qc", scalar3d_layout_mid, Q, grid_name, ps); + add_field("qc", scalar3d_layout_mid, kg/kg, grid_name, ps); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); diff --git a/components/eamxx/src/diagnostics/relative_humidity.cpp b/components/eamxx/src/diagnostics/relative_humidity.cpp index e7ceca4b8951..909213a95b65 100644 --- a/components/eamxx/src/diagnostics/relative_humidity.cpp +++ b/components/eamxx/src/diagnostics/relative_humidity.cpp @@ -18,8 +18,6 @@ void RelativeHumidityDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -29,11 +27,11 @@ void RelativeHumidityDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, SCREAM_PACK_SIZE); - add_field("p_dry_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); - add_field("qv", scalar3d_layout_mid, Q, grid_name, SCREAM_PACK_SIZE); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); - add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("T_mid", scalar3d_layout_mid, K, grid_name, SCREAM_PACK_SIZE); + add_field("p_dry_mid", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, SCREAM_PACK_SIZE); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); + add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name, SCREAM_PACK_SIZE); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, nondim, grid_name); diff --git a/components/eamxx/src/diagnostics/sea_level_pressure.cpp b/components/eamxx/src/diagnostics/sea_level_pressure.cpp index 54f6b52fbea8..fdbab884762b 100644 --- a/components/eamxx/src/diagnostics/sea_level_pressure.cpp +++ b/components/eamxx/src/diagnostics/sea_level_pressure.cpp @@ -17,10 +17,8 @@ void SeaLevelPressureDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp index 9ca5ea4a8a14..89b687123f9d 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp @@ -17,8 +17,7 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -29,13 +28,13 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("SW_flux_dn", scalar3d_layout_mid, radflux_units, grid_name); - add_field("SW_flux_up", scalar3d_layout_mid, radflux_units, grid_name); - add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, radflux_units, grid_name); - add_field("SW_clrsky_flux_up", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_flux_dn", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_flux_up", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_col, radflux_units, grid_name); + FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(); diff --git a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp index 009f8e6dd77f..91ee2833888b 100644 --- a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp +++ b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp @@ -20,10 +20,8 @@ SurfaceUpwardLatentHeatFlux(const ekat::Comm& comm, const ekat::ParameterList& p void SurfaceUpwardLatentHeatFlux:: set_grids (const std::shared_ptr grids_manager) { - const auto m2 = ekat::units::m * ekat::units::m; - const auto W = ekat::units::W; - auto radflux_units = W/(m2); - radflux_units.set_string("W/m2"); + using namespace ekat::units; + Units m2(m*m,"m2"); const auto surf_evap_units = ekat::units::kg / m2 / ekat::units::s; @@ -38,7 +36,7 @@ set_grids (const std::shared_ptr grids_manager) add_field("surf_evap", scalar2d_layout_mid, surf_evap_units, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, radflux_units, grid_name); + FieldIdentifier fid(name(), scalar2d_layout_mid, W/m2, grid_name); // handle parent class member variables m_diagnostic_output = Field(fid); m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); diff --git a/components/eamxx/src/diagnostics/vapor_flux.cpp b/components/eamxx/src/diagnostics/vapor_flux.cpp index 66abf10d5bb1..ce88c1443015 100644 --- a/components/eamxx/src/diagnostics/vapor_flux.cpp +++ b/components/eamxx/src/diagnostics/vapor_flux.cpp @@ -36,12 +36,6 @@ void VaporFluxDiagnostic::set_grids(const std::shared_ptr gr using namespace ekat::units; using namespace ShortFieldTagsNames; - auto Q = kg/kg; - Q.set_string("kg/kg"); - - auto vel = m/s; - vel.set_string("m/s"); - auto grid = grids_manager->get_grid("Physics"); const auto& grid_name = grid->name(); m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank @@ -52,9 +46,9 @@ void VaporFluxDiagnostic::set_grids(const std::shared_ptr gr auto vector3d = grid->get_3d_vector_layout(true,2); // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d, Pa, grid_name); - add_field("qv", scalar3d, Q, grid_name); - add_field("horiz_winds", vector3d, m/s, grid_name); + add_field("pseudo_density", scalar3d, Pa, grid_name); + add_field("qv", scalar3d, kg/kg, grid_name); + add_field("horiz_winds", vector3d, m/s, grid_name); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar2d, kg/m/s, grid_name); diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index ad6b785d2af7..8766649cbaf7 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -29,10 +29,8 @@ set_grids(const std::shared_ptr grids_manager) using namespace ekat::units; using namespace ShortFieldTagsNames; - auto Q = kg/kg; - Q.set_string("kg/kg"); - auto m2 = m*m; - auto s2 = s*s; + auto m2 = pow(m,2); + auto s2 = pow(s,2); auto grid = grids_manager->get_grid("Physics"); const auto& grid_name = grid->name(); @@ -48,7 +46,7 @@ set_grids(const std::shared_ptr grids_manager) add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, ps); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, ps); // Only need phis if computing geopotential_* if (not m_only_compute_dz and not m_from_sea_level) { diff --git a/components/eamxx/src/diagnostics/virtual_temperature.cpp b/components/eamxx/src/diagnostics/virtual_temperature.cpp index 9ddcd18549bb..47b042ffb096 100644 --- a/components/eamxx/src/diagnostics/virtual_temperature.cpp +++ b/components/eamxx/src/diagnostics/virtual_temperature.cpp @@ -15,9 +15,6 @@ void VirtualTemperatureDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank @@ -27,8 +24,8 @@ void VirtualTemperatureDiagnostic::set_grids(const std::shared_ptr("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("qv", scalar3d_layout_mid, Q, grid_name, ps); + add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, ps); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar3d_layout_mid, K, grid_name); diff --git a/components/eamxx/src/diagnostics/water_path.cpp b/components/eamxx/src/diagnostics/water_path.cpp index 2e8da274bfbe..15fa5cdef38c 100644 --- a/components/eamxx/src/diagnostics/water_path.cpp +++ b/components/eamxx/src/diagnostics/water_path.cpp @@ -42,9 +42,7 @@ set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; - const auto m2 = m*m; - auto Q = kg/kg; - Q.set_string("kg/kg"); + auto m2 = pow (m,2); auto grid = grids_manager->get_grid("Physics"); const auto& grid_name = grid->name(); @@ -55,8 +53,8 @@ set_grids(const std::shared_ptr grids_manager) auto scalar3d = grid->get_3d_scalar_layout(true); // The fields required for this diagnostic to be computed - add_field("pseudo_density", scalar3d, Pa, grid_name); - add_field(m_qname, scalar3d, Q, grid_name); + add_field("pseudo_density", scalar3d, Pa, grid_name); + add_field(m_qname, scalar3d, kg/kg, grid_name); // Construct and allocate the diagnostic field FieldIdentifier fid (name(), scalar2d, kg/m2, grid_name); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp index 4e815c7266a9..a121600536b8 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp @@ -238,8 +238,6 @@ ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& g if (trace_gases_workaround.is_restart()) return; // always false b/c it hasn't been set yet using namespace ekat::units; using namespace ShortFieldTagsNames; - auto molmol = mol/mol; - molmol.set_string("mol/mol"); const auto& rgn = m_cgll_grid->name(); const auto& pgn = m_phys_grid->name(); const auto rnc = m_cgll_grid->get_num_local_dofs(); @@ -247,10 +245,10 @@ ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& g const auto nlev = m_cgll_grid->get_num_vertical_levels(); constexpr int ps = SCREAM_SMALL_PACK_SIZE; for (const auto& e : trace_gases_workaround.get_active_gases()) { - add_field(e, FieldLayout({COL,LEV},{rnc,nlev}), molmol, rgn, ps); + add_field(e, FieldLayout({COL,LEV},{rnc,nlev}), mol/mol, rgn, ps); // 'Updated' rather than just 'Computed' so that it gets written to the // restart file. - add_field(e, FieldLayout({COL,LEV},{pnc,nlev}), molmol, pgn, ps); + add_field(e, FieldLayout({COL,LEV},{pnc,nlev}), mol/mol, pgn, ps); } trace_gases_workaround.set_remapper(gm->create_remapper(m_cgll_grid, m_dyn_grid)); } diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index dfd116b0832b..5c71bf92244c 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -115,10 +115,6 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m constexpr int QTL = HOMMEXX_Q_NUM_TIME_LEVELS; constexpr int N = HOMMEXX_PACK_SIZE; - // Some units - auto Q = kg/kg; - Q.set_string("kg/kg"); - const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const int nlev_mid = m_dyn_grid->get_num_vertical_levels(); const int nlev_int = nlev_mid+1; @@ -161,8 +157,8 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m // into the n0 time-slice of Homme's vtheta_dp, and then do the conversion // T_mid->VTheta_dp in place. - const auto m2 = m*m; - const auto s2 = s*s; + const auto m2 = pow(m,2); + const auto s2 = pow(s,2); // Note: qv is needed to transform T<->Theta @@ -176,7 +172,7 @@ void HommeDynamics::set_grids (const std::shared_ptr grids_m add_field("pseudo_density", pg_scalar3d_mid, Pa, pgn,N); add_field("pseudo_density_dry", pg_scalar3d_mid, Pa, pgn,N); add_field ("ps", pg_scalar2d , Pa, pgn); - add_field("qv", pg_scalar3d_mid, Q, pgn,"tracers",N); + add_field("qv", pg_scalar3d_mid, kg/kg, pgn,"tracers",N); add_field("phis", pg_scalar2d , m2/s2, pgn); add_field("p_int", pg_scalar3d_int, Pa, pgn,N); add_field("p_mid", pg_scalar3d_mid, Pa, pgn,N); diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 270f1ffbdda7..87009c7d074e 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -140,6 +140,7 @@ void HommeGridsManager::build_dynamics_grid () { } using gid_type = AbstractGrid::gid_type; + using namespace ekat::units; // Get dimensions and create "empty" grid const int nlelem = get_num_local_elems_f90(); @@ -149,7 +150,7 @@ void HommeGridsManager::build_dynamics_grid () { dyn_grid->setSelfPointer(dyn_grid); const auto layout2d = dyn_grid->get_2d_scalar_layout(); - const auto rad = ekat::units::Units::nondimensional(); + const Units rad (Units::nondimensional(),"rad"); // Filling the cg/dg gids, elgpgp, coords, lat/lon views auto dg_dofs = dyn_grid->get_dofs_gids(); @@ -218,8 +219,9 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { // Create the gids, coords, area views using namespace ShortFieldTagsNames; + using namespace ekat::units; const auto layout2d = phys_grid->get_2d_scalar_layout(); - const auto rad = ekat::units::Units::nondimensional(); + const Units rad (Units::nondimensional(),"rad"); auto dofs = phys_grid->get_dofs_gids(); auto lat = phys_grid->create_geometry_data("lat",layout2d,rad); @@ -260,15 +262,15 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { if (get_grid("Dynamics")->has_geometry_data("hyam")) { auto layout_mid = phys_grid->get_vertical_layout(true); auto layout_int = phys_grid->get_vertical_layout(false); - const auto nondim = ekat::units::Units::nondimensional(); - auto lev_unit = ekat::units::Units::nondimensional();; - lev_unit.set_string("mb"); + using namespace ekat::units; + Units nondim = Units::nondimensional(); + Units mbar(bar/1000,"mb"); auto hyai = phys_grid->create_geometry_data("hyai",layout_int,nondim); auto hybi = phys_grid->create_geometry_data("hybi",layout_int,nondim); auto hyam = phys_grid->create_geometry_data("hyam",layout_mid,nondim); auto hybm = phys_grid->create_geometry_data("hybm",layout_mid,nondim); - auto lev = phys_grid->create_geometry_data("lev", layout_mid, lev_unit); + auto lev = phys_grid->create_geometry_data("lev", layout_mid,mbar); for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); diff --git a/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp b/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp index 28015bc7e9ab..6a7b2bf04e8c 100644 --- a/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp +++ b/components/eamxx/src/physics/cld_fraction/eamxx_cld_fraction_process_interface.cpp @@ -24,8 +24,6 @@ void CldFraction::set_grids(const std::shared_ptr grids_mana // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Q = kg/kg; - Q.set_string("kg/kg"); auto nondim = Units::nondimensional(); m_grid = grids_manager->get_grid("Physics"); @@ -40,7 +38,7 @@ void CldFraction::set_grids(const std::shared_ptr grids_mana // Set of fields used strictly as input constexpr int ps = Pack::n; - add_field("qi", scalar3d_layout_mid, Q, grid_name,"tracers",ps); + add_field("qi", scalar3d_layout_mid, kg/kg, grid_name,"tracers",ps); add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name,ps); // Set of fields used strictly as output diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index d3d146797d1f..0f4b9f1559e5 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -31,16 +31,14 @@ Cosp::Cosp (const ekat::Comm& comm, const ekat::ParameterList& params) void Cosp::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; + using namespace ekat::prefixes; using namespace ShortFieldTagsNames; // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Q = kg/kg; - Q.set_string("kg/kg"); - auto nondim = Units::nondimensional(); - auto percent = Units::nondimensional(); - percent.set_string("%"); - auto micron = m / 1000000; + Units nondim = Units::nondimensional(); + Units percent (nondim,"%"); + auto micron = micro*m; m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -68,9 +66,9 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) //add_field("height_mid", scalar3d_mid, m, grid_name); //add_field("height_int", scalar3d_int, m, grid_name); add_field("T_mid", scalar3d_mid, K, grid_name); - add_field("qv", scalar3d_mid, Q, grid_name, "tracers"); - add_field("qc", scalar3d_mid, Q, grid_name, "tracers"); - add_field("qi", scalar3d_mid, Q, grid_name, "tracers"); + add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers"); + add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers"); + add_field("qi", scalar3d_mid, kg/kg, grid_name, "tracers"); add_field("cldfrac_rad", scalar3d_mid, nondim, grid_name); // Optical properties, should be computed in radiation interface add_field("dtau067", scalar3d_mid, nondim, grid_name); // 0.67 micron optical depth diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 4ad61b96fe07..8bc727c325e9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -91,13 +91,10 @@ void MAMMicrophysics::configure(const ekat::ParameterList& params) { void MAMMicrophysics::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; - auto q_unit = kg/kg; // mass mixing ratios [kg stuff / kg air] - q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; // number mixing ratios [# / kg air] - n_unit.set_string("#/kg"); - Units nondim(0,0,0,0,0,0,0); - const auto m2 = m*m; - const auto s2 = s*s; + Units nondim = Units::nondimensional(); + Units n_unit (1/kg,"#/kg"); // number mixing ratios [# / kg air] + const auto m2 = pow(m,2); + const auto s2 = pow(s,2); grid_ = grids_manager->get_grid("Physics"); const auto& grid_name = grid_->name(); @@ -125,8 +122,8 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // vertical pressure velocity add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // specific humidity - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice wet mixing ratio + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers"); // specific humidity + add_field("qi", scalar3d_layout_mid, kg/kg, grid_name, "tracers"); // ice wet mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // p_del, hydrostatic pressure @@ -134,7 +131,7 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction // droplet activation can alter cloud liquid and number mixing ratios - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid wet mixing ratio + add_field("qc", scalar3d_layout_mid, kg/kg, grid_name, "tracers"); // cloud liquid wet mixing ratio add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud liquid wet number mixing ratio // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios @@ -144,7 +141,7 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_layout_mid, kg/kg, grid_name, "tracers"); } } } @@ -152,7 +149,7 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ // aerosol-related gases: mass mixing ratios for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field(gas_mmr_field_name, scalar3d_layout_mid, kg/kg, grid_name, "tracers"); } // Tracers group -- do we need this in addition to the tracers above? In any diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index d818890a9b46..a55bf84ca17a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -25,12 +25,9 @@ void MAMOptics::set_grids( grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - auto q_unit = kg / kg; // mass mixing ratios [kg stuff / kg air] - q_unit.set_string("kg/kg"); - auto n_unit = 1 / kg; // number mixing ratios [# / kg air] - n_unit.set_string("#/kg"); - const auto m2 = m * m; - const auto s2 = s * s; + Units n_unit (1 / kg, "#/kg"); // number mixing ratios [# / kg air] + const auto m2 = pow(m,2); + const auto s2 = pow(s,2); ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // number of levels per column @@ -58,12 +55,12 @@ void MAMOptics::set_grids( add_field("p_int", scalar3d_int, Pa, grid_name); // total pressure add_field("pseudo_density", scalar3d_mid, Pa, grid_name); add_field("pseudo_density_dry", scalar3d_mid, Pa, grid_name); - add_field("qv", scalar3d_mid, q_unit, grid_name,"tracers"); // specific humidity - add_field("qi", scalar3d_mid, q_unit, grid_name,"tracers"); // ice wet mixing ratio + add_field("qv", scalar3d_mid, kg/kg, grid_name,"tracers"); // specific humidity + add_field("qi", scalar3d_mid, kg/kg, grid_name,"tracers"); // ice wet mixing ratio add_field("ni", scalar3d_mid, n_unit, grid_name,"tracers"); // ice number mixing ratio // droplet activation can alter cloud liquid and number mixing ratios - add_field("qc", scalar3d_mid, q_unit, grid_name,"tracers"); // cloud liquid wet mixing ratio + add_field("qc", scalar3d_mid, kg/kg, grid_name,"tracers"); // cloud liquid wet mixing ratio add_field("nc", scalar3d_mid, n_unit, grid_name,"tracers"); // cloud liquid wet number mixing ratio add_field("phis", scalar2d, m2 / s2, grid_name); @@ -94,7 +91,7 @@ void MAMOptics::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit,grid_name, "tracers"); + add_field(int_mmr_field_name, scalar3d_mid, kg/kg,grid_name, "tracers"); } } } @@ -108,7 +105,7 @@ void MAMOptics::set_grids( mam_coupling::cld_aero_mmr_field_name(m, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + add_field(cld_mmr_field_name, scalar3d_mid, kg/kg, grid_name); } } } @@ -116,7 +113,7 @@ void MAMOptics::set_grids( // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); + add_field(gas_mmr_field_name, scalar3d_mid, kg/kg, grid_name, "tracers"); } } diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 171e3486ba4f..6047188735a1 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -24,8 +24,6 @@ void MLCorrection::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Q = kg / kg; - Q.set_string("kg/kg"); constexpr int ps = Pack::n; m_grid = grids_manager->get_grid("Physics"); const auto &grid_name = m_grid->name(); @@ -40,16 +38,15 @@ void MLCorrection::set_grids( FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); FieldLayout vector3d_mid = m_grid->get_3d_vector_layout(true,2); - const auto m2 = m*m; + const auto m2 = pow(m,2); if (not m_ML_correction_unit_test) { - const auto s2 = s*s; - auto Wm2 = W / m / m; - auto nondim = m/m; + const auto s2 = pow(s,2); + auto nondim = Units::nondimensional(); add_field("phis", scalar2d, m2/s2, grid_name); add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); - add_field("SW_flux_dn", scalar3d_int, Wm2, grid_name, ps); - add_field("sfc_flux_sw_net", scalar2d, Wm2, grid_name); - add_field("sfc_flux_lw_dn", scalar2d, Wm2, grid_name); + add_field("SW_flux_dn", scalar3d_int, W/m2, grid_name, ps); + add_field("sfc_flux_sw_net", scalar2d, W/m2, grid_name); + add_field("sfc_flux_lw_dn", scalar2d, W/m2, grid_name); m_lat = m_grid->get_geometry_data("lat"); m_lon = m_grid->get_geometry_data("lon"); } @@ -60,9 +57,9 @@ void MLCorrection::set_grids( * is adapting the infrastructure to allow for a generic "add_field" call * to be used here which we can then setup using the m_fields_ml_output_variables variable */ - add_field("T_mid", scalar3d_mid, K, grid_name, ps); - add_field("qv", scalar3d_mid, Q, grid_name, "tracers", ps); - add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); + add_field("T_mid", scalar3d_mid, K, grid_name, ps); + add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); + add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); /* Note: we also need to update the precipitation after ML commits any column drying */ add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); add_field("precip_liq_surf_mass", scalar2d, kg/m2, grid_name); @@ -107,7 +104,6 @@ void MLCorrection::run_impl(const double dt) { host_view2d_type qv_told("",qv.extent(0),qv.extent(1)); Kokkos::deep_copy(qv_told,qv); - auto h_lat = m_lat.get_view(); auto h_lon = m_lon.get_view(); diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 832c030e97b9..3af33eaea164 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -68,8 +68,6 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) FieldLayout horiz_wind_layout = m_grid->get_3d_vector_layout(true,2); constexpr int ps = 1; - auto Q = kg/kg; - Q.set_string("kg/kg"); add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); /* ----------------------- WARNING --------------------------------*/ @@ -85,7 +83,7 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); } if (ekat::contains(m_fields_nudge,"qv")) { - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); } if (ekat::contains(m_fields_nudge,"U") or ekat::contains(m_fields_nudge,"V")) { add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index d6264c38b8b7..d886478ac2af 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -25,14 +25,13 @@ P3Microphysics::P3Microphysics (const ekat::Comm& comm, const ekat::ParameterLis void P3Microphysics::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; + using namespace ekat::prefixes; // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Q = kg/kg; - Q.set_string("kg/kg"); auto nondim = Units::nondimensional(); - auto micron = m / 1000000; - auto m2 = m * m; + auto micron = micro*m; + auto m2 = pow(m,2); m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -73,27 +72,27 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field ("T_mid", scalar3d_layout_mid, K, grid_name, ps); // T_mid is the only one of these variables that is also updated. // Prognostic State: (all fields are both input and output) - add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("qc", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("qr", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("qi", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("qm", scalar3d_layout_mid, Q, grid_name, "tracers", ps); - add_field("nc", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); - add_field("nr", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); - add_field("ni", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); - add_field("bm", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); + add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); + add_field("qc", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); + add_field("qr", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); + add_field("qi", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); + add_field("qm", scalar3d_layout_mid, kg/kg, grid_name, "tracers", ps); + add_field("nc", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); + add_field("nr", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); + add_field("ni", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); + add_field("bm", scalar3d_layout_mid, 1/kg, grid_name, "tracers", ps); // Diagnostic Inputs: (only the X_prev fields are both input and output, all others are just inputs) add_field("nc_nuceat_tend", scalar3d_layout_mid, 1/(kg*s), grid_name, ps); if (infrastructure.prescribedCCN) { add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); } - add_field("ni_activated", scalar3d_layout_mid, 1/kg, grid_name, ps); - add_field("inv_qc_relvar", scalar3d_layout_mid, Q*Q, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name, ps); - add_field ("qv_prev_micro_step", scalar3d_layout_mid, Q, grid_name, ps); - add_field ("T_prev_micro_step", scalar3d_layout_mid, K, grid_name, ps); + add_field("ni_activated", scalar3d_layout_mid, 1/kg, grid_name, ps); + add_field("inv_qc_relvar", scalar3d_layout_mid, pow(kg/kg,2), grid_name, ps); + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); + add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name, ps); + add_field ("qv_prev_micro_step", scalar3d_layout_mid, kg/kg, grid_name, ps); + add_field ("T_prev_micro_step", scalar3d_layout_mid, K, grid_name, ps); // Diagnostic Outputs: (all fields are just outputs w.r.t. P3) add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); @@ -106,9 +105,9 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m // TODO: These should be averaged over subcycle as well. But there is no simple mechanism // yet to reset these values at the beginning of the atmosphere timestep. When this // mechanism is developed we should add these variables to the accumulated variables. - add_field("micro_liq_ice_exchange", scalar3d_layout_mid, Q, grid_name, ps); - add_field("micro_vap_liq_exchange", scalar3d_layout_mid, Q, grid_name, ps); - add_field("micro_vap_ice_exchange", scalar3d_layout_mid, Q, grid_name, ps); + add_field("micro_liq_ice_exchange", scalar3d_layout_mid, kg/kg, grid_name, ps); + add_field("micro_vap_liq_exchange", scalar3d_layout_mid, kg/kg, grid_name, ps); + add_field("micro_vap_ice_exchange", scalar3d_layout_mid, kg/kg, grid_name, ps); add_field("rainfrac", scalar3d_layout_mid, nondim, grid_name, ps); // Boundary flux fields for energy and mass conservation checks diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 256174baef63..88f7cdc68eda 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -42,17 +42,12 @@ RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params) void RRTMGPRadiation::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; + using namespace ekat::prefixes; // Declare the set of fields used by rrtmgp - auto kgkg = kg/kg; - kgkg.set_string("kg/kg"); - auto m2 = m * m; - auto Wm2 = W / m / m; - Wm2.set_string("W/m2"); - auto nondim = m/m; // dummy unit for non-dimensional fields - auto micron = m / 1000000; - auto molmol = mol/mol; - molmol.set_string("mol/mol"); + Units m2(m*m,"m2"); + auto nondim = Units::nondimensional(); + auto micron = micro*m; m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -92,24 +87,24 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ add_field("sfc_alb_dir_nir", scalar2d, nondim, grid_name); add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); add_field("sfc_alb_dif_nir", scalar2d, nondim, grid_name); - add_field("qc", scalar3d_mid, kgkg, grid_name); + add_field("qc", scalar3d_mid, kg/kg, grid_name); add_field("nc", scalar3d_mid, 1/kg, grid_name); - add_field("qi", scalar3d_mid, kgkg, grid_name); + add_field("qi", scalar3d_mid, kg/kg, grid_name); add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); add_field("eff_radius_qc", scalar3d_mid, micron, grid_name); add_field("eff_radius_qi", scalar3d_mid, micron, grid_name); - add_field("qv",scalar3d_mid,kgkg,grid_name); + add_field("qv",scalar3d_mid,kg/kg,grid_name); add_field("surf_lw_flux_up",scalar2d,W/(m*m),grid_name); // Set of required gas concentration fields for (auto& it : m_gas_names) { // Add gas VOLUME mixing ratios (moles of gas / moles of air; what actually gets input to RRTMGP) if (it == "o3") { // o3 is read from file, or computed by chemistry - add_field(it + "_volume_mix_ratio", scalar3d_mid, molmol, grid_name); + add_field(it + "_volume_mix_ratio", scalar3d_mid, mol/mol, grid_name); } else { // the rest are computed by RRTMGP from prescribed surface values // NOTE: this may change at some point - add_field(it + "_volume_mix_ratio", scalar3d_mid, molmol, grid_name); + add_field(it + "_volume_mix_ratio", scalar3d_mid, mol/mol, grid_name); } } // Required aerosol optical properties from SPA @@ -127,26 +122,26 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ // Set computed (output) fields add_field("T_mid" , scalar3d_mid, K , grid_name); - add_field("SW_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("SW_flux_up", scalar3d_int, Wm2, grid_name); - add_field("SW_flux_dn_dir", scalar3d_int, Wm2, grid_name); - add_field("LW_flux_up", scalar3d_int, Wm2, grid_name); - add_field("LW_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("SW_clnclrsky_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("SW_clnclrsky_flux_up", scalar3d_int, Wm2, grid_name); - add_field("SW_clnclrsky_flux_dn_dir", scalar3d_int, Wm2, grid_name); - add_field("SW_clrsky_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("SW_clrsky_flux_up", scalar3d_int, Wm2, grid_name); - add_field("SW_clrsky_flux_dn_dir", scalar3d_int, Wm2, grid_name); - add_field("SW_clnsky_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("SW_clnsky_flux_up", scalar3d_int, Wm2, grid_name); - add_field("SW_clnsky_flux_dn_dir", scalar3d_int, Wm2, grid_name); - add_field("LW_clnclrsky_flux_up", scalar3d_int, Wm2, grid_name); - add_field("LW_clnclrsky_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("LW_clrsky_flux_up", scalar3d_int, Wm2, grid_name); - add_field("LW_clrsky_flux_dn", scalar3d_int, Wm2, grid_name); - add_field("LW_clnsky_flux_up", scalar3d_int, Wm2, grid_name); - add_field("LW_clnsky_flux_dn", scalar3d_int, Wm2, grid_name); + add_field("SW_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("SW_flux_up", scalar3d_int, W/m2, grid_name); + add_field("SW_flux_dn_dir", scalar3d_int, W/m2, grid_name); + add_field("LW_flux_up", scalar3d_int, W/m2, grid_name); + add_field("LW_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("SW_clnclrsky_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("SW_clnclrsky_flux_up", scalar3d_int, W/m2, grid_name); + add_field("SW_clnclrsky_flux_dn_dir", scalar3d_int, W/m2, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_int, W/m2, grid_name); + add_field("SW_clrsky_flux_dn_dir", scalar3d_int, W/m2, grid_name); + add_field("SW_clnsky_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("SW_clnsky_flux_up", scalar3d_int, W/m2, grid_name); + add_field("SW_clnsky_flux_dn_dir", scalar3d_int, W/m2, grid_name); + add_field("LW_clnclrsky_flux_up", scalar3d_int, W/m2, grid_name); + add_field("LW_clnclrsky_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("LW_clrsky_flux_up", scalar3d_int, W/m2, grid_name); + add_field("LW_clrsky_flux_dn", scalar3d_int, W/m2, grid_name); + add_field("LW_clnsky_flux_up", scalar3d_int, W/m2, grid_name); + add_field("LW_clnsky_flux_dn", scalar3d_int, W/m2, grid_name); add_field("rad_heating_pdel", scalar3d_mid, Pa*K/s, grid_name); // Cloud properties added as computed fields for diagnostic purposes add_field("cldlow" , scalar2d, nondim, grid_name); @@ -179,12 +174,12 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ // netsw sfc_flux_sw_net net (down - up) SW flux at surface // flwds sfc_flux_lw_dn downwelling LW flux at surface // -------------------------------------------------------------- - add_field("sfc_flux_dir_nir", scalar2d, Wm2, grid_name); - add_field("sfc_flux_dir_vis", scalar2d, Wm2, grid_name); - add_field("sfc_flux_dif_nir", scalar2d, Wm2, grid_name); - add_field("sfc_flux_dif_vis", scalar2d, Wm2, grid_name); - add_field("sfc_flux_sw_net" , scalar2d, Wm2, grid_name); - add_field("sfc_flux_lw_dn" , scalar2d, Wm2, grid_name); + add_field("sfc_flux_dir_nir", scalar2d, W/m2, grid_name); + add_field("sfc_flux_dir_vis", scalar2d, W/m2, grid_name); + add_field("sfc_flux_dif_nir", scalar2d, W/m2, grid_name); + add_field("sfc_flux_dif_vis", scalar2d, W/m2, grid_name); + add_field("sfc_flux_sw_net" , scalar2d, W/m2, grid_name); + add_field("sfc_flux_lw_dn" , scalar2d, W/m2, grid_name); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 2504c7ddd8af..6204421c1ea5 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -23,12 +23,6 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Qunit = kg/kg; - Qunit.set_string("kg/kg"); - auto nondim = Units::nondimensional(); - m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -56,21 +50,22 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids constexpr int ps = Spack::n; - const auto m2 = m*m; - const auto s2 = s*s; + const auto nondim = Units::nondimensional(); + const auto m2 = pow(m,2); + const auto s2 = pow(s,2); // These variables are needed by the interface, but not actually passed to shoc_main. add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); add_field("surf_sens_flux", scalar2d , W/m2, grid_name); add_field("surf_mom_flux", vector2d , N/m2, grid_name); - add_field("surf_evap", scalar2d , kg/m2/s, grid_name); - add_field ("T_mid", scalar3d_mid, K, grid_name, ps); - add_field ("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("surf_evap", scalar2d , kg/(m2*s), grid_name); + add_field ("T_mid", scalar3d_mid, K, grid_name, ps); + add_field ("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { - add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); } // Input variables @@ -84,19 +79,19 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, pow(kg/kg,2), grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); + add_field("vapor_flux", scalar2d, kg/(m2*s), grid_name); add_field("water_flux", scalar2d, m/s, grid_name); add_field("ice_flux", scalar2d, m/s, grid_name); add_field("heat_flux", scalar2d, W/m2, grid_name); diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index 89f521e3d966..61c61b878c2f 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -26,10 +26,6 @@ void SPA::set_grids(const std::shared_ptr grids_manager) using namespace ekat::units; using namespace ShortFieldTagsNames; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Q = kg/kg; - Q.set_string("kg/kg"); auto nondim = Units::nondimensional(); m_grid = grids_manager->get_grid("Physics"); diff --git a/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp index 87e145917987..a8996c3dd400 100644 --- a/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp +++ b/components/eamxx/src/physics/tms/eamxx_tms_process_interface.cpp @@ -26,10 +26,8 @@ void TurbulentMountainStress::set_grids(const std::shared_ptrget_grid("Physics"); @@ -52,12 +50,12 @@ void TurbulentMountainStress::set_grids(const std::shared_ptr("T_mid", scalar3d_mid, K, grid_name, ps); add_field("p_mid", scalar3d_mid, Pa, grid_name, ps); add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("sgh30", scalar2d , m, grid_name); add_field("landfrac", scalar2d , nondim, grid_name); - add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); - add_field("wind_stress_tms", vector2d, N/m2, grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); + add_field("wind_stress_tms", vector2d, N/m2, grid_name); } // ========================================================================================= diff --git a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp index 2dbb048957e6..f36aac6133d9 100644 --- a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp +++ b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp @@ -29,12 +29,7 @@ ZMDeepConvection::ZMDeepConvection (const ekat::Comm& comm,const ekat::Parameter void ZMDeepConvection::set_grids(const std::shared_ptr grids_manager) { using namespace std; - using namespace ekat; - using namespace units; - - auto Q = kg/kg; - auto nondim = m/m; - Q.set_string("kg/kg"); + using namespace ekat::units; constexpr int NVL = 72; /* TODO THIS NEEDS TO BE CHANGED TO A CONFIGURABLE */ constexpr int QSZ = 35; /* TODO THIS NEEDS TO BE CHANGED TO A CONFIGURABLE */ @@ -57,9 +52,9 @@ void ZMDeepConvection::set_grids(const std::shared_ptr grids set_grid_opts(opt_map); for ( auto i = opt_map.begin(); i != opt_map.end(); ++i) { - add_required_field((i->second).name, layout_opts[((i->second).field_idx)], Q, grid->name()); + add_required_field((i->second).name, layout_opts[((i->second).field_idx)], kg/kg, grid->name()); if ( (i->second).isOut == true ) { - add_computed_field((i->second).name, layout_opts[((i->second).field_idx)], Q, grid->name()); + add_computed_field((i->second).name, layout_opts[((i->second).field_idx)], kg/kg, grid->name()); } } diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp index a77dd6e4c56c..7b6fc218a0b8 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp @@ -146,7 +146,7 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const s.back() = ')'; if (verbosity>2) { - s += " [" + fid.get_units().get_string() + "]"; + s += " [" + fid.get_units().to_string() + "]"; } } return s; diff --git a/components/eamxx/src/share/field/field_identifier.cpp b/components/eamxx/src/share/field/field_identifier.cpp index 56974aa06968..7430ffa892e2 100644 --- a/components/eamxx/src/share/field/field_identifier.cpp +++ b/components/eamxx/src/share/field/field_identifier.cpp @@ -48,7 +48,7 @@ void FieldIdentifier::update_identifier () { m_identifier += ">"; m_identifier += "(" + ekat::join(m_layout.dims(),",") + ")"; - m_identifier += " [" + m_units.get_string() + "]"; + m_identifier += " [" + m_units.to_string() + "]"; } // Free functions for identifiers comparison diff --git a/components/eamxx/src/share/field/field_manager.cpp b/components/eamxx/src/share/field/field_manager.cpp index 4db9fd0fa15e..311a0a6b14be 100644 --- a/components/eamxx/src/share/field/field_manager.cpp +++ b/components/eamxx/src/share/field/field_manager.cpp @@ -46,8 +46,8 @@ void FieldManager::register_field (const FieldRequest& req) const auto id0 = m_fields[id.name()]->get_header().get_identifier(); EKAT_REQUIRE_MSG(id.get_units()==id0.get_units(), "Error! Field '" + id.name() + "' already registered with different units:\n" - " - input field units: " + to_string(id.get_units()) + "\n" - " - stored field units: " + to_string(id0.get_units()) + "\n" + " - input field units: " + id.get_units().to_string() + "\n" + " - stored field units: " + id0.get_units().to_string() + "\n" " Please, check and make sure all atmosphere processes use the same units.\n"); EKAT_REQUIRE_MSG(id.get_layout()==id0.get_layout(), diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index e0f78e10cf06..e7e11ef20e2a 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -230,14 +230,15 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using geo_view_host = AtmosphereInput::view_1d_host; using namespace ShortFieldTagsNames; + using namespace ekat::units; + FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); - const auto units = ekat::units::Units::nondimensional(); - auto lev_unit = ekat::units::Units::nondimensional();; - lev_unit.set_string("mb"); + Units nondim = Units::nondimensional(); + Units mbar (100*Pa,"mb"); - auto hyam = grid->create_geometry_data("hyam", layout_mid, units); - auto hybm = grid->create_geometry_data("hybm", layout_mid, units); - auto lev = grid->create_geometry_data("lev", layout_mid, lev_unit); + auto hyam = grid->create_geometry_data("hyam", layout_mid, nondim); + auto hybm = grid->create_geometry_data("hybm", layout_mid, nondim); + auto lev = grid->create_geometry_data("lev", layout_mid, mbar); // Create host mirrors for reading in data std::map host_views = { diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 3db201f424ba..1852785eeacf 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -953,7 +953,7 @@ register_variables(const std::string& filename, // in the simulation and not the one used in the output file. const auto& layout = fid.get_layout(); auto vec_of_dims = set_vec_of_dims(layout); - std::string units = fid.get_units().get_string(); + std::string units = fid.get_units().to_string(); // Gather longname auto longname = m_longnames.get_longname(name); From 6c2660373f19da419eeae9932c6a9f2e27884b58 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 15 May 2024 20:58:44 -0600 Subject: [PATCH 192/476] EAMxx: fix compiler warning --- .../eamxx/src/diagnostics/tests/number_paths_tests.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp index 850c535529a2..1fb23ef94228 100644 --- a/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/number_paths_tests.cpp @@ -37,8 +37,6 @@ template void run(std::mt19937_64 &engine) { using PC = scream::physics::Constants; using KT = ekat::KokkosTypes; - using ESU = ekat::ExeSpaceUtils; - using MemberType = typename KT::MemberType; using view_1d = typename KT::template view_1d; constexpr int num_levs = 33; @@ -52,9 +50,6 @@ void run(std::mt19937_64 &engine) { const int ncols = 10; auto gm = create_gm(comm, ncols, num_levs); - // Kokkos Policy - auto policy = ESU::get_default_team_policy(ncols, num_levs); - // Input (randomized) views view_1d pseudo_density("pseudo_density", num_levs), qc("qc", num_levs), nc("nc", num_levs), qr("qr", num_levs), nr("nr", num_levs), From 56dd496e8e9116a861d4b8491d20f1071a988e81 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 16 May 2024 10:32:32 -0600 Subject: [PATCH 193/476] EAMxx: fix weaver env setup for standalone testing --- components/eamxx/scripts/machines_specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index fbaabf5e5dec..9536d415c0de 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -23,7 +23,7 @@ ["mpicxx","mpifort","mpicc"], "salloc -N 1 srun -n1 --preserve-env", "/home/projects/e3sm/scream/pr-autotester/master-baselines/blake/"), - "weaver" : (["source /etc/profile.d/modules.sh", "module purge", "module load cmake/3.25.1 git/2.39.1 python/3.10.8 py-netcdf4/1.5.8 gcc/11.3.0 cuda/11.8.0 openmpi netcdf-c netcdf-fortran parallel-netcdf netlib-lapack", + "weaver" : (["source /etc/profile.d/modules.sh", "module purge", "module load cmake/3.25.1 git/2.39.1 python/3.10.8 py-netcdf4/1.5.8 gcc/11.3.0 cuda/11.8.0 openmpi netcdf-c netcdf-fortran parallel-netcdf netlib-lapack", "export HDF5_USE_FILE_LOCKING=FALSE", ], ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", From 16e9fe88c8d5d1daa6ca7f6c5c4ee0c4314d8a56 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 16 May 2024 14:55:46 -0600 Subject: [PATCH 194/476] GH feedback --- components/eamxx/CMakeLists.txt | 7 ++----- .../src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index cf8936c940ad..8567512abf2b 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -189,9 +189,6 @@ if (SCREAM_DEBUG) endif () endif() -set(DEFAULT_RRTMGP_ENABLE_YAKL TRUE) -set(DEFAULT_RRTMGP_ENABLE_KOKKOS FALSE) - ### Now that reasonable defaults have been computed, set CACHE vars set(SCREAM_MIMIC_GPU ${DEFAULT_MIMIC_GPU} CACHE BOOL "Mimic GPU to correctness-test inter-column parallelism on non-GPU platform") set(SCREAM_PACK_CHECK_BOUNDS FALSE CACHE BOOL "If defined, scream::pack objects check indices against bounds") @@ -213,8 +210,8 @@ endif() # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) -option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" ${DEFAULT_RRTMGP_ENABLE_YAKL}) -option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" ${DEFAULT_RRTMGP_ENABLE_KOKKOS}) +option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE) +option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE) if (SCREAM_RRTMGP_ENABLE_YAKL) add_definitions("-DRRTMGP_ENABLE_YAKL") endif() diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 646de6931a07..0f701c0253db 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -1193,7 +1193,6 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS COMPARE_WRAP(tmp2d, tmp2d_k); m_gas_concs_k.set_vmr(name, tmp2d_k); - //VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); broken for some reason #endif } @@ -1378,7 +1377,6 @@ void RRTMGPRadiation::run_impl (const double dt) { COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); - //VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif // Update heating tendency @@ -1669,7 +1667,6 @@ void RRTMGPRadiation::run_impl (const double dt) { #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.concs = gas_concs_k; m_gas_concs_k.ncol = orig_ncol_k; - //VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); #endif } // update_rad From 2843c2ec131ca6ffb6c80e40166bc50e9a514ffe Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 16 May 2024 15:03:34 -0600 Subject: [PATCH 195/476] Fix build error --- .../tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 5197104787d0..3d8a91b50eeb 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -153,7 +153,8 @@ TEST_CASE("rrtmgp_scream_standalone", "") { // times. We will then fill some fraction of these columns with clouds for // the test problem. GasConcs gas_concs; - read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, ncol_all); + real2d col_dry; + read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); // Setup dummy problem; need to use tmp arrays with ncol_all size rrtmgpTest::dummy_atmos( inputfile, ncol_all, p_lay_all, t_lay_all, From 2106b8d74a9c6a0f0e519111ec772e1054df3f90 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 16 May 2024 15:32:38 -0600 Subject: [PATCH 196/476] Need to deallocate col_dry --- components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp | 1 + .../eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 26c7ff2b6305..65d1da9f1d68 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -309,6 +309,7 @@ int run_yakl(int argc, char** argv) { cld_tau_lw.deallocate(); cld_tau_sw_bnd.deallocate(); cld_tau_lw_bnd.deallocate(); + col_dry.deallocate(); yakl::finalize(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 3d8a91b50eeb..9e6e94a2164f 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -412,6 +412,7 @@ TEST_CASE("rrtmgp_scream_standalone", "") { sw_flux_dn_dir_loc.deallocate(); lw_flux_up_loc.deallocate(); lw_flux_dn_loc.deallocate(); + col_dry.deallocate(); // Finalize the driver. YAKL will be finalized inside // RRTMGPRadiation::finalize_impl after RRTMGP has had the From 5daa0c7dab1e22c48204d9f117131eea1ea5efca Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 17 May 2024 15:17:50 -0400 Subject: [PATCH 197/476] fix the typo and improve banner message --- components/eamxx/docs/overrides/main.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/docs/overrides/main.html b/components/eamxx/docs/overrides/main.html index f888486682d7..a36c22640666 100644 --- a/components/eamxx/docs/overrides/main.html +++ b/components/eamxx/docs/overrides/main.html @@ -3,6 +3,6 @@ {% block announce %} EAMxx is not supported yet. - Pleaae visit docs.e3sm.org instead! + Please visit docs.e3sm.org for general E3SM documentation. {% endblock %} From 3fba8857b8272f514fcd8be81da08d06f7dbc8e6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 17 May 2024 15:52:11 -0600 Subject: [PATCH 198/476] GPU fixes. No idea why the enable_ifs on some functions were causing problems on GPU --- .../eamxx/src/physics/rrtmgp/CMakeLists.txt | 1 + .../eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp | 6 +- .../rrtmgp/scream_rrtmgp_interface.cpp | 60 +++++++++---------- .../rrtmgp/scream_rrtmgp_interface.hpp | 6 +- .../rrtmgp/tests/rrtmgp_unit_tests.cpp | 6 +- 5 files changed, 40 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index 446c1842192e..c098a0829d52 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -139,6 +139,7 @@ target_include_directories(rrtmgp PUBLIC ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels + ${NetCDF_C_PATH}/include ) # Build RRTMGP interface; note that we separate the SCREAM-specific RRTMGP interface diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index f72acd7528e2..0dfe4945e0c8 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -83,7 +83,7 @@ inline bool radiation_do(const int irad, const int nstep) { // Verify that array only contains values within valid range, and if not // report min and max of array #ifdef RRTMGP_ENABLE_YAKL -template >::type* = nullptr> +template bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = minval(x); @@ -109,8 +109,8 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template >::type* = nullptr> -bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { +template +bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 203336f395dd..450d7ad216be 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -124,8 +124,8 @@ OpticalProps2strK get_cloud_optics_sw( // Limit effective radii to be within bounds of lookup table auto rel_limited = real2dk("rel_limited", ncol, nlay); auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); @@ -179,8 +179,8 @@ OpticalProps1sclK get_cloud_optics_lw( // Limit effective radii to be within bounds of lookup table auto rel_limited = real2dk("rel_limited", ncol, nlay); auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); // Calculate cloud optics cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); @@ -883,17 +883,17 @@ void rrtmgp_main( #ifdef SCREAM_RRTMGP_DEBUG // Sanity check inputs, and possibly repair - check_range(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); - check_range(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); - check_range(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); - check_range(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); - check_range(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); + check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); #endif // Setup pointers to RRTMGP SW fluxes @@ -962,10 +962,10 @@ void rrtmgp_main( // Check aerosol optical properties // NOTE: these should already have been checked by precondition checks, but someday we might have // non-trivial aerosol optics, so this is still good to do here. - check_range(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); + check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); #endif // Convert cloud physical properties to optical properties for input to RRTMGP @@ -999,10 +999,10 @@ void rrtmgp_main( // a parameterization of their own, and we might want to swap different choices. These checks go here // only because we need to run them on computed optical props, so if the optical props themselves get // computed up higher, then perform these checks higher as well - check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); #endif // Do shortwave @@ -1591,7 +1591,7 @@ void rrtmgp_sw( // Limit temperatures for gas optics look-up tables auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); - limit_to_bounds(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); // Do gas optics real2dk toa_flux("toa_flux", nday, ngpt); @@ -1609,9 +1609,9 @@ void rrtmgp_sw( #ifdef SCREAM_RRTMGP_DEBUG // Check gas optics - check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); #endif // Apply tsi_scaling @@ -1914,8 +1914,8 @@ void rrtmgp_lw( // Limit temperatures for gas optics look-up tables auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); - limit_to_bounds(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); - limit_to_bounds(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); // Do gas optics realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); @@ -1926,7 +1926,7 @@ void rrtmgp_lw( #ifdef SCREAM_RRTMGP_DEBUG // Check gas optics - check_range(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); #endif if (extra_clnclrsky_diag) { diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 291f1978e98a..49fbfeb4486b 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -338,7 +338,7 @@ void mixing_ratio_to_cloud_mass( * fields as well. */ #ifdef RRTMGP_ENABLE_YAKL -template>::type* = nullptr> +template void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); @@ -346,8 +346,8 @@ void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template>::type* = nullptr> -void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { +template +void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); }); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 5b560456183a..ddef61808fe0 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -1250,13 +1250,13 @@ TEST_CASE("rrtmgp_test_check_range_k") { auto dummy = real2dk("dummy", 2, 1); // All values within range Kokkos::deep_copy(dummy, 0.1); - REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == true); + REQUIRE(scream::rrtmgp::check_range_k(dummy, 0.0, 1.0, "dummy") == true); // At least one value below lower bound Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = -0.1;}); - REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); + REQUIRE(scream::rrtmgp::check_range_k(dummy, 0.0, 1.0, "dummy") == false); // At least one value above upper bound Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = 1.1;}); - REQUIRE(scream::rrtmgp::check_range(dummy, 0.0, 1.0, "dummy") == false); + REQUIRE(scream::rrtmgp::check_range_k(dummy, 0.0, 1.0, "dummy") == false); scream::finalize_kls(); } From ede1ab92e3a6d143ba2204b5c112235ae731a8fd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 20 May 2024 14:23:17 -0600 Subject: [PATCH 199/476] EAMxx: fix VerticalLayer diagnostic * Surface value for vertical integral was wrong * Ensure geopotential is indeed a potential * Use altitude for elevation above surface --- .../eamxx/src/diagnostics/field_at_height.cpp | 2 +- .../eamxx/src/diagnostics/vertical_layer.cpp | 70 ++++++++++++------- .../eamxx/src/diagnostics/vertical_layer.hpp | 9 ++- .../eamxx/src/share/io/scorpio_output.cpp | 41 +++++------ 4 files changed, 69 insertions(+), 53 deletions(-) diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index 3982ed9a3ac9..cb2346e5595e 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -44,7 +44,7 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) " - field name: " + m_field_name + "\n" " - surface reference: " + surf_ref + "\n" " - valid options: sealevel, surface\n"); - m_z_name = (surf_ref == "sealevel") ? "z" : "geopotential"; + m_z_name = (surf_ref == "sealevel") ? "z" : "altitude"; const auto& location = m_params.get("vertical_location"); auto chars_start = location.find_first_not_of("0123456789."); EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index 8766649cbaf7..9e15b68aa19a 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -11,16 +11,14 @@ VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& para m_diag_name = params.get("diag_name"); EKAT_REQUIRE_MSG(m_diag_name == "z_int" or m_diag_name == "z_mid" or m_diag_name == "geopotential_int" or m_diag_name == "geopotential_mid" or + m_diag_name == "altitude_int" or m_diag_name == "altitude_mid" or m_diag_name == "dz", "Error! VerticalLayerDiagnostic has been given an unknown name: "+m_diag_name+".\n"); - m_only_compute_dz = (m_diag_name == "dz"); m_is_interface_layout = m_diag_name.find("_int") != std::string::npos; - // Whether or not diagnostic is computed from sea level depends on the name. - // "z_" -> from sea level, "geopotential_" -> from topography data. - // This boolean is irrelevant for vertical layer thickness (dz). - m_from_sea_level = m_diag_name.find("z_") != std::string::npos; + m_geopotential = m_diag_name.substr(0,12)=="geopotential"; + m_from_sea_level = m_diag_name[0]=='z' or m_geopotential; } // ======================================================================================== void VerticalLayerDiagnostic:: @@ -49,7 +47,7 @@ set_grids(const std::shared_ptr grids_manager) add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, ps); // Only need phis if computing geopotential_* - if (not m_only_compute_dz and not m_from_sea_level) { + if (not m_geopotential) { add_field("phis", scalar2d_layout, m2/s2, grid_name); } @@ -62,7 +60,7 @@ set_grids(const std::shared_ptr grids_manager) m_diagnostic_output.allocate_view(); // Initialize temporary views based on need. - if (not m_only_compute_dz) { + if (m_diag_name!="dz") { if (m_is_interface_layout) { const auto npacks = ekat::npack(m_num_levs); m_tmp_midpoint_view = view_2d("tmp_mid",m_num_cols,npacks); @@ -75,6 +73,10 @@ set_grids(const std::shared_ptr grids_manager) // ========================================================================================= void VerticalLayerDiagnostic::compute_diagnostic_impl() { + using column_ops = ColumnOps; + // To use in column_ops, since we integrate from surface + constexpr bool FromTop = false; + const auto npacks = ekat::npack(m_num_levs); const auto default_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); @@ -84,14 +86,16 @@ void VerticalLayerDiagnostic::compute_diagnostic_impl() const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); view_1d_const phis; - if (not m_only_compute_dz and not m_from_sea_level) { + if (m_from_sea_level) { phis = get_field_in("phis").get_view(); } - const bool only_compute_dz = m_only_compute_dz; + const bool only_compute_dz = m_diag_name=="dz"; const bool is_interface_layout = m_is_interface_layout; const bool from_sea_level = m_from_sea_level; + const bool do_geopotential = m_geopotential; const int num_levs = m_num_levs; + constexpr auto g = scream::physics::Constants::gravit; // Alias correct view for diagnostic output and for tmp class views view_2d interface_view; @@ -111,27 +115,39 @@ void VerticalLayerDiagnostic::compute_diagnostic_impl() KOKKOS_LAMBDA(const MemberType& team) { const int icol = team.league_rank(); - // Calculate dz. Use the memory in tmp_mid_view for dz and z_mid, - // since we don't set z_mid until after dz is no longer needed. - const auto& dz_s = ekat::subview(midpoint_view, icol); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, npacks), [&] (const Int& jpack) { - dz_s(jpack) = PF::calculate_dz(pseudo_density_mid(icol,jpack), p_mid(icol,jpack), T_mid(icol,jpack), qv_mid(icol,jpack)); - }); + // Whatever the output needs, the first thing to compute is dz. + const auto& dz = ekat::subview(midpoint_view, icol); + PF::calculate_dz(team,ekat::subview(pseudo_density_mid,icol), + ekat::subview(p_mid,icol), + ekat::subview(T_mid,icol), + ekat::subview(qv_mid,icol), + dz); team.team_barrier(); - if (not only_compute_dz) { - // Calculate z_int if this diagnostic is not dz - const auto& z_int_s = ekat::subview(interface_view, icol); - const Real surf_geopotential = from_sea_level ? 0.0 : phis(icol); - PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,z_int_s); - - if (not is_interface_layout) { - // Calculate z_mid if this diagnostic is not dz or an interface value - const auto& z_mid_s = ekat::subview(midpoint_view, icol); - PF::calculate_z_mid(team,num_levs,z_int_s,z_mid_s); - } + // If dz is all we need, we're done + if (only_compute_dz) { return; } + + // Now integrate to compute quantity at interfaces + const auto& v_int = ekat::subview(interface_view, icol); + + // phi and z are related by phi=z*g, so dphi=dz*g, and z_surf = phis/g + if (do_geopotential) { + auto dphi = [&](const int ilev) { + return dz(ilev) * g; + }; + column_ops::template column_scan(team,num_levs,dphi,v_int,phis(icol)); + } else { + const Real surf_val = from_sea_level ? phis(icol)/g : 0; + column_ops::template column_scan(team,num_levs,dz,v_int,surf_val); + } + + // If we need quantity at midpoints, simply do int->mid averaging + if (not is_interface_layout) { + team.team_barrier(); + const auto& v_mid = ekat::subview(midpoint_view, icol); + column_ops::compute_midpoint_values(team,num_levs,v_int,v_mid); } }); } -// ========================================================================================= + } //namespace scream diff --git a/components/eamxx/src/diagnostics/vertical_layer.hpp b/components/eamxx/src/diagnostics/vertical_layer.hpp index a7a6bbf68f5a..0a511303d525 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.hpp +++ b/components/eamxx/src/diagnostics/vertical_layer.hpp @@ -57,16 +57,15 @@ class VerticalLayerDiagnostic : public AtmosphereDiagnostic // field in the computation is output (dz, z_int, or z_mid). std::string m_diag_name; - // Store if we only need to compute dz to save computation/memory requirements. - bool m_only_compute_dz; - // Store if the diagnostic output field exists on interface values bool m_is_interface_layout; - // If z_int or z_mid is computed, determine whether the BC - // is from sea level or not (from topography data). + // True z_mid/int, false for altitude_mid/int. Unused for others bool m_from_sea_level; + // If true, output is a geopotential (units m2/s2), otherwise an elevation + bool m_geopotential; + }; // class VerticalLayerDiagnostic } //namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 1852785eeacf..be1b3015488f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1315,25 +1315,25 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { if (units.find("_above_") != std::string::npos) { // The field is at a height above a specific reference. // Currently we only support FieldAtHeight above "sealevel" or "surface" - auto subtokens = ekat::split(units,"_above_"); - params.set("surface_reference",subtokens[1]); - units = subtokens[0]; - // Need to reset the vertical location to strip the "_above_" part of the string. - params.set("vertical_location", tokens[1].substr(0,units_start)+subtokens[0]); - // If the slice is "above_sealevel" then we need to track the avg cnt uniquely. - // Note, "above_surface" is expected to never have masking and can thus use - // the typical 2d layout avg cnt. - if (subtokens[1]=="sealevel") { - diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice - // If we have 2D slices we need to be tracking the average count, - // if m_avg_type is not Instant - m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; - } + auto subtokens = ekat::split(units,"_above_"); + params.set("surface_reference",subtokens[1]); + units = subtokens[0]; + // Need to reset the vertical location to strip the "_above_" part of the string. + params.set("vertical_location", tokens[1].substr(0,units_start)+subtokens[0]); + // If the slice is "above_sealevel" then we need to track the avg cnt uniquely. + // Note, "above_surface" is expected to never have masking and can thus use + // the typical 2d layout avg cnt. + if (subtokens[1]=="sealevel") { + diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice + // If we have 2D slices we need to be tracking the average count, + // if m_avg_type is not Instant + m_track_avg_cnt = m_track_avg_cnt || m_avg_type!=OutputAvgType::Instant; + } } if (units=="m") { diag_name = "FieldAtHeight"; - EKAT_REQUIRE_MSG(params.isParameter("surface_reference"),"Error! Output field request for " + diag_field_name + " is missing a surface reference." - " Please add either '_above_sealevel' or '_above_surface' to the field name"); + EKAT_REQUIRE_MSG(params.isParameter("surface_reference"),"Error! Output field request for " + diag_field_name + " is missing a surface reference." + " Please add either '_above_sealevel' or '_above_surface' to the field name"); } else if (units=="mb" or units=="Pa" or units=="hPa") { diag_name = "FieldAtPressureLevel"; diag_avg_cnt_name = "_" + tokens[1]; // Set avg_cnt tracking for this specific slice @@ -1390,9 +1390,10 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { } // These fields are special case of VerticalLayer diagnostic. - // The diagnostics requires the names be given as param value. - if (diag_name == "z_int" or diag_name == "geopotential_int" or - diag_name == "z_mid" or diag_name == "geopotential_mid" or + // The diagnostics requires the name to be given as param value. + if (diag_name == "z_int" or diag_name == "z_mid" or + diag_name == "geopotential_int" or diag_name == "geopotential_mid" or + diag_name == "altitude_int" or diag_name == "altitude_mid" or diag_name == "dz") { params.set("diag_name", diag_name); } @@ -1401,7 +1402,7 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { auto diag = diag_factory.create(diag_name,m_comm,params); diag->set_grids(m_grids_manager); - // Add empty entry for this map, so .at(..) always works + // Ensure there's an entry in the map for this diag, so .at(diag_name) always works auto& deps = m_diag_depends_on_diags[diag->name()]; // Initialize the diagnostic From 08f484f0f7b8d40f3775aaf157fc3612a17d23b4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 20 May 2024 15:24:16 -0600 Subject: [PATCH 200/476] EAMxx: use the LongNames util only if "long_name" is not in the field extra data --- components/eamxx/src/share/io/scorpio_output.cpp | 11 ++++++----- components/eamxx/src/share/io/scream_io_utils.hpp | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index be1b3015488f..b3ed08496df4 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -955,9 +955,6 @@ register_variables(const std::string& filename, auto vec_of_dims = set_vec_of_dims(layout); std::string units = fid.get_units().to_string(); - // Gather longname - auto longname = m_longnames.get_longname(name); - // TODO Need to change dtype to allow for other variables. // Currently the field_manager only stores Real variables so it is not an issue, // but in the future if non-Real variables are added we will want to accomodate that. @@ -991,8 +988,6 @@ register_variables(const std::string& filename, scorpio::define_var (filename, name, units, vec_of_dims, "real",fp_precision, m_add_time_dim); - scorpio::set_attribute(filename, name, "long_name", longname); - // Add FillValue as an attribute of each variable // FillValue is a protected metadata, do not add it if it already existed if (fp_precision=="double" or @@ -1033,6 +1028,12 @@ register_variables(const std::string& filename, for (const auto& [att_name,att_val] : str_atts) { scorpio::set_attribute(filename,name,att_name,att_val); } + + // Gather longname (if not already in the io: string attributes) + if (str_atts.count("long_name")==0) { + auto longname = m_longnames.get_longname(name); + scorpio::set_attribute(filename, name, "long_name", longname); + } } } // Now register the average count variables diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 781b376858a4..efb2a4fd65bb 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -80,9 +80,9 @@ struct LongNames { std::map name_2_longname = { {"lev","hybrid level at midpoints (1000*(A+B))"}, {"hyai","hybrid A coefficient at layer interfaces"}, - {"hybi","hybrid B coefficient at layer interfaces"}, - {"hyam","hybrid A coefficient at layer midpoints"}, - {"hybm","hybrid B coefficient at layer midpoints"} + {"hybi","hybrid B coefficient at layer interfaces"}, + {"hyam","hybrid A coefficient at layer midpoints"}, + {"hybm","hybrid B coefficient at layer midpoints"} }; }; From 0fcadb8d4bd2aa1d1dccc92861c5e021d874ba03 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 20 May 2024 15:25:00 -0600 Subject: [PATCH 201/476] EAMxx: use field extra data to set long_name for VerticalLayer diagnostic --- .../eamxx/src/diagnostics/vertical_layer.cpp | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index 9e15b68aa19a..4dae513893b9 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -35,30 +35,49 @@ set_grids(const std::shared_ptr grids_manager) m_num_cols = grid->get_num_local_dofs(); // Number of columns on this rank m_num_levs = grid->get_num_vertical_levels(); // Number of levels per column - FieldLayout scalar2d_layout { {COL }, {m_num_cols } }; - FieldLayout scalar3d_layout_mid { {COL,LEV }, {m_num_cols,m_num_levs } }; - FieldLayout scalar3d_layout_int { {COL,ILEV}, {m_num_cols,m_num_levs+1} }; + const auto scalar2d = grid->get_2d_scalar_layout(); + const auto scalar3d_mid = grid->get_3d_scalar_layout(true); + const auto scalar3d_int = grid->get_3d_scalar_layout(false); constexpr int ps = Pack::n; // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_layout_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_layout_mid, kg/kg, grid_name, ps); + add_field("T_mid", scalar3d_mid, K, grid_name, ps); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); + add_field("p_mid", scalar3d_mid, Pa, grid_name, ps); + add_field("qv", scalar3d_mid, kg/kg, grid_name, ps); // Only need phis if computing geopotential_* if (not m_geopotential) { - add_field("phis", scalar2d_layout, m2/s2, grid_name); + add_field("phis", scalar2d, m2/s2, grid_name); } // Construct and allocate the diagnostic field based on the diagnostic name. - const auto diag_layout = m_is_interface_layout ? scalar3d_layout_int : scalar3d_layout_mid; - FieldIdentifier fid (name(), diag_layout, m, grid_name); + const auto diag_layout = m_is_interface_layout ? scalar3d_int : scalar3d_mid; + FieldIdentifier fid (name(), diag_layout, m_geopotential ? m2/s2 : m, grid_name); m_diagnostic_output = Field(fid); - auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); - C_ap.request_allocation(ps); + auto& fap = m_diagnostic_output.get_header().get_alloc_properties(); + fap.request_allocation(ps); m_diagnostic_output.allocate_view(); + using stratts_t = std::map; + auto& io_atts = m_diagnostic_output.get_header().get_extra_data("io: string attributes"); + auto& long_name = io_atts["long_name"]; + if (m_diag_name=="dz") { + long_name = "level thickness"; + } else if (m_diag_name=="z_mid") { + long_name = "elevation above sealevel at level midpoints"; + } else if (m_diag_name=="z_int") { + long_name = "elevation above sealevel at level interfaces"; + } else if (m_diag_name=="altitude_mid") { + long_name= "elevation above surface at level midpoints"; + } else if (m_diag_name=="altitude_int") { + long_name = "elevation above surface at level interfaces"; + } else if (m_diag_name=="geopotential_mid") { + long_name = "geopotential height relative to sealevel at level midpoints"; + } else { + long_name = "geopotential height relative to sealevel at level interfaces"; + } + // Initialize temporary views based on need. if (m_diag_name!="dz") { if (m_is_interface_layout) { From 0da3b22c6b35b74f3c620152fbe49979d5a6f5e6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 20 May 2024 15:25:12 -0600 Subject: [PATCH 202/476] EAMxx: simplify unit testing for VerticalLayer --- .../tests/vertical_layer_tests.cpp | 192 +++++++----------- 1 file changed, 71 insertions(+), 121 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp index 32402955f8da..fc90c1a56f83 100644 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp @@ -39,10 +39,11 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { } //-----------------------------------------------------------------------------------------------// -template -void run(std::mt19937_64& engine, std::string diag_type, const bool from_sea_level = false) +template +void run( Engine& engine, + std::string diag_name, + const std::string& location) { - using PF = scream::PhysicsFunctions; using PC = scream::physics::Constants; using Pack = ekat::Pack; using KT = ekat::KokkosTypes; @@ -64,12 +65,15 @@ void run(std::mt19937_64& engine, std::string diag_type, const bool from_sea_lev const int ncols = 1; auto gm = create_gm(comm,ncols,num_levs); + // A time stamp + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + // Kokkos Policy auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); // Input (randomized) views view_1d temperature("temperature",num_mid_packs), - pseudodensity("pseudodensity",num_mid_packs), + pseudodensity("pseudo_density",num_mid_packs), pressure("pressure",num_mid_packs), watervapor("watervapor",num_mid_packs); @@ -77,38 +81,21 @@ void run(std::mt19937_64& engine, std::string diag_type, const bool from_sea_lev return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); }; - // Construct random input data - using RPDF = std::uniform_real_distribution; - RPDF pdf_qv(1e-6,1e-3), - pdf_pseudodens(1.0,100.0), - pdf_pres(0.0,PC::P0), - pdf_temp(200.0,400.0), - pdf_phis(0.0,10000.0); - - // A time stamp - util::TimeStamp t0 ({2022,1,1},{0,0,0}); - + register_diagnostics(); + auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + // Construct the Diagnostic ekat::ParameterList params; - std::string diag_name; - if (diag_type == "thickness") { - diag_name = "dz"; - } - else if (diag_type == "interface") { - diag_name = from_sea_level ? "z_int" : "geopotential_int"; - } else if (diag_type == "midpoint") { - diag_name = from_sea_level ? "z_mid" : "geopotential_mid"; + if (location=="midpoints") { + diag_name += "_mid"; + } else if (location=="interfaces") { + diag_name += "_int"; } params.set("diag_name", diag_name); - register_diagnostics(); - auto& diag_factory = AtmosphereDiagnosticFactory::instance(); auto diag = diag_factory.create(diag_name,comm,params); diag->set_grids(gm); - // Helpful bools - const bool only_compute_dz = (diag_type == "thickness"); - const bool is_interface_layout = (diag_type == "interface"); - const bool generate_phis_data = (not only_compute_dz and not from_sea_level); + const bool needs_phis = diag_name=="z" or diag_name=="geopotential"; // Set the required fields for the diagnostic. std::map input_fields; @@ -124,85 +111,51 @@ void run(std::mt19937_64& engine, std::string diag_type, const bool from_sea_lev input_fields.emplace(name,f); } - // Initialize the diagnostic - diag->initialize(t0,RunType::Initial); - - // Run tests - { - // Construct random data to use for test - // Get views of input data and set to random values - const auto& T_mid_f = input_fields["T_mid"]; - const auto& T_mid_v = T_mid_f.get_view(); - const auto& pseudo_dens_f = input_fields["pseudo_density"]; - const auto& pseudo_dens_v = pseudo_dens_f.get_view(); - const auto& p_mid_f = input_fields["p_mid"]; - const auto& p_mid_v = p_mid_f.get_view(); - const auto& qv_mid_f = input_fields["qv"]; - const auto& qv_mid_v = qv_mid_f.get_view(); - Field phis_f; - rview_1d phis_v; - if (generate_phis_data) { - phis_f = input_fields["phis"]; - phis_v = phis_f.get_view(); - } - - for (int icol=0;icolcompute_diagnostic(); - const auto& diag_out = diag->get_diagnostic(); - - // Need to generate temporary values for calculation - const auto& dz_v = view_2d("",ncols, num_mid_packs); - const auto& zmid_v = view_2d("",ncols, num_mid_packs); - const auto& zint_v = view_2d("",ncols, num_mid_packs_p1); - - Kokkos::parallel_for("", policy, KOKKOS_LAMBDA(const MemberType& team) { - const int icol = team.league_rank(); - - const auto& dz_s = ekat::subview(dz_v,icol); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { - dz_s(jpack) = PF::calculate_dz(pseudo_dens_v(icol,jpack),p_mid_v(icol,jpack),T_mid_v(icol,jpack),qv_mid_v(icol,jpack)); - }); - team.team_barrier(); - - if (not only_compute_dz) { - const auto& zint_s = ekat::subview(zint_v,icol); - const Real surf_geopotential = from_sea_level ? 0.0 : phis_v(icol); - PF::calculate_z_int(team,num_levs,dz_s,surf_geopotential,zint_s); - - if (not is_interface_layout) { - const auto& zmid_s = ekat::subview(zmid_v,icol); - PF::calculate_z_mid(team,num_levs,zint_s,zmid_s); - } + // Initialize and run the diagnostic + diag->initialize(t0,RunType::Initial); + diag->compute_diagnostic(); + const auto& diag_out = diag->get_diagnostic(); + diag_out.sync_to_host(); + auto d_h = diag_out.get_view(); + + // Compare against expecte value + const auto last_lev = location=="interfaces" ? num_levs : num_levs-1; + for (int icol=0; icol=0; --ilev) { + int num_mid_levs_below = ilev-last_lev; + Real tgt_mid, tgt_int; + if (diag_name=="dz") { + tgt_mid = dz; + } else if (diag_name=="altitude") { + tgt_int = phis/PC::gravit + num_mid_levs_below*dz; + tgt_mid = tgt_int + dz/2; + } else if (diag_name=="geopotential") { + tgt_int = phis + num_mid_levs_below*dz*PC::gravit; + tgt_mid = tgt_int + PC::gravit*dz/2; + } else { + tgt_int = num_mid_levs_below*dz; + tgt_mid = tgt_int + dz/2; } - }); - Kokkos::fence(); - - Field diag_calc = diag_out.clone(); - auto field_v = diag_calc.get_view(); - if (diag_type == "thickness") Kokkos::deep_copy(field_v, dz_v); - else if (diag_type == "interface") Kokkos::deep_copy(field_v, zint_v); - else if (diag_type == "midpoint") Kokkos::deep_copy(field_v, zmid_v); - diag_calc.sync_to_host(); - REQUIRE(views_are_equal(diag_out,diag_calc)); + if (location=="interfaces") { + REQUIRE (d_h(icol,ilev)==tgt_int); + } else { + REQUIRE (d_h(icol,ilev)==tgt_mid); + } + } } // Finalize the diagnostic @@ -219,25 +172,22 @@ TEST_CASE("vertical_layer_test", "vertical_layer_test]"){ auto engine = scream::setup_random_test(); - printf(" -> Number of randomized runs: %d, Pack scalar type\n\n", num_runs, SCREAM_PACK_SIZE); + printf("Test specs\n"); + printf(" - number of randomized runs: %d\n",num_runs); + printf(" - scalar type: Pack\n\n", SCREAM_PACK_SIZE); - printf(" -> Testing dz..."); for (int irun=0; irun(engine, "thickness"); - } - printf("ok!\n"); - printf(" -> Testing z_int/geopotential_int..."); - for (int irun=0; irun(engine, "interface", irun%2==0); // alternate from_sea_level=true/false - } - printf("ok!\n"); - printf(" -> Testing z_mid/geopotential_mid..."); - for (int irun=0; irun(engine, "midpoint", irun%2==0); // alternate from_sea_level=true/false + for (std::string loc : {"midpoints","interfaces"}) { + for (std::string diag : {"geopotential","altitude","z"}) { + printf(" -> Testing diag=%s at %s ...\n",diag.c_str(),loc.c_str()); + run(engine, loc, diag); + printf(" -> Testing diag=%s at %s ... PASS!\n",diag.c_str(),loc.c_str()); + } + } + printf(" -> Testing diag=dz ...\n"); + run(engine, "", "dz"); + printf(" -> Testing diag=dz ... PASS!\n"); } - printf("ok!\n"); - - printf("\n"); } // TEST_CASE From 2f4be63d7ede318fae8b458902d1b934bd7333ac Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 20 May 2024 17:25:57 -0600 Subject: [PATCH 203/476] EAMxx: cleanup the VerticalLayer diagnostic * Rename altitude with height * Clean up includes * Clean up unit tests --- .../src/diagnostics/register_diagnostics.hpp | 4 +- .../tests/vertical_layer_tests.cpp | 176 +++++++++--------- .../eamxx/src/diagnostics/vertical_layer.cpp | 176 ++++++++++++------ .../eamxx/src/diagnostics/vertical_layer.hpp | 23 +-- .../eamxx/src/share/io/scorpio_output.cpp | 2 +- 5 files changed, 215 insertions(+), 166 deletions(-) diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index efb55980a2fb..21d82b045dab 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -36,9 +36,11 @@ inline void register_diagnostics () { diag_factory.register_product("Exner",&create_atmosphere_diagnostic); diag_factory.register_product("VirtualTemperature",&create_atmosphere_diagnostic); diag_factory.register_product("z_int",&create_atmosphere_diagnostic); - diag_factory.register_product("geopotential_int",&create_atmosphere_diagnostic); diag_factory.register_product("z_mid",&create_atmosphere_diagnostic); + diag_factory.register_product("geopotential_int",&create_atmosphere_diagnostic); diag_factory.register_product("geopotential_mid",&create_atmosphere_diagnostic); + diag_factory.register_product("height_int",&create_atmosphere_diagnostic); + diag_factory.register_product("height_mid",&create_atmosphere_diagnostic); diag_factory.register_product("dz",&create_atmosphere_diagnostic); diag_factory.register_product("DryStaticEnergy",&create_atmosphere_diagnostic); diag_factory.register_product("SeaLevelPressure",&create_atmosphere_diagnostic); diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp index fc90c1a56f83..a3a8f2a93ce8 100644 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp @@ -1,20 +1,8 @@ #include "catch2/catch.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "diagnostics/vertical_layer.hpp" #include "diagnostics/register_diagnostics.hpp" - #include "physics/share/physics_constants.hpp" - -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "share/field/field_utils.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_test_utils.hpp" - -#include +#include "share/grid/mesh_free_grids_manager.hpp" namespace scream { @@ -39,24 +27,13 @@ create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { } //-----------------------------------------------------------------------------------------------// -template -void run( Engine& engine, - std::string diag_name, - const std::string& location) +template +void run (const std::string& diag_name, const std::string& location) { - using PC = scream::physics::Constants; - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; - using ExecSpace = typename KT::ExeSpace; - using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using rview_1d = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; - - const int packsize = SCREAM_PACK_SIZE; + using PC = scream::physics::Constants; + + const int packsize = N; constexpr int num_levs = packsize*2 + 1; // Number of levels to use for tests, make sure the last pack can also have some empty slots (packsize>1). - const int num_mid_packs = ekat::npack(num_levs); - const int num_mid_packs_p1 = ekat::npack(num_levs+1); // A world comm ekat::Comm comm(MPI_COMM_WORLD); @@ -68,31 +45,19 @@ void run( Engine& engine, // A time stamp util::TimeStamp t0 ({2022,1,1},{0,0,0}); - // Kokkos Policy - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncols, num_mid_packs); - - // Input (randomized) views - view_1d temperature("temperature",num_mid_packs), - pseudodensity("pseudo_density",num_mid_packs), - pressure("pressure",num_mid_packs), - watervapor("watervapor",num_mid_packs); - - auto dview_as_real = [&] (const view_1d& v) -> rview_1d { - return rview_1d(reinterpret_cast(v.data()),v.size()*packsize); - }; - register_diagnostics(); auto& diag_factory = AtmosphereDiagnosticFactory::instance(); // Construct the Diagnostic ekat::ParameterList params; + std::string name = diag_name; if (location=="midpoints") { - diag_name += "_mid"; + name += "_mid"; } else if (location=="interfaces") { - diag_name += "_int"; + name += "_int"; } - params.set("diag_name", diag_name); - auto diag = diag_factory.create(diag_name,comm,params); + params.set("diag_name", name); + auto diag = diag_factory.create(name,comm,params); diag->set_grids(gm); const bool needs_phis = diag_name=="z" or diag_name=="geopotential"; @@ -107,21 +72,31 @@ void run( Engine& engine, const auto name = f.name(); f.get_header().get_tracking().update_time_stamp(t0); diag->set_required_field(f.get_const()); - REQUIRE_THROWS(diag->set_computed_field(f)); input_fields.emplace(name,f); } + // Can't set computed fields in the diag + REQUIRE_THROWS(diag->set_computed_field(input_fields.begin()->second)); + // Note: we are not testing the calculate_dz utility. We are testing // the diag class, so use some inputs that make checking results easier // With these inputs, T_virt=T, and dz=8*rd/g - const Real dz = PC::RD/PC::gravit; - const Real phis = 3; - input_fields["T_mid"].deep_copy(Real(4)); - input_fields["p_mid"].deep_copy(Real(2)); - input_fields["pseudo_density"].deep_copy(Real(4)); - input_fields["qv"].deep_copy(Real(0)); + const Real g = PC::gravit; + const Real rho_val = 4; + const Real qv_val = 0; + const Real p_val = 2; + const Real T_val = 4; + const Real c1 = -PC::ONE + PC::ONE / PC::ep_2; + const Real Tvirt_val = T_val*(PC::ONE + c1*qv_val); + const Real dz_val = (PC::RD/g) * rho_val*Tvirt_val / p_val; + const Real phis_val = 3; + + input_fields["T_mid"].deep_copy(T_val); + input_fields["p_mid"].deep_copy(p_val); + input_fields["pseudo_density"].deep_copy(rho_val); + input_fields["qv"].deep_copy(qv_val); if (needs_phis) { - input_fields["phis"].deep_copy(phis); + input_fields["phis"].deep_copy(phis_val); } // Initialize and run the diagnostic @@ -132,28 +107,43 @@ void run( Engine& engine, auto d_h = diag_out.get_view(); // Compare against expecte value - const auto last_lev = location=="interfaces" ? num_levs : num_levs-1; + const auto last_int = num_levs; + const auto last_mid = last_int-1; + + // Precompute surface value and increment depending on the diag type + Real delta, surf_val; + if (diag_name=="altitude") { + surf_val = 0; + delta = dz_val; + } else if (diag_name=="z") { + surf_val = phis_val/g; + delta = dz_val; + } else { + surf_val = phis_val; + delta = dz_val*g; + } + for (int icol=0; icol=0; --ilev) { - int num_mid_levs_below = ilev-last_lev; - Real tgt_mid, tgt_int; - if (diag_name=="dz") { - tgt_mid = dz; - } else if (diag_name=="altitude") { - tgt_int = phis/PC::gravit + num_mid_levs_below*dz; - tgt_mid = tgt_int + dz/2; - } else if (diag_name=="geopotential") { - tgt_int = phis + num_mid_levs_below*dz*PC::gravit; - tgt_mid = tgt_int + PC::gravit*dz/2; - } else { - tgt_int = num_mid_levs_below*dz; - tgt_mid = tgt_int + dz/2; - } + Real prev_int_val = surf_val; - if (location=="interfaces") { - REQUIRE (d_h(icol,ilev)==tgt_int); + if (location=="interfaces") { + // Check surface value + REQUIRE (d_h(icol,num_levs)==prev_int_val); + } + + for (int ilev=last_mid; ilev>=0; --ilev) { + if (diag_name=="dz") { + REQUIRE (d_h(icol,ilev)==dz_val); } else { - REQUIRE (d_h(icol,ilev)==tgt_mid); + // If interface, check value, otherwise perform int->mid averaging and check value + auto int_val = prev_int_val + delta; + if (location=="interfaces") { + REQUIRE_THAT(d_h(icol,ilev), Catch::Matchers::WithinRel(int_val,1e-5)); + } else { + auto mid_val = (int_val + prev_int_val) / 2; + REQUIRE_THAT(d_h(icol,ilev), Catch::Matchers::WithinRel(mid_val,1e-5)); + } + prev_int_val = int_val; } } } @@ -168,26 +158,36 @@ TEST_CASE("vertical_layer_test", "vertical_layer_test]"){ using scream::Real; using Device = scream::DefaultDevice; - constexpr int num_runs = 5; - - auto engine = scream::setup_random_test(); - - printf("Test specs\n"); - printf(" - number of randomized runs: %d\n",num_runs); - printf(" - scalar type: Pack\n\n", SCREAM_PACK_SIZE); - - for (int irun=0; irun Testing diagnostic for pack_size=" + std::to_string(N) + "\n"); for (std::string loc : {"midpoints","interfaces"}) { for (std::string diag : {"geopotential","altitude","z"}) { - printf(" -> Testing diag=%s at %s ...\n",diag.c_str(),loc.c_str()); - run(engine, loc, diag); - printf(" -> Testing diag=%s at %s ... PASS!\n",diag.c_str(),loc.c_str()); + std::string msg = " -> Testing diag=" + diag + " at " + loc + " "; + std::string dots (50-msg.size(),'.'); + root_print (msg + dots + "\n"); + run(diag, loc); + root_print (msg + dots + " PASS!\n"); } } - printf(" -> Testing diag=dz ...\n"); - run(engine, "", "dz"); - printf(" -> Testing diag=dz ... PASS!\n"); + std::string msg = " -> Testing diag=dz "; + std::string dots (50-msg.size(),'.'); + root_print (msg + dots + "\n"); + run("dz", "UNUSED"); + root_print (msg + dots + " PASS!\n"); + }; + + if (SCREAM_PACK_SIZE!=1) { + do_run(std::integral_constant()); } + do_run(std::integral_constant()); } // TEST_CASE diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index 4dae513893b9..262aae90e9f1 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -1,5 +1,9 @@ #include "diagnostics/vertical_layer.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/util/scream_common_physics_functions.hpp" +#include "share/util/scream_column_ops.hpp" + namespace scream { @@ -9,11 +13,20 @@ VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& para : AtmosphereDiagnostic(comm,params) { m_diag_name = params.get("diag_name"); - EKAT_REQUIRE_MSG(m_diag_name == "z_int" or m_diag_name == "z_mid" or - m_diag_name == "geopotential_int" or m_diag_name == "geopotential_mid" or - m_diag_name == "altitude_int" or m_diag_name == "altitude_mid" or - m_diag_name == "dz", - "Error! VerticalLayerDiagnostic has been given an unknown name: "+m_diag_name+".\n"); + std::vector supported = { + "z_int", + "z_mid", + "geopotential_int", + "geopotential_mid", + "height_int", + "height_mid", + "dz" + }; + + EKAT_REQUIRE_MSG(ekat::contains(supported,m_diag_name), + "[VerticalLayerDiagnostic] Error! Invalid diag_name.\n" + " - diag_name : " + m_diag_name + "\n" + " - valid names: " + ekat::join(supported,", ") + "\n"); m_is_interface_layout = m_diag_name.find("_int") != std::string::npos; @@ -38,25 +51,52 @@ set_grids(const std::shared_ptr grids_manager) const auto scalar2d = grid->get_2d_scalar_layout(); const auto scalar3d_mid = grid->get_3d_scalar_layout(true); const auto scalar3d_int = grid->get_3d_scalar_layout(false); - constexpr int ps = Pack::n; // The fields required for this diagnostic to be computed - add_field("T_mid", scalar3d_mid, K, grid_name, ps); - add_field("pseudo_density", scalar3d_mid, Pa, grid_name, ps); - add_field("p_mid", scalar3d_mid, Pa, grid_name, ps); - add_field("qv", scalar3d_mid, kg/kg, grid_name, ps); + add_field("T_mid", scalar3d_mid, K, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + add_field("p_mid", scalar3d_mid, Pa, grid_name); + add_field("qv", scalar3d_mid, kg/kg, grid_name); // Only need phis if computing geopotential_* - if (not m_geopotential) { + if (m_from_sea_level) { add_field("phis", scalar2d, m2/s2, grid_name); } +} + +void VerticalLayerDiagnostic:: +initialize_impl (const RunType /*run_type*/) +{ + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + auto m2 = pow(m,2); + auto s2 = pow(s,2); - // Construct and allocate the diagnostic field based on the diagnostic name. - const auto diag_layout = m_is_interface_layout ? scalar3d_int : scalar3d_mid; + const auto& T = get_field_in("T_mid"); + const auto& rho = get_field_in("pseudo_density"); + const auto& p = get_field_in("p_mid"); + const auto& qv = get_field_in("qv"); + const auto& phis = m_from_sea_level ? get_field_in("phis") : T; // unused if m_from_sea_level=false + + // Construct and allocate the diagnostic field. + // Notes: + // - consider diag name to set long name + // - check input fields alloc props to set alloc props for output + + const auto& grid_name = T.get_header().get_identifier().get_grid_name(); + const auto VLEV = m_is_interface_layout ? ILEV : LEV; + const auto nlevs = m_is_interface_layout ? m_num_levs+1 : m_num_levs; + FieldLayout diag_layout ({COL,VLEV},{m_num_cols,nlevs}); FieldIdentifier fid (name(), diag_layout, m_geopotential ? m2/s2 : m, grid_name); + m_diagnostic_output = Field(fid); - auto& fap = m_diagnostic_output.get_header().get_alloc_properties(); - fap.request_allocation(ps); + auto& diag_fap = m_diagnostic_output.get_header().get_alloc_properties(); + for (const auto& f : {T,rho,p,qv,phis}) { + const auto& fap = f.get_header().get_alloc_properties(); + const auto& ps = fap.get_largest_pack_size(); + diag_fap.request_allocation(ps); + } m_diagnostic_output.allocate_view(); using stratts_t = std::map; @@ -68,9 +108,9 @@ set_grids(const std::shared_ptr grids_manager) long_name = "elevation above sealevel at level midpoints"; } else if (m_diag_name=="z_int") { long_name = "elevation above sealevel at level interfaces"; - } else if (m_diag_name=="altitude_mid") { + } else if (m_diag_name=="height_mid") { long_name= "elevation above surface at level midpoints"; - } else if (m_diag_name=="altitude_int") { + } else if (m_diag_name=="height_int") { long_name = "elevation above surface at level interfaces"; } else if (m_diag_name=="geopotential_mid") { long_name = "geopotential height relative to sealevel at level midpoints"; @@ -78,68 +118,81 @@ set_grids(const std::shared_ptr grids_manager) long_name = "geopotential height relative to sealevel at level interfaces"; } - // Initialize temporary views based on need. - if (m_diag_name!="dz") { - if (m_is_interface_layout) { - const auto npacks = ekat::npack(m_num_levs); - m_tmp_midpoint_view = view_2d("tmp_mid",m_num_cols,npacks); - } else { - const auto npacks_p1 = ekat::npack(m_num_levs+1); - m_tmp_interface_view = view_2d("tmp_int",m_num_cols,npacks_p1); - } + // Initialize temporary views based on need. Can alias the diag if a temp is not needed + auto create_temp = [&](const std::string& name, int levs) { + auto u = Units::nondimensional(); + auto ps = diag_fap.get_largest_pack_size(); + FieldLayout fl({COL,LEV},{m_num_cols,levs}); + FieldIdentifier fid (name,fl,u,grid_name); + Field f = Field(fid); + f.get_header().get_alloc_properties().request_allocation(ps); + f.allocate_view(); + return f; + }; + if (m_diag_name == "dz") { + m_tmp_midpoint = m_diagnostic_output; + m_tmp_interface = m_diagnostic_output; // Not really used + } else if (m_is_interface_layout) { + m_tmp_midpoint = create_temp("tmp_mid",m_num_levs); + m_tmp_interface = m_diagnostic_output; + } else { + m_tmp_interface = create_temp("tmp_int",m_num_levs+1); + m_tmp_midpoint = m_diagnostic_output; } } // ========================================================================================= void VerticalLayerDiagnostic::compute_diagnostic_impl() +{ + const auto& fap = m_diagnostic_output.get_header().get_alloc_properties(); + if (fap.get_largest_pack_size()==SCREAM_PACK_SIZE) { + do_compute_diagnostic_impl(); + } else { + do_compute_diagnostic_impl<1>(); + } +} + +template +void VerticalLayerDiagnostic::do_compute_diagnostic_impl() { using column_ops = ColumnOps; + using PackT = ekat::Pack; + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + using PF = PhysicsFunctions; + // To use in column_ops, since we integrate from surface constexpr bool FromTop = false; - const auto npacks = ekat::npack(m_num_levs); - const auto default_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); - - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv_mid = get_field_in("qv").get_view(); - const auto& pseudo_density_mid = get_field_in("pseudo_density").get_view(); + const auto npacks = ekat::npack(m_num_levs); + const auto policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(m_num_cols, npacks); - view_1d_const phis; - if (m_from_sea_level) { - phis = get_field_in("phis").get_view(); - } + const auto& T = get_field_in("T_mid").get_view(); + const auto& p = get_field_in("p_mid").get_view(); + const auto& qv = get_field_in("qv").get_view(); + const auto& rho = get_field_in("pseudo_density").get_view(); + const auto phis = m_from_sea_level ? get_field_in("phis").get_view() : typename KT::view_1d(); const bool only_compute_dz = m_diag_name=="dz"; const bool is_interface_layout = m_is_interface_layout; const bool from_sea_level = m_from_sea_level; - const bool do_geopotential = m_geopotential; + const bool geopotential = m_geopotential; const int num_levs = m_num_levs; constexpr auto g = scream::physics::Constants::gravit; // Alias correct view for diagnostic output and for tmp class views - view_2d interface_view; - view_2d midpoint_view; - if (only_compute_dz) { - midpoint_view = m_diagnostic_output.get_view(); - } else if (is_interface_layout) { - interface_view = m_diagnostic_output.get_view(); - midpoint_view = m_tmp_midpoint_view; - } else { - midpoint_view = m_diagnostic_output.get_view(); - interface_view = m_tmp_interface_view; - } + auto tmp_mid = m_tmp_midpoint.get_view(); + auto tmp_int = m_tmp_interface.get_view(); - Kokkos::parallel_for("VerticalLayerDiagnostic", - default_policy, - KOKKOS_LAMBDA(const MemberType& team) { + // Define the lambda, then dispatch the ||for + auto lambda = KOKKOS_LAMBDA(const MemberType& team) { const int icol = team.league_rank(); // Whatever the output needs, the first thing to compute is dz. - const auto& dz = ekat::subview(midpoint_view, icol); - PF::calculate_dz(team,ekat::subview(pseudo_density_mid,icol), - ekat::subview(p_mid,icol), - ekat::subview(T_mid,icol), - ekat::subview(qv_mid,icol), + const auto& dz = ekat::subview(tmp_mid, icol); + PF::calculate_dz(team,ekat::subview(rho,icol), + ekat::subview(p,icol), + ekat::subview(T,icol), + ekat::subview(qv,icol), dz); team.team_barrier(); @@ -147,10 +200,10 @@ void VerticalLayerDiagnostic::compute_diagnostic_impl() if (only_compute_dz) { return; } // Now integrate to compute quantity at interfaces - const auto& v_int = ekat::subview(interface_view, icol); + const auto& v_int = ekat::subview(tmp_int, icol); // phi and z are related by phi=z*g, so dphi=dz*g, and z_surf = phis/g - if (do_geopotential) { + if (geopotential) { auto dphi = [&](const int ilev) { return dz(ilev) * g; }; @@ -163,10 +216,11 @@ void VerticalLayerDiagnostic::compute_diagnostic_impl() // If we need quantity at midpoints, simply do int->mid averaging if (not is_interface_layout) { team.team_barrier(); - const auto& v_mid = ekat::subview(midpoint_view, icol); + const auto& v_mid = ekat::subview(tmp_mid, icol); column_ops::compute_midpoint_values(team,num_levs,v_int,v_mid); } - }); + }; + Kokkos::parallel_for(m_diag_name, policy, lambda); } } //namespace scream diff --git a/components/eamxx/src/diagnostics/vertical_layer.hpp b/components/eamxx/src/diagnostics/vertical_layer.hpp index 0a511303d525..53a482ce060c 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.hpp +++ b/components/eamxx/src/diagnostics/vertical_layer.hpp @@ -2,8 +2,6 @@ #define EAMXX_VERTICAL_LAY_MID_DIAGNOSTIC_HPP #include "share/atm_process/atmosphere_diagnostic.hpp" -#include "share/util/scream_common_physics_functions.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" namespace scream { @@ -22,13 +20,6 @@ namespace scream class VerticalLayerDiagnostic : public AtmosphereDiagnostic { public: - using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using MemberType = typename KT::MemberType; - using view_1d_const = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; - // Constructors VerticalLayerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -39,19 +30,22 @@ class VerticalLayerDiagnostic : public AtmosphereDiagnostic void set_grids (const std::shared_ptr grids_manager); protected: + void compute_diagnostic_impl (); + void initialize_impl (const RunType /* run_type */); #ifdef KOKKOS_ENABLE_CUDA public: #endif - void compute_diagnostic_impl (); + template + void do_compute_diagnostic_impl (); protected: // Keep track of field dimensions Int m_num_cols; Int m_num_levs; - // Temporary view to set dz, z_mid, and z_int - view_2d m_tmp_interface_view; - view_2d m_tmp_midpoint_view; + // Temporaries to use for calculation of dz, z_int, and z_mid + Field m_tmp_interface; + Field m_tmp_midpoint; // The diagnostic name. This will dictate which // field in the computation is output (dz, z_int, or z_mid). @@ -65,8 +59,7 @@ class VerticalLayerDiagnostic : public AtmosphereDiagnostic // If true, output is a geopotential (units m2/s2), otherwise an elevation bool m_geopotential; - -}; // class VerticalLayerDiagnostic +}; } //namespace scream diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index b3ed08496df4..e79d75e97b13 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1394,7 +1394,7 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { // The diagnostics requires the name to be given as param value. if (diag_name == "z_int" or diag_name == "z_mid" or diag_name == "geopotential_int" or diag_name == "geopotential_mid" or - diag_name == "altitude_int" or diag_name == "altitude_mid" or + diag_name == "height_int" or diag_name == "height_mid" or diag_name == "dz") { params.set("diag_name", diag_name); } From 4b1212755eeaae6b2b80403d0418f816c39d5222 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 20 May 2024 17:32:26 -0600 Subject: [PATCH 204/476] EAMxx: fix pack size determination for VerticalLayer diag --- components/eamxx/src/diagnostics/vertical_layer.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/diagnostics/vertical_layer.cpp b/components/eamxx/src/diagnostics/vertical_layer.cpp index 262aae90e9f1..32d870da03cc 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.cpp +++ b/components/eamxx/src/diagnostics/vertical_layer.cpp @@ -92,11 +92,15 @@ initialize_impl (const RunType /*run_type*/) m_diagnostic_output = Field(fid); auto& diag_fap = m_diagnostic_output.get_header().get_alloc_properties(); + int ps = SCREAM_PACK_SIZE; for (const auto& f : {T,rho,p,qv,phis}) { const auto& fap = f.get_header().get_alloc_properties(); - const auto& ps = fap.get_largest_pack_size(); - diag_fap.request_allocation(ps); + const auto& f_ps = fap.get_largest_pack_size(); + + // We must use a pack size that works with all inputs, so pick the smallest + ps = std::min(ps,f_ps); } + diag_fap.request_allocation(ps); m_diagnostic_output.allocate_view(); using stratts_t = std::map; @@ -121,7 +125,6 @@ initialize_impl (const RunType /*run_type*/) // Initialize temporary views based on need. Can alias the diag if a temp is not needed auto create_temp = [&](const std::string& name, int levs) { auto u = Units::nondimensional(); - auto ps = diag_fap.get_largest_pack_size(); FieldLayout fl({COL,LEV},{m_num_cols,levs}); FieldIdentifier fid (name,fl,u,grid_name); Field f = Field(fid); From c04f93cba854212ac3de957d7ddfda1136cbf3e9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 21 May 2024 12:03:09 -0600 Subject: [PATCH 205/476] EAMxx: fix nasty bug in scorpio interface to get attribute When the attribute type does not match the type of the variable passed in to store the result, we need to use a temporary, since we cannot pass T* when scorpio expects a different pointer. --- .../src/share/io/scream_scorpio_interface.cpp | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 0d4918ad230b..f354b44efc1d 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -1378,12 +1378,50 @@ T get_attribute (const std::string& filename, if (varname=="GLOBAL") { varid = PIO_GLOBAL; } else { - varid = impl::get_var(filename,varname,"scorpio::set_any_attribute").ncid; + varid = impl::get_var(filename,varname,"scorpio::get_attribute").ncid; } + // If the attribute type does not match T, we need a temporary, since we can't pass T* where pio expects + // a different type of pointer + int att_type, err; + err = PIOc_inq_atttype(pf.file->ncid,varid,attname.c_str(),&att_type); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","inq_atttype"); + T val; - int err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&val)); - check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); + if (att_type!=nctype(get_dtype())) { + + if (att_type==PIO_INT) { + int tmp; + err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&tmp)); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","get_att"); + val = tmp; + } else if (att_type==PIO_INT64) { + std::int64_t tmp; + err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&tmp)); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","get_att"); + val = tmp; + } else if (att_type==PIO_FLOAT) { + float tmp; + err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&tmp)); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","get_att"); + val = tmp; + } else if (att_type==PIO_DOUBLE) { + double tmp; + err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&tmp)); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","get_att"); + val = tmp; + } else { + EKAT_ERROR_MSG ( + "Unrecognized/unsupported att type\n" + " - filename: " + filename + "\n" + " - varname : " + varname + "\n" + " - attname : " + attname + "\n" + " - attype : " + std::to_string(att_type) + "\n"); + } + } else { + err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),reinterpret_cast(&val)); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","get_att"); + } return val; } @@ -1421,12 +1459,12 @@ std::string get_attribute (const std::string& filename, int err; PIO_Offset len; err = PIOc_inq_attlen(pf.file->ncid,varid,attname.c_str(),&len); - check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","inq_attlen"); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","inq_attlen"); std::string val(len,'\0'); err = PIOc_get_att(pf.file->ncid,varid,attname.c_str(),val.data()); - check_scorpio_noerr(err,filename,"attribute",attname,"set_attribute","put_att"); + check_scorpio_noerr(err,filename,"attribute",attname,"get_attribute","put_att"); return val; } From 4ea459b5f92282c515ae6e1adecd1c73d97165f6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 21 May 2024 13:57:20 -0600 Subject: [PATCH 206/476] Update YAKL --- externals/YAKL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/YAKL b/externals/YAKL index 4109dc02fe1e..e3757faedffd 160000 --- a/externals/YAKL +++ b/externals/YAKL @@ -1 +1 @@ -Subproject commit 4109dc02fe1e951a95f401719587e981ea4f4fe4 +Subproject commit e3757faedffd41c6ae68cf4dbd1324e628a48ddd From 0ec46cf6b17651cd394a706b54a51b9d6526ac04 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 21 May 2024 14:42:41 -0600 Subject: [PATCH 207/476] EAMxx: allow adding geo field to grid objects, even if they are const This can be used by atm procs to add fields corresponding to the values of dimensions peculiar to vars of that process. E.g., rad could add a field storing the values of the swband dim coordinates. However, grid will crap out if geo data for that field already exists. --- components/eamxx/src/share/grid/abstract_grid.cpp | 2 +- components/eamxx/src/share/grid/abstract_grid.hpp | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 0cc83822c23c..7b9bbf42e46d 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -311,7 +311,7 @@ AbstractGrid::delete_geometry_data (const std::string& name) } void -AbstractGrid::set_geometry_data (const Field& f) +AbstractGrid::set_geometry_data (const Field& f) const { EKAT_REQUIRE_MSG (not has_geometry_data(f.name()), "Error! Cannot set geometry data, since it already exists.\n" diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 2098fd49e946..fe392448e281 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -142,7 +142,10 @@ class AbstractGrid : public ekat::enable_shared_from_this } // Sets pre-existing field as geometry data. - void set_geometry_data (const Field& f); + // NOTE: setter is const, since we do allow adding new data even if grid is const + // E.g., this allows atm procs to define coordinate vars for dimensions + // peculiar to that process + void set_geometry_data (const Field& f) const; void delete_geometry_data (const std::string& name); bool has_geometry_data (const std::string& name) const { @@ -244,7 +247,7 @@ class AbstractGrid : public ekat::enable_shared_from_this // The map lid->idx Field m_lid_to_idx; - std::map m_geo_fields; + mutable std::map m_geo_fields; // The MPI comm containing the ranks across which the global mesh is partitioned ekat::Comm m_comm; From c700ee26ced6e23937df82db16c5f90c56546ffe Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 21 May 2024 15:47:51 -0600 Subject: [PATCH 208/476] EAMxx: make RRTMGP store swband/lwband info in the grid This way, it can later be outputed as part of "grid data" during IO --- .../rrtmgp/eamxx_rrtmgp_process_interface.cpp | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index ca06490a3d6f..531e562a8b1c 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -4,6 +4,7 @@ #include "physics/rrtmgp/shr_orb_mod_c2f.hpp" #include "physics/share/scream_trcmix.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" #include "share/property_checks/field_within_interval_check.hpp" #include "share/util/scream_common_physics_functions.hpp" @@ -190,6 +191,45 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ add_field("ice_flux", scalar2d, m/s, grid_name); add_field("heat_flux", scalar2d, W/m2, grid_name); } + + // Load bands bounds from coefficients files and compute the band centerpoint. + // Store both in the grid (if not already present) + const auto cm = centi*m; + for (std::string prefix : {"sw", "lw"} ) { + int nbands = prefix == "sw" ? m_nswbands : m_nlwbands; + + if (not m_grid->has_geometry_data(prefix + "band_bounds")) { + using namespace ShortFieldTagsNames; + + // NOTE: use append, so we get builtin name for (CMP,2) dim, without hard-coding it here + FieldLayout layout({CMP},{nbands},{prefix+"band"}); + layout.append_dim(CMP,2); + Field bounds (FieldIdentifier(prefix + "band_bounds", layout, 1/cm, grid_name)); + bounds.allocate_view(); + + std::string fname = m_params.get("rrtmgp_coefficients_file_" + prefix); + scorpio::register_file(fname,scorpio::FileMode::Read); + scorpio::read_var(fname,"bnd_limits_wavenumber",bounds.get_view().data()); + scorpio::release_file(fname); + + bounds.sync_to_dev(); + m_grid->set_geometry_data(bounds); + } + + // If no bounds were in the grid, the bands centerpoint likely wouldn't either. Still, let's check... + if (not m_grid->has_geometry_data(prefix + "bands")) { + auto bounds = m_grid->get_geometry_data(prefix + "band_bounds"); + auto bounds_h = bounds.get_view(); + + auto bands = bounds.subfield(1,0).clone(prefix + "band"); + auto bands_h = bands.get_view(); + for (int i=0; iset_geometry_data(bands); + } + } } // RRTMGPRadiation::set_grids size_t RRTMGPRadiation::requested_buffer_size_in_bytes() const From f35c9c7cc06a12805e6a4f60096ec47e1bc2ccdc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 21 May 2024 15:48:30 -0600 Subject: [PATCH 209/476] EAMxx: add some support for Tensor0D layouts --- components/eamxx/src/share/field/field_layout.cpp | 2 ++ components/eamxx/src/share/field/field_layout.hpp | 1 + components/eamxx/src/share/grid/abstract_grid.cpp | 12 ++++++++++++ components/eamxx/src/share/grid/abstract_grid.hpp | 3 +++ .../grid/remap/horiz_interp_remapper_base.cpp | 14 ++++++++++++++ 5 files changed, 32 insertions(+) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 12ad26729e63..1893bbada849 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -285,6 +285,8 @@ void FieldLayout::compute_type () { m_type = LayoutType::Vector0D; return; } else if (tags.size()==1 and nvlevs==1) { m_type = LayoutType::Scalar1D; return; + } else if (tags[0]==CMP and tags[1]==CMP) { + m_type = LayoutType::Tensor0D; return; } else if (tags.size()==2 and ncomps==1 and nvlevs==1) { m_type = LayoutType::Vector1D; return; } else { diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index b2c08533f419..7c0eaf9f040f 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -19,6 +19,7 @@ enum class LayoutType { Invalid, Scalar0D, Vector0D, + Tensor0D, Scalar1D, Vector1D, Scalar2D, diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index 7b9bbf42e46d..1f94593c2b6b 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -74,6 +74,17 @@ get_vertical_layout (const bool midpoints) const return FieldLayout({t},{d}).rename_dims(m_special_tag_names); } +FieldLayout AbstractGrid:: +get_vertical_layout (const bool midpoints, + const int vector_dim, + const std::string& vec_dim_name) const +{ + using namespace ShortFieldTagsNames; + auto l = get_vertical_layout(midpoints); + l.append_dim(CMP,vector_dim,vec_dim_name); + return l; +} + FieldLayout AbstractGrid::get_2d_vector_layout (const int vector_dim) const { @@ -180,6 +191,7 @@ is_valid_layout (const FieldLayout& layout) const switch (layout.type()) { case LayoutType::Scalar0D: [[fallthrough]]; case LayoutType::Vector0D: + case LayoutType::Tensor0D: // 0d quantities are always ok return true; case LayoutType::Scalar1D: [[fallthrough]]; diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index fe392448e281..d88e05b29f02 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -72,6 +72,9 @@ class AbstractGrid : public ekat::enable_shared_from_this // E.g., for a scalar 2d field on a SE grid, this will be (nelem,np,np), // for a vector 3d field on a Point grid it will be (ncols,vector_dim,nlevs) FieldLayout get_vertical_layout (const bool midpoints) const; + FieldLayout get_vertical_layout (const bool midpoints, + const int vector_dim, + const std::string& vec_dim_name = e2str(FieldTag::Component)) const; virtual FieldLayout get_2d_scalar_layout () const = 0; virtual FieldLayout get_2d_vector_layout (const int vector_dim, const std::string& vec_dim_name) const = 0; virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_dims, diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index f55e0f56ae8e..65da6f2447ae 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -124,6 +124,20 @@ create_layout (const FieldLayout& fl_in, std::vector tdims_names; std::string vdim_name; switch (fl_in.type()) { + case LayoutType::Scalar0D: [[ fallthrough ]]; + case LayoutType::Vector0D: [[ fallthrough ]]; + case LayoutType::Tensor0D: + // 0d layouts are the same on all grids + fl_out = fl_in; + break; + case LayoutType::Scalar1D: + // 1d layouts require the grid correct number of levs + fl_out = grid->get_vertical_layout(midpoints); + break; + case LayoutType::Vector1D: + vdim_name = fl_in.names()[fl_in.get_vector_component_idx()]; + fl_out = grid->get_vertical_layout(midpoints,fl_in.get_vector_dim(),vdim_name); + break; case LayoutType::Scalar2D: fl_out = grid->get_2d_scalar_layout(); break; From 2eff286687056ce8fcfce26912f9a4ac7b648fa2 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 21 May 2024 16:11:39 -0600 Subject: [PATCH 210/476] EAMxx: fix comment and altitude->height renaming --- components/eamxx/src/diagnostics/field_at_height.cpp | 2 +- .../eamxx/src/diagnostics/tests/vertical_layer_tests.cpp | 6 +++--- components/eamxx/src/diagnostics/vertical_layer.hpp | 5 ++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index cb2346e5595e..f61cd3a76c1f 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -44,7 +44,7 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) " - field name: " + m_field_name + "\n" " - surface reference: " + surf_ref + "\n" " - valid options: sealevel, surface\n"); - m_z_name = (surf_ref == "sealevel") ? "z" : "altitude"; + m_z_name = (surf_ref == "sealevel") ? "z" : "height"; const auto& location = m_params.get("vertical_location"); auto chars_start = location.find_first_not_of("0123456789."); EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, diff --git a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp index a3a8f2a93ce8..fe75114611d5 100644 --- a/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vertical_layer_tests.cpp @@ -106,13 +106,13 @@ void run (const std::string& diag_name, const std::string& location) diag_out.sync_to_host(); auto d_h = diag_out.get_view(); - // Compare against expecte value + // Compare against expected value const auto last_int = num_levs; const auto last_mid = last_int-1; // Precompute surface value and increment depending on the diag type Real delta, surf_val; - if (diag_name=="altitude") { + if (diag_name=="height") { surf_val = 0; delta = dz_val; } else if (diag_name=="z") { @@ -169,7 +169,7 @@ TEST_CASE("vertical_layer_test", "vertical_layer_test]"){ root_print("\n"); root_print(" -> Testing diagnostic for pack_size=" + std::to_string(N) + "\n"); for (std::string loc : {"midpoints","interfaces"}) { - for (std::string diag : {"geopotential","altitude","z"}) { + for (std::string diag : {"geopotential","height","z"}) { std::string msg = " -> Testing diag=" + diag + " at " + loc + " "; std::string dots (50-msg.size(),'.'); root_print (msg + dots + "\n"); diff --git a/components/eamxx/src/diagnostics/vertical_layer.hpp b/components/eamxx/src/diagnostics/vertical_layer.hpp index 53a482ce060c..805fd70028f6 100644 --- a/components/eamxx/src/diagnostics/vertical_layer.hpp +++ b/components/eamxx/src/diagnostics/vertical_layer.hpp @@ -47,14 +47,13 @@ class VerticalLayerDiagnostic : public AtmosphereDiagnostic Field m_tmp_interface; Field m_tmp_midpoint; - // The diagnostic name. This will dictate which - // field in the computation is output (dz, z_int, or z_mid). + // The diagnostic name. std::string m_diag_name; // Store if the diagnostic output field exists on interface values bool m_is_interface_layout; - // True z_mid/int, false for altitude_mid/int. Unused for others + // True z_mid/int and geopotential_mid/int, false for height_mid/int. Unused for dz bool m_from_sea_level; // If true, output is a geopotential (units m2/s2), otherwise an elevation From 200d84e6f98229d07a0f4ea532b4293191378812 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 22 May 2024 08:44:04 -0600 Subject: [PATCH 211/476] EAMxx: fix field_at_height unit test --- .../tests/field_at_height_tests.cpp | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp index 57a057116102..e26d34dd1a55 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp @@ -46,8 +46,8 @@ TEST_CASE("field_at_height") FieldIdentifier z_surf_fid ("z_surf", FieldLayout({COL },{ncols }),m,grid->name()); FieldIdentifier z_mid_fid ("z_mid", FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); FieldIdentifier z_int_fid ("z_int", FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); - FieldIdentifier geo_mid_fid ("geopotential_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); - FieldIdentifier geo_int_fid ("geopotential_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + FieldIdentifier h_mid_fid ("height_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier h_int_fid ("height_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); // Keep track of reference fields for comparison FieldIdentifier s_tgt_fid ("scalar_target",FieldLayout({COL },{ncols }),m,grid->name()); FieldIdentifier v_tgt_fid ("vector_target",FieldLayout({COL,CMP},{ncols,ndims}),m,grid->name()); @@ -59,8 +59,8 @@ TEST_CASE("field_at_height") Field z_surf (z_surf_fid); Field z_mid (z_mid_fid); Field z_int (z_int_fid); - Field geo_mid (geo_mid_fid); - Field geo_int (geo_int_fid); + Field h_mid (h_mid_fid); + Field h_int (h_int_fid); Field s_tgt (s_tgt_fid); Field v_tgt (v_tgt_fid); @@ -71,8 +71,8 @@ TEST_CASE("field_at_height") z_surf.allocate_view(); z_mid.allocate_view(); z_int.allocate_view(); - geo_mid.allocate_view(); - geo_int.allocate_view(); + h_mid.allocate_view(); + h_int.allocate_view(); s_tgt.allocate_view(); v_tgt.allocate_view(); @@ -83,8 +83,8 @@ TEST_CASE("field_at_height") z_surf.get_header().get_tracking().update_time_stamp(t0); z_mid.get_header().get_tracking().update_time_stamp(t0); z_int.get_header().get_tracking().update_time_stamp(t0); - geo_mid.get_header().get_tracking().update_time_stamp(t0); - geo_int.get_header().get_tracking().update_time_stamp(t0); + h_mid.get_header().get_tracking().update_time_stamp(t0); + h_int.get_header().get_tracking().update_time_stamp(t0); s_tgt.get_header().get_tracking().update_time_stamp(t0); v_tgt.get_header().get_tracking().update_time_stamp(t0); @@ -124,9 +124,9 @@ TEST_CASE("field_at_height") // Set up vertical structure for the tests. Note, // z_mid/int represents the height in m above sealevel - // geo_mid/int represente the hegith in m above the surface + // h_mid/int represente the hegith in m above the surface // So we first construct z_mid/int using z_surf as reference, and - // then can build geo_mid/int from z_mid/int + // then can build h_mid/int from z_mid/int // Furthermore, z_mid is just the midpoint between two adjacent z_int // points, so we back z_mid out of z_int. // @@ -137,8 +137,8 @@ TEST_CASE("field_at_height") const auto& zint_v = z_int.get_view(); const auto& zmid_v = z_mid.get_view(); const auto& zsurf_v = z_surf.get_view(); - const auto& geoint_v = geo_int.get_view(); - const auto& geomid_v = geo_mid.get_view(); + const auto& geoint_v = h_int.get_view(); + const auto& geomid_v = h_mid.get_view(); int min_col_thickness = z_top; int max_surf = 0; for (int ii=0; ii Testing throws error with unsupported reference height...\n"); { - REQUIRE_THROWS(run_diag (s_mid,geo_mid,"1m","foobar")); + REQUIRE_THROWS(run_diag (s_mid,h_mid,"1m","foobar")); } print(" -> Testing throws error with unsupported reference height... OK\n"); @@ -182,8 +182,8 @@ TEST_CASE("field_at_height") std::string loc; for (std::string surf_ref : {"sealevel","surface"}) { printf(" -> Testing for a reference height above %s...\n",surf_ref.c_str()); - const auto mid_src = surf_ref == "sealevel" ? z_mid : geo_mid; - const auto int_src = surf_ref == "sealevel" ? z_int : geo_int; + const auto mid_src = surf_ref == "sealevel" ? z_mid : h_mid; + const auto int_src = surf_ref == "sealevel" ? z_int : h_int; const int max_surf_4test = surf_ref == "sealevel" ? max_surf : 0; for (int irun=0; irun Forced extrapolation ...............\n"); + print(" -> Forced extrapolation at top...............\n"); auto slope = pdf_m(engine); auto inter = pdf_y0(engine); f_z_src(inter, slope, int_src, s_int); - print(" -> at top...............\n"); z_tgt = 2*z_top; std::string loc = std::to_string(z_tgt) + "m"; auto dtop = run_diag(s_int,int_src,loc,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(dtop,s_tgt,tol)); - print(" -> at bot...............\n"); + print(" -> Forced extrapolation at top............... OK!\n"); + print(" -> Forced extrapolation at bot...............\n"); z_tgt = 0; loc = std::to_string(z_tgt) + "m"; auto dbot = run_diag(s_int,int_src,loc,surf_ref); f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); REQUIRE (views_are_approx_equal(dbot,s_tgt,tol)); - print(" -> Forced extrapolation............... OK!\n"); + print(" -> Forced extrapolation at bot............... OK!\n"); } printf(" -> Testing for a reference height above %s... OK!\n",surf_ref.c_str()); } From 6bb577cb949df41ff5cd338e83174a2e8c7902dc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 22 May 2024 13:12:45 -0600 Subject: [PATCH 212/476] EAMxx: change output field for p3_standalone Requesting field above sealevel requires phis, which this test doesn't have --- components/eamxx/tests/single-process/p3/output.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/tests/single-process/p3/output.yaml b/components/eamxx/tests/single-process/p3/output.yaml index e52e02c4a19b..251adec1f3cd 100644 --- a/components/eamxx/tests/single-process/p3/output.yaml +++ b/components/eamxx/tests/single-process/p3/output.yaml @@ -4,7 +4,7 @@ filename_prefix: p3_standalone_output Averaging Type: Instant Field Names: - T_mid - - T_mid_at_2m_above_sealevel + - T_mid_at_2m_above_surface - T_prev_micro_step - qv - qc From eb93f26cc46b45642bdac5573908178e243f7f28 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 22 May 2024 15:54:58 -0600 Subject: [PATCH 213/476] EAMxx: fix generate_baseline test for RRTMGP --- .../src/physics/rrtmgp/tests/generate_baseline.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp b/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp index 4e678c7a825c..97b3887acc8f 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/generate_baseline.cpp @@ -1,12 +1,13 @@ #include "physics/rrtmgp/scream_rrtmgp_interface.hpp" -#include "physics/rrtmgp/mo_garand_atmos_io.h" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #include "share/scream_types.hpp" #include "share/scream_session.hpp" -#include "cpp/rrtmgp/mo_gas_concentrations.h" +// From RRTMGP submodule +#include +#include -#include "YAKL.h" +#include #include @@ -70,8 +71,9 @@ int main (int argc, char** argv) { real2d t_lay ("t_lay", ncol, nlay); real2d p_lev ("p_lev", ncol, nlay+1); real2d t_lev ("t_lev", ncol, nlay+1); + real2d col_dry; GasConcs gas_concs; - read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, ncol); + read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize the RRTMGP interface; this will read in the k-distribution // data that contains information about absorption coefficients for gases @@ -203,6 +205,7 @@ int main (int argc, char** argv) { t_lay.deallocate(); p_lev.deallocate(); t_lev.deallocate(); + col_dry.deallocate(); sfc_alb_dir_vis.deallocate(); sfc_alb_dir_nir.deallocate(); sfc_alb_dif_vis.deallocate(); From be93f5d2ec7a46e55ea7d2fa21d59eecdd62e7bd Mon Sep 17 00:00:00 2001 From: "Michael J. Schmidt" Date: Thu, 21 Mar 2024 17:04:14 -0600 Subject: [PATCH 214/476] multi-slice subfield written and compiling --- components/eamxx/src/share/field/field.cpp | 68 ++++++++++++++++++- components/eamxx/src/share/field/field.hpp | 15 +++- .../src/share/field/field_alloc_prop.cpp | 52 ++++++++++++++ .../src/share/field/field_alloc_prop.hpp | 44 ++++++++---- .../eamxx/src/share/field/field_header.cpp | 37 ++++++++-- .../eamxx/src/share/field/field_header.hpp | 11 +++ 6 files changed, 206 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index e022278d5030..11c8e695b4be 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -82,6 +82,7 @@ Field Field:: subfield (const std::string& sf_name, const ekat::units::Units& sf_units, const int idim, const int index, const bool dynamic) const { + // const auto& id = m_header->get_identifier(); const auto& lt = id.get_layout(); @@ -115,6 +116,50 @@ subfield (const int idim, const int index, const bool dynamic) const { return subfield(m_header->get_identifier().name(),idim,index,dynamic); } +// slice at index idim, entries \in [index_beg, index_end] +Field Field::subfield(const std::string& sf_name, + const ekat::units::Units& sf_units, const int idim, + const int index_beg, const int index_end, + const bool dynamic) const { + + const auto& id = m_header->get_identifier(); + const auto& lt = id.get_layout(); + + // Sanity checks + EKAT_REQUIRE_MSG( + is_allocated(), + "Error! Input field must be allocated in order to subview it.\n"); + EKAT_REQUIRE_MSG(idim == 0 || idim == 1, + "Error! Subview dimension index must be either 0 or 1.\n"); + + auto sf_layout = + lt.clone_with_different_extent(idim, index_end - index_beg + 1); + // Create identifier for subfield + FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name()); + + // Create empty subfield, then set header and views + // Note: we can access protected members, since it's the same type + Field sf; + sf.m_header = create_subfield_header(sf_id, m_header, idim, index_beg, + index_end, dynamic); + sf.m_data = m_data; + + return sf; +} + +Field Field::subfield(const std::string& sf_name, const int idim, + const int index_beg, const int index_end, + const bool dynamic) const { + const auto& id = m_header->get_identifier(); + return subfield(sf_name, id.get_units(), idim, index_beg, index_end, dynamic); +} + +Field Field::subfield(const int idim, const int index_beg, const int index_end, + const bool dynamic) const { + return subfield(m_header->get_identifier().name(), idim, index_beg, index_end, + dynamic); +} + Field Field:: get_component (const int i, const bool dynamic) { const auto& layout = get_header().get_identifier().get_layout(); @@ -127,11 +172,32 @@ get_component (const int i, const bool dynamic) { EKAT_REQUIRE_MSG (i>=0 && i= 0 && i2 < layout.dim(idim), + "Error! Component index range out of bounds [0," + + std::to_string(layout.dim(idim)) + ").\n"); + EKAT_REQUIRE_MSG(i1 < i2, "Error! Invalid component indices (i1 >= i2).\n"); + + // Add _$i1-$i2 to the field name, to avoid issues if the subfield is stored + // in some structure that requires unique names (e.g., a remapper) + return subfield(fname + "_" + std::to_string(i1) + "-" + std::to_string(i2), + idim, i1, i2, dynamic); +} + bool Field::equivalent(const Field& rhs) const { return (m_header==rhs.m_header && diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 4a4b20525a3e..c5c7ef6de406 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -241,11 +241,24 @@ class Field { Field subfield (const std::string& sf_name, const int idim, const int index, const bool dynamic = false) const; Field subfield (const int idim, const int k, const bool dynamic = false) const; - + // get subfield to extract multiple slices in a continuous range of indices + // e.g., (in matlab syntax) subf = f.subfield(:, 1:3, :) + // but NOT subf = f.subfield(:, [1, 3, 4], :) + // NOTE: use std::pair(index_beg, index_end) or a kokkos::pair()? + Field subfield (const std::string& sf_name, const ekat::units::Units& sf_units, + const int idim, const int index_beg, const int index_end, + const bool dynamic = false) const; + Field subfield (const std::string& sf_name, const int idim, + const int index_beg, const int index_end, + const bool dynamic = false) const; + Field subfield (const int idim, const int k_beg, const int k_end, + const bool dynamic = false) const; // If this field is a vector field, get a subfield for the ith component. // If dynamic = true, it is possible to "reset" the component index at runtime. // Note: throws if this is not a vector field. Field get_component (const int i, const bool dynamic = false); + // version for slicing across multiple, contiguous indices + Field get_component (const int i1, const int i2, const bool dynamic = false); // Checks whether the underlying view has been already allocated. bool is_allocated () const { return m_data.d_view.data()!=nullptr; } diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index fcffc821bbaa..f2375869f84f 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -77,6 +77,58 @@ subview (const int idim, const int k, const bool dynamic) const { return props; } +FieldAllocProp FieldAllocProp::subview(const int idim, const int k_beg, + const int k_end, + const bool dynamic) const { + EKAT_REQUIRE_MSG( + is_committed(), + "Error! Subview requires alloc properties to be committed.\n"); + EKAT_REQUIRE_MSG( + idim == 0 || idim == 1, + "Error! Subviewing is only allowed along first or second dimension.\n"); + EKAT_REQUIRE_MSG(idim < m_layout.rank(), + "Error! Dimension index out of bounds.\n"); + EKAT_REQUIRE_MSG(k_beg <= k_end, + "Error! Slice indices are invalid (non-increasing).\n"); + EKAT_REQUIRE_MSG( + k_beg >= 0 && k_end < m_layout.dim(idim), + "Error! Index range along the dimension is out of bounds.\n"); + + // Set new layout basic stuff + FieldAllocProp props(m_scalar_type_size); + props.m_committed = true; + props.m_scalar_type_size = m_scalar_type_size; + props.m_layout = + m_layout.clone_with_different_extent(idim, k_end - k_beg + 1); + + // Output is contiguous if either + // - this->m_contiguous=true AND idim==0 + // - m_layout.dim(i)==1 for all i. * Let's say you have one class that uses the field as View, - * and another class that used the field in a packed way, taht is, - * as View**>. The second view will have dimensions (3,4), + * and another class that used the field in a packed way, that is, + * as View**>. The second view will have dimensions (3,3), in terms of its value type (i.e., Pack). * This means the field needs an allocation bigger than 30 Real's, namely - * 3*4*4=36 Real's. This class does the book-keeping for the allocation size, + * 3*4*3=36 Real's. This class does the book-keeping for the allocation size, * so that 1) the field can be allocated with enough memory to accommodate * all requests, 2) customers of the field can check what the allocation * is, so that they know whether there is padding in the field, and @@ -61,16 +61,30 @@ namespace scream // Helper struct to store info related to the subviewing process struct SubviewInfo { - SubviewInfo () = default; - SubviewInfo (const int dim, const int slice, const int extent, const bool is_dynamic) : - dim_idx(dim), slice_idx(slice), dim_extent(extent), dynamic(is_dynamic) {} - SubviewInfo (const SubviewInfo&) = default; - SubviewInfo& operator= (const SubviewInfo&) = default; - - int dim_idx = -1; // Dimension along which slicing happened - int slice_idx = -1; // Slice along dimension $dim_idx - int dim_extent = -1; // Extent of dimension $dim_idx - bool dynamic = false; // Whether this is a dynamic subview (slice_idx can change) + SubviewInfo() = default; + SubviewInfo(const int dim, const int slice, const int extent, + const bool is_dynamic) + : dim_idx(dim), slice_idx(slice), dim_extent(extent), + dynamic(is_dynamic) {} + // multi-slice subview across contiguous indices + SubviewInfo(const int dim, const int slice_beg, const int slice_end, + const int extent, const bool is_dynamic) + : dim_idx(dim), slice_idx_beg(slice_beg), slice_idx_end(slice_end), + dim_extent(extent), dynamic(is_dynamic) {} + SubviewInfo(const SubviewInfo&) = default; + SubviewInfo& operator=(const SubviewInfo&) = default; + + // TODO: is it better to do it this way, or to create an alternative struct? + int dim_idx = -1; // Dimension along which slicing happened + // Slice along dimension $dim_idx (uninitialized if multi-slice subview) + int slice_idx = -1; + // beginning slice index for multi-slice (remains == -1 if single-slice subview) + int slice_idx_beg = -1; + // ending slice index for multi-slice (remains == -1 if single-slice subview) + int slice_idx_end = -1; + int dim_extent = -1; // Extent of dimension $dim_idx + bool dynamic = + false; // Whether this is a dynamic subview (slice_idx can change) }; inline bool operator== (const SubviewInfo& lhs, const SubviewInfo& rhs) { @@ -95,6 +109,10 @@ class FieldAllocProp { // possible to change the entry index (k) at runtime. FieldAllocProp subview (const int idim, const int k, const bool dynamic) const; + // multi-slice subview over contiguous indices + FieldAllocProp subview (const int idim, const int k_beg, const int k_end, + const bool dynamic) const; + // Request allocation able to accommodate a pack of ScalarType of the given pack size void request_allocation (const int pack_size = 1); diff --git a/components/eamxx/src/share/field/field_header.cpp b/components/eamxx/src/share/field/field_header.cpp index 9bb642f2236f..934084b30805 100644 --- a/components/eamxx/src/share/field/field_header.cpp +++ b/components/eamxx/src/share/field/field_header.cpp @@ -2,8 +2,7 @@ #include "ekat/std_meta/ekat_std_utils.hpp" -namespace scream -{ +namespace scream { FieldHeader::FieldHeader (const identifier_type& id) : m_identifier (id) @@ -33,10 +32,7 @@ set_extra_data (const std::string& key, } } -std::shared_ptr -FieldHeader:: -alias (const std::string& name) const -{ +std::shared_ptr FieldHeader::alias(const std::string& name) const { auto fh = create_header(get_identifier().alias(name)); fh->m_tracking = m_tracking; fh->m_alloc_prop = m_alloc_prop; @@ -72,4 +68,33 @@ create_subfield_header (const FieldIdentifier& id, return fh; } +// subfield with multiple, contiguous slices +std::shared_ptr +create_subfield_header(const FieldIdentifier& id, + std::shared_ptr parent, const int idim, + const int k_beg, const int k_end, const bool dynamic) { + // Sanity checks + EKAT_REQUIRE_MSG(parent != nullptr, + "Error! Invalid pointer for parent header.\n"); + EKAT_REQUIRE_MSG(k_end > k_beg, + "Error! Slice indices are invalid (non-increasing).\n"); + + // Create header, and set up parent/child + auto fh = create_header(id); + fh->create_parent_child_link(parent); + + // Create tracking, and set up parent/child + fh->m_tracking = create_tracking(); + fh->m_tracking->create_parent_child_link(parent->get_tracking_ptr()); + if (parent->get_tracking().get_time_stamp().is_valid()) { + fh->m_tracking->update_time_stamp(parent->get_tracking().get_time_stamp()); + } + + // Create alloc props + fh->m_alloc_prop = std::make_shared( + parent->get_alloc_properties().subview(idim, k_beg, k_end, dynamic)); + + return fh; +} + } // namespace scream diff --git a/components/eamxx/src/share/field/field_header.hpp b/components/eamxx/src/share/field/field_header.hpp index 511c6b6f5056..842842720137 100644 --- a/components/eamxx/src/share/field/field_header.hpp +++ b/components/eamxx/src/share/field/field_header.hpp @@ -91,6 +91,12 @@ class FieldHeader : public FamilyTracking { create_subfield_header (const FieldIdentifier&, std::shared_ptr, const int, const int, const bool); + // for creating multi-slice subfield (continuous indices) + friend std::shared_ptr + create_subfield_header (const FieldIdentifier&, + std::shared_ptr, + const int idim, const int k_beg, const int k_end, + const bool dynamic); // NOTE: the identifier *cannot* be a shared_ptr, b/c we // don't foresee sharing an identifier between two @@ -169,6 +175,11 @@ std::shared_ptr create_subfield_header (const FieldIdentifier& id, std::shared_ptr parent, const int idim, const int k, const bool dynamic); +std::shared_ptr +create_subfield_header (const FieldIdentifier& id, + std::shared_ptr parent, + const int idim, const int k_beg, const int k_end, + const bool dynamic); } // namespace scream From ccc846a30d87ddb3caf066d4134145e69dcdbcb4 Mon Sep 17 00:00:00 2001 From: "Michael J. Schmidt" Date: Mon, 1 Apr 2024 17:26:19 -0600 Subject: [PATCH 215/476] got get_strided_view() working for multi-sliced field--ugly impl at the moment --- components/eamxx/src/share/field/field.hpp | 6 +- .../src/share/field/field_alloc_prop.hpp | 8 +- .../eamxx/src/share/field/field_impl.hpp | 89 ++++++++++++++++++- .../eamxx/src/share/tests/field_tests.cpp | 49 ++++++++++ 4 files changed, 144 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index c5c7ef6de406..b716572f0f4d 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -136,6 +136,10 @@ class Field { get_strided_view_type get_strided_view () const; + template + Kokkos::View + get_strided_view (bool special) const; + // These two getters are convenience function for commonly accessed metadata. // The same info can be extracted from the metadata stored in the FieldHeader DataType data_type () const { return get_header().get_identifier().data_type(); } @@ -327,7 +331,7 @@ class Field { auto get_ND_view () const -> if_t<(N,HD>>; - // Metadata (name, rank, dims, customere/providers, time stamp, ...) + // Metadata (name, rank, dims, customer/providers, time stamp, ...) std::shared_ptr m_header; // Actual data. diff --git a/components/eamxx/src/share/field/field_alloc_prop.hpp b/components/eamxx/src/share/field/field_alloc_prop.hpp index c3f870b42800..6f23d579d897 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.hpp +++ b/components/eamxx/src/share/field/field_alloc_prop.hpp @@ -69,14 +69,15 @@ struct SubviewInfo { // multi-slice subview across contiguous indices SubviewInfo(const int dim, const int slice_beg, const int slice_end, const int extent, const bool is_dynamic) - : dim_idx(dim), slice_idx_beg(slice_beg), slice_idx_end(slice_end), + : dim_idx(dim), slice_idx(slice_beg), slice_idx_end(slice_end), dim_extent(extent), dynamic(is_dynamic) {} SubviewInfo(const SubviewInfo&) = default; SubviewInfo& operator=(const SubviewInfo&) = default; - // TODO: is it better to do it this way, or to create an alternative struct? int dim_idx = -1; // Dimension along which slicing happened - // Slice along dimension $dim_idx (uninitialized if multi-slice subview) + // Slice along dimension $dim_idx if taking single-index slice for subfield + // If taking multi-slice subfield, then this is the starting slice index + // e.g., slicing (:, slice_idx_end : slice_idx_end, :) int slice_idx = -1; // beginning slice index for multi-slice (remains == -1 if single-slice subview) int slice_idx_beg = -1; @@ -90,6 +91,7 @@ struct SubviewInfo { inline bool operator== (const SubviewInfo& lhs, const SubviewInfo& rhs) { return lhs.dim_idx==rhs.dim_idx && lhs.slice_idx==rhs.slice_idx && + // FIXME: change for multi-slice (throw error to check?) lhs.dim_extent==rhs.dim_extent && lhs.dynamic==rhs.dynamic; } diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index f192d70bb688..7ad4e231bd83 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -40,7 +40,7 @@ Field (const identifier_type& id, " - field name: " + id.name() + "\n" " - idim: " + std::to_string(i) + "\n" " - layout i-th dim: " + std::to_string(fl.dims()[i]) + "\n" - " - view i-th dim: " + std::to_string(view_d.extent(i)) + "\n"); + " - view i-th dim: " + std::to_string(view_d.extent(i)) + "\n"); } auto& alloc_prop = m_header->get_alloc_properties(); @@ -49,7 +49,7 @@ Field (const identifier_type& id, "Error! Input view has the wrong last extent.\n" " - field name: " + id.name() + "\n" " - layout last dim: " + std::to_string(fl.dims()[N-1]) + "\n" - " - view last dim: " + std::to_string(view_d.extent(N-1)) + "\n"); + " - view last dim: " + std::to_string(view_d.extent(N-1)) + "\n"); // We have a padded view. We don't know what the pack size was, so we pick the largest // power of 2 that divides the last extent @@ -73,8 +73,6 @@ Field (const identifier_type& id, // Create an unmanaged dev view, and its host mirror const auto view_dim = alloc_prop.get_alloc_size(); char* data = reinterpret_cast(view_d.data()); - std::cout << "fl: " << fl.to_string() << "\n" - << "view dim: " << view_dim << "\n"; m_data.d_view = decltype(m_data.d_view)(data,view_dim); m_data.h_view = Kokkos::create_mirror_view(m_data.d_view); @@ -101,6 +99,10 @@ auto Field::get_view () const // Make sure input field is allocated EKAT_REQUIRE_MSG(is_allocated(), "Error! Cannot extract a field's view before allocation happens.\n"); + // FIXME: add check + // EKAT_REQUIRE_MSG(true /*is_multiSlice_subview()*/, + // "Error! Multi-sliced subfield is incompatible--must employ " + // "get_strided_view().\n") EKAT_REQUIRE_MSG (not m_is_read_only || std::is_const::value, "Error! Cannot get a view to non-const data if the field is read-only.\n"); @@ -205,6 +207,83 @@ auto Field::get_strided_view () const return DstView(get_ND_view()); } +template +Kokkos::View Field::get_strided_view (bool special) const +{ + // The destination view type on correct mem space + using DstView = get_strided_view_type; + // The dst value types + using DstValueType = typename DstView::traits::value_type; + // We only allow to reshape to a view of the correct rank + constexpr int DstRank = DstView::rank; + constexpr int DstRankDynamic = DstView::rank_dynamic; + + // Get src details + const auto& alloc_prop = m_header->get_alloc_properties(); + const auto& fl = m_header->get_identifier().get_layout(); + + // Checks + // EKAT_REQUIRE_MSG (DstRank==1 && fl.rank()==1, + // "Error! Strided view only available for rank-1 fields.\n"); + // EKAT_REQUIRE_MSG (DstRankDynamic==1, + // "Error! Strided view not allowed with compile-time dimensions.\n"); + EKAT_REQUIRE_MSG(is_allocated(), + "Error! Cannot extract a field's view before allocation happens.\n"); + EKAT_REQUIRE_MSG (not m_is_read_only || std::is_const::value, + "Error! Cannot get a view to non-const data if the field is read-only.\n"); + EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), + "Error! Source field allocation is not compatible with the requested value type.\n"); + + // Check if this field is a subview of another field + const auto parent = m_header->get_parent().lock(); + if (parent!=nullptr) { + // Parent field has correct layout to reinterpret the view into N+1-dim view + // So create the parent field on the fly, use it to get the N+1-dim view, then subview it. + // NOTE: we can set protected members, since f is the same type of this class. + Field f; + f.m_header = parent; + f.m_data = m_data; + + // // Take 2 dimensional view with normal LayoutRight + // auto v_np1 = f.get_ND_view(); + // const auto vfs_rank = f.get_header().get_identifier().get_layout().rank(); + auto v_fullsize = f.get_ND_view(); + + // Now we can subview v_np1 at the correct slice + const auto& info = m_header->get_alloc_properties().get_subview_info(); + const int idim = info.dim_idx; + const int k = info.slice_idx; + const int k_end = info.slice_idx_end; + + // // So far we can only subview at first or second dimension. + // EKAT_REQUIRE_MSG (idim==0 || idim==1, + // "Error! Subview dimension index is out of bounds.\n"); + + // Use correct subview utility + if (idim==0) { + // FIXME: what's a better way to do this? can't use v_fullsize after the + // logic block b/c of scoping, and it's also not easy to know what type of + // exotic + // we know it's a subview, so if it has the same rank as its parent, then + // we know that it's a multi-slice subview + if (fl.rank() == f.get_header().get_identifier().get_layout().rank()) { + auto svs = ekat::subview(v_fullsize, Kokkos::make_pair(k, k_end), idim); + // for (size_t i = 0; i < 4; i++) + // { + // std::cout << "i = " << i << "\n"; + // std::cout << "svs.stride(i) = " << svs.stride(i) << "\n"; + // std::cout << "svs.extent(i) = " << svs.extent(i) << "\n"; + // } + return svs; + } else { + // return DstView(ekat::subview(v_np1,k)); + } + } else { + // return DstView(ekat::subview_1(v_np1,k)); + } + } +} + template void Field:: deep_copy (const Field& src) { @@ -764,6 +843,7 @@ auto Field::get_ND_view () const -> return ret_type (ptr,kl); } +// TODO: will need to set this up for multi-sliced subfield template auto Field::get_ND_view () const -> if_t,HD>> @@ -773,6 +853,7 @@ auto Field::get_ND_view () const -> "Error! Input Rank must either be 1 (flat array) or the actual field rank.\n"); // Given that N==MaxRank, this field cannot be a subview of another field + // NOTE: this will not be true for multi-slice EKAT_REQUIRE_MSG (m_header->get_parent().expired(), "Error! A view of rank " + std::to_string(MaxRank) + " should not be the subview of another field.\n"); diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index a1336ae346fb..0411755dda40 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -308,6 +308,55 @@ TEST_CASE("field", "") { REQUIRE (f3.is_read_only()); } + // Subfields (multi-sliced) + SECTION ("subfield--multi-slice") { + std::vector t1 = {COL,CMP,CMP,LEV}; + std::vector d1 = {5,10,2,24}; + + FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); + + Field f1(fid1); + f1.allocate_view(); + randomize(f1,engine,pdf); + + const int idim = 0; + const int ivar = 2; + const int sl_beg = 2; + const int sl_end = 4; + + auto f3 = f1.subfield(idim, sl_beg, sl_end); + + // const auto fl3 = f3.get_header().get_identifier().get_layout(); + // const auto alprop3 = f3.get_header().get_alloc_properties(); + // std::cout << "fl3.rank() = " << fl3.rank() << "\n"; + // auto f3dim = fl3.dims(); + // for (size_t i = 0; i < 4; i++) + // { + // std::cout << "f3dim(i) = " << f3dim[i] << "\n"; + // } + // auto f3ext = fl3.extents(); + // for (size_t i = 0; i < 4; i++) + // { + // std::cout << "f3ext(i) = " << f3ext(i) << "\n"; + // } + // std::cout << "alprop3.is_subfield() = " << alprop3.is_subfield() << "\n"; + + auto v3_h = f3.get_strided_view(true); + auto v4d_h = f1.get_view(); + + for (size_t i = sl_beg; i < sl_end; i++) { + for (size_t j = 0; j < d1[1]; j++) { + for (size_t k = 0; k < d1[2]; k++) { + for (size_t l = 0; l < d1[3]; l++) { + // std::cout << "v4d_h(i, j, k, l) = " << v4d_h(i, j, k, l) << "\n"; + // std::cout << "v3_h(i - sl_beg, j, k, l) = " << v3_h(i - sl_beg, j, k, l) << "\n"; + REQUIRE(v4d_h(i, j, k, l) == v3_h(i - sl_beg, j, k, l)); + } + } + } + } + } + // Dynamic Subfields SECTION ("dynamic_subfield") { const int vec_dim = 10; From da94fdb73082dce84474ea6a55b74f8c9cf25f4c Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 3 Apr 2024 19:40:44 -0600 Subject: [PATCH 216/476] multi-sliced subfield/subviews working --- components/eamxx/src/share/field/field.cpp | 4 +- components/eamxx/src/share/field/field.hpp | 11 +-- .../src/share/field/field_alloc_prop.cpp | 12 ++-- .../src/share/field/field_alloc_prop.hpp | 2 +- .../eamxx/src/share/field/field_impl.hpp | 72 +++++-------------- .../eamxx/src/share/tests/field_tests.cpp | 53 ++++++-------- 6 files changed, 56 insertions(+), 98 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 11c8e695b4be..1e790ae3ceac 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -129,8 +129,8 @@ Field Field::subfield(const std::string& sf_name, EKAT_REQUIRE_MSG( is_allocated(), "Error! Input field must be allocated in order to subview it.\n"); - EKAT_REQUIRE_MSG(idim == 0 || idim == 1, - "Error! Subview dimension index must be either 0 or 1.\n"); + // EKAT_REQUIRE_MSG(idim == 0 || idim == 1, + // "Error! Subview dimension index must be either 0 or 1.\n"); auto sf_layout = lt.clone_with_different_extent(idim, index_end - index_beg + 1); diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index b716572f0f4d..98169ece28d4 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -136,9 +136,11 @@ class Field { get_strided_view_type get_strided_view () const; - template - Kokkos::View - get_strided_view (bool special) const; + // this is the same as above but for a multi-sliced view + // which is to say, also a strided view + template + auto get_multi_sliced_view () const + -> get_strided_view_type, HD>; // These two getters are convenience function for commonly accessed metadata. // The same info can be extracted from the metadata stored in the FieldHeader @@ -245,10 +247,9 @@ class Field { Field subfield (const std::string& sf_name, const int idim, const int index, const bool dynamic = false) const; Field subfield (const int idim, const int k, const bool dynamic = false) const; - // get subfield to extract multiple slices in a continuous range of indices + // subfield fxn to extract multiple slices in a continuous range of indices // e.g., (in matlab syntax) subf = f.subfield(:, 1:3, :) // but NOT subf = f.subfield(:, [1, 3, 4], :) - // NOTE: use std::pair(index_beg, index_end) or a kokkos::pair()? Field subfield (const std::string& sf_name, const ekat::units::Units& sf_units, const int idim, const int index_beg, const int index_end, const bool dynamic = false) const; diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index f2375869f84f..08d850a9325f 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -77,22 +77,20 @@ subview (const int idim, const int k, const bool dynamic) const { return props; } -FieldAllocProp FieldAllocProp::subview(const int idim, const int k_beg, +FieldAllocProp FieldAllocProp::subview(const int idim, + const int k_beg, const int k_end, const bool dynamic) const { EKAT_REQUIRE_MSG( is_committed(), "Error! Subview requires alloc properties to be committed.\n"); - EKAT_REQUIRE_MSG( - idim == 0 || idim == 1, - "Error! Subviewing is only allowed along first or second dimension.\n"); - EKAT_REQUIRE_MSG(idim < m_layout.rank(), + EKAT_REQUIRE_MSG(idim <= m_layout.rank(), "Error! Dimension index out of bounds.\n"); - EKAT_REQUIRE_MSG(k_beg <= k_end, + EKAT_REQUIRE_MSG(k_beg < k_end, "Error! Slice indices are invalid (non-increasing).\n"); EKAT_REQUIRE_MSG( k_beg >= 0 && k_end < m_layout.dim(idim), - "Error! Index range along the dimension is out of bounds.\n"); + "Error! Slice index range along the idim dimension is out of bounds.\n"); // Set new layout basic stuff FieldAllocProp props(m_scalar_type_size); diff --git a/components/eamxx/src/share/field/field_alloc_prop.hpp b/components/eamxx/src/share/field/field_alloc_prop.hpp index 6f23d579d897..481182fbdc6b 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.hpp +++ b/components/eamxx/src/share/field/field_alloc_prop.hpp @@ -106,7 +106,7 @@ class FieldAllocProp { FieldAllocProp& operator= (const FieldAllocProp&); - // Return allocation props of an unmanaged subivew of this field + // Return allocation props of an unmanaged subview of this field // at entry k along dimension idim. If dynamic = true, then it will be // possible to change the entry index (k) at runtime. FieldAllocProp subview (const int idim, const int k, const bool dynamic) const; diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 7ad4e231bd83..b938cb6070f7 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -99,10 +99,6 @@ auto Field::get_view () const // Make sure input field is allocated EKAT_REQUIRE_MSG(is_allocated(), "Error! Cannot extract a field's view before allocation happens.\n"); - // FIXME: add check - // EKAT_REQUIRE_MSG(true /*is_multiSlice_subview()*/, - // "Error! Multi-sliced subfield is incompatible--must employ " - // "get_strided_view().\n") EKAT_REQUIRE_MSG (not m_is_read_only || std::is_const::value, "Error! Cannot get a view to non-const data if the field is read-only.\n"); @@ -207,26 +203,26 @@ auto Field::get_strided_view () const return DstView(get_ND_view()); } -template -Kokkos::View Field::get_strided_view (bool special) const +// NOTE: multi-slicing a view is only supported for strided view return type +template +auto Field::get_multi_sliced_view () const + -> get_strided_view_type, HD> { // The destination view type on correct mem space - using DstView = get_strided_view_type; + using DstView = get_strided_view_type, HD>; // The dst value types using DstValueType = typename DstView::traits::value_type; - // We only allow to reshape to a view of the correct rank - constexpr int DstRank = DstView::rank; - constexpr int DstRankDynamic = DstView::rank_dynamic; // Get src details const auto& alloc_prop = m_header->get_alloc_properties(); const auto& fl = m_header->get_identifier().get_layout(); // Checks - // EKAT_REQUIRE_MSG (DstRank==1 && fl.rank()==1, - // "Error! Strided view only available for rank-1 fields.\n"); - // EKAT_REQUIRE_MSG (DstRankDynamic==1, + // TODO: decide whether a dynamic-rank view is ok + // EKAT_REQUIRE_MSG (DstRankDynamic == 1, // "Error! Strided view not allowed with compile-time dimensions.\n"); + EKAT_REQUIRE_MSG (N == fl.rank(), + "Error! Input Rank must be equal to parent view's rank for multi-sliced subview.\n"); EKAT_REQUIRE_MSG(is_allocated(), "Error! Cannot extract a field's view before allocation happens.\n"); EKAT_REQUIRE_MSG (not m_is_read_only || std::is_const::value, @@ -236,51 +232,23 @@ Kokkos::View Field::get_strided_view (bool speci // Check if this field is a subview of another field const auto parent = m_header->get_parent().lock(); - if (parent!=nullptr) { - // Parent field has correct layout to reinterpret the view into N+1-dim view - // So create the parent field on the fly, use it to get the N+1-dim view, then subview it. - // NOTE: we can set protected members, since f is the same type of this class. + if (parent != nullptr) { Field f; f.m_header = parent; f.m_data = m_data; - // // Take 2 dimensional view with normal LayoutRight - // auto v_np1 = f.get_ND_view(); - // const auto vfs_rank = f.get_header().get_identifier().get_layout().rank(); - auto v_fullsize = f.get_ND_view(); + auto v_fullsize = f.get_ND_view(); // Now we can subview v_np1 at the correct slice const auto& info = m_header->get_alloc_properties().get_subview_info(); - const int idim = info.dim_idx; - const int k = info.slice_idx; - const int k_end = info.slice_idx_end; - - // // So far we can only subview at first or second dimension. - // EKAT_REQUIRE_MSG (idim==0 || idim==1, - // "Error! Subview dimension index is out of bounds.\n"); - - // Use correct subview utility - if (idim==0) { - // FIXME: what's a better way to do this? can't use v_fullsize after the - // logic block b/c of scoping, and it's also not easy to know what type of - // exotic - // we know it's a subview, so if it has the same rank as its parent, then - // we know that it's a multi-slice subview - if (fl.rank() == f.get_header().get_identifier().get_layout().rank()) { - auto svs = ekat::subview(v_fullsize, Kokkos::make_pair(k, k_end), idim); - // for (size_t i = 0; i < 4; i++) - // { - // std::cout << "i = " << i << "\n"; - // std::cout << "svs.stride(i) = " << svs.stride(i) << "\n"; - // std::cout << "svs.extent(i) = " << svs.extent(i) << "\n"; - // } - return svs; - } else { - // return DstView(ekat::subview(v_np1,k)); - } - } else { - // return DstView(ekat::subview_1(v_np1,k)); - } + const int idim = info.dim_idx; + const int k = info.slice_idx; + const int k_end = info.slice_idx_end; + + // this version of ekat::subview overloaded conveniently, so only the single + // version of the call is required here + return DstView(ekat::subview(v_fullsize, + Kokkos::make_pair(k, k_end), idim)); } } @@ -843,7 +811,6 @@ auto Field::get_ND_view () const -> return ret_type (ptr,kl); } -// TODO: will need to set this up for multi-sliced subfield template auto Field::get_ND_view () const -> if_t,HD>> @@ -853,7 +820,6 @@ auto Field::get_ND_view () const -> "Error! Input Rank must either be 1 (flat array) or the actual field rank.\n"); // Given that N==MaxRank, this field cannot be a subview of another field - // NOTE: this will not be true for multi-slice EKAT_REQUIRE_MSG (m_header->get_parent().expired(), "Error! A view of rank " + std::to_string(MaxRank) + " should not be the subview of another field.\n"); diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 0411755dda40..232ef565830a 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -319,38 +319,31 @@ TEST_CASE("field", "") { f1.allocate_view(); randomize(f1,engine,pdf); - const int idim = 0; - const int ivar = 2; - const int sl_beg = 2; - const int sl_end = 4; - - auto f3 = f1.subfield(idim, sl_beg, sl_end); - - // const auto fl3 = f3.get_header().get_identifier().get_layout(); - // const auto alprop3 = f3.get_header().get_alloc_properties(); - // std::cout << "fl3.rank() = " << fl3.rank() << "\n"; - // auto f3dim = fl3.dims(); - // for (size_t i = 0; i < 4; i++) - // { - // std::cout << "f3dim(i) = " << f3dim[i] << "\n"; - // } - // auto f3ext = fl3.extents(); - // for (size_t i = 0; i < 4; i++) - // { - // std::cout << "f3ext(i) = " << f3ext(i) << "\n"; - // } - // std::cout << "alprop3.is_subfield() = " << alprop3.is_subfield() << "\n"; - - auto v3_h = f3.get_strided_view(true); + const int idim[4] = {0, 1, 2, 3}; + const int sl_beg[4] = {2, 3, 0, 9}; + const int sl_end[4] = {4, 6, 1, 15}; + auto v4d_h = f1.get_view(); - for (size_t i = sl_beg; i < sl_end; i++) { - for (size_t j = 0; j < d1[1]; j++) { - for (size_t k = 0; k < d1[2]; k++) { - for (size_t l = 0; l < d1[3]; l++) { - // std::cout << "v4d_h(i, j, k, l) = " << v4d_h(i, j, k, l) << "\n"; - // std::cout << "v3_h(i - sl_beg, j, k, l) = " << v3_h(i - sl_beg, j, k, l) << "\n"; - REQUIRE(v4d_h(i, j, k, l) == v3_h(i - sl_beg, j, k, l)); + int i1, i2, j1, j2, k1, k2, l1, l2; + + for (int ens = 0; ens < 4; ens++) { + auto sf = f1.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + auto sv_h = sf.get_multi_sliced_view(); + i1 = (ens == 0) ? sl_beg[0] : 0; + i2 = (ens == 0) ? sl_end[0] : d1[0]; + j1 = (ens == 1) ? sl_beg[1] : 0; + j2 = (ens == 1) ? sl_end[1] : d1[1]; + k1 = (ens == 2) ? sl_beg[2] : 0; + k2 = (ens == 2) ? sl_end[2] : d1[2]; + l1 = (ens == 3) ? sl_beg[3] : 0; + l2 = (ens == 3) ? sl_end[3] : d1[3]; + for (int i = i1; i < i2; i++) { + for (int j = j1; j < j2; j++) { + for (int k = k1; k < k2; k++) { + for (int l = l1; l < l2; l++) { + REQUIRE(v4d_h(i, j, k, l) == sv_h(i - i1, j - j1, k - k1, l - l1)); + } } } } From bcda3c0dc2ac444eb3a565d261941f392827a1a3 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 8 Apr 2024 18:23:41 -0600 Subject: [PATCH 217/476] tests finished for multi-slice sub{field,view} --- .../eamxx/src/share/field/field_impl.hpp | 44 +- .../eamxx/src/share/tests/CMakeLists.txt | 3 + .../eamxx/src/share/tests/field_tests.cpp | 149 ------- .../eamxx/src/share/tests/subfield_tests.cpp | 401 ++++++++++++++++++ 4 files changed, 426 insertions(+), 171 deletions(-) create mode 100644 components/eamxx/src/share/tests/subfield_tests.cpp diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index b938cb6070f7..b0854569b248 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -204,6 +204,8 @@ auto Field::get_strided_view () const } // NOTE: multi-slicing a view is only supported for strided view return type +// TODO: N doesn't actually need to be passed in, since the subview must have +// the same rank as its parent template auto Field::get_multi_sliced_view () const -> get_strided_view_type, HD> @@ -218,9 +220,6 @@ auto Field::get_multi_sliced_view () const const auto& fl = m_header->get_identifier().get_layout(); // Checks - // TODO: decide whether a dynamic-rank view is ok - // EKAT_REQUIRE_MSG (DstRankDynamic == 1, - // "Error! Strided view not allowed with compile-time dimensions.\n"); EKAT_REQUIRE_MSG (N == fl.rank(), "Error! Input Rank must be equal to parent view's rank for multi-sliced subview.\n"); EKAT_REQUIRE_MSG(is_allocated(), @@ -230,26 +229,27 @@ auto Field::get_multi_sliced_view () const EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), "Error! Source field allocation is not compatible with the requested value type.\n"); - // Check if this field is a subview of another field + // get parent header and check if this field is a subview of another field const auto parent = m_header->get_parent().lock(); - if (parent != nullptr) { - Field f; - f.m_header = parent; - f.m_data = m_data; - - auto v_fullsize = f.get_ND_view(); - - // Now we can subview v_np1 at the correct slice - const auto& info = m_header->get_alloc_properties().get_subview_info(); - const int idim = info.dim_idx; - const int k = info.slice_idx; - const int k_end = info.slice_idx_end; - - // this version of ekat::subview overloaded conveniently, so only the single - // version of the call is required here - return DstView(ekat::subview(v_fullsize, - Kokkos::make_pair(k, k_end), idim)); - } + EKAT_REQUIRE_MSG(parent != nullptr, + "Error! Multi-sliced subview is unavailable for non-subfields.\n"); + Field f; + // create new field with the header/data from the parent + f.m_header = parent; + f.m_data = m_data; + + auto v_fullsize = f.get_ND_view(); + + // Now we can subview v_np1 at the correct slice + const auto& info = m_header->get_alloc_properties().get_subview_info(); + const int idim = info.dim_idx; + const int k_beg = info.slice_idx; + const int k_end = info.slice_idx_end; + + // this version of ekat::subview overloaded conveniently, so only the single + // version of the call is required here + return DstView(ekat::subview(v_fullsize, + Kokkos::make_pair(k_beg, k_end), idim)); } template diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index bfdc0bba2b7e..4aab3ef95da6 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -14,6 +14,9 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test fields CreateUnitTest(field "field_tests.cpp") + # Test fields + CreateUnitTest(subfield "subfield_tests.cpp") + # Test field utils CreateUnitTest(field_utils "field_utils.cpp" MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS}) diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 232ef565830a..1cb69f474dee 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -276,155 +276,6 @@ TEST_CASE("field", "") { } } - // Subfields - SECTION ("subfield") { - std::vector t1 = {COL,CMP,CMP,LEV}; - std::vector d1 = {3,10,2,24}; - - FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); - - Field f1(fid1); - f1.allocate_view(); - randomize(f1,engine,pdf); - - const int idim = 1; - const int ivar = 2; - - auto f2 = f1.subfield(idim,ivar); - - // Wrong rank for the subfield f2 - REQUIRE_THROWS(f2.get_view()); - - auto v4d_h = f1.get_view(); - auto v3d_h = f2.get_view(); - for (int i=0; i t1 = {COL,CMP,CMP,LEV}; - std::vector d1 = {5,10,2,24}; - - FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); - - Field f1(fid1); - f1.allocate_view(); - randomize(f1,engine,pdf); - - const int idim[4] = {0, 1, 2, 3}; - const int sl_beg[4] = {2, 3, 0, 9}; - const int sl_end[4] = {4, 6, 1, 15}; - - auto v4d_h = f1.get_view(); - - int i1, i2, j1, j2, k1, k2, l1, l2; - - for (int ens = 0; ens < 4; ens++) { - auto sf = f1.subfield(idim[ens], sl_beg[ens], sl_end[ens]); - auto sv_h = sf.get_multi_sliced_view(); - i1 = (ens == 0) ? sl_beg[0] : 0; - i2 = (ens == 0) ? sl_end[0] : d1[0]; - j1 = (ens == 1) ? sl_beg[1] : 0; - j2 = (ens == 1) ? sl_end[1] : d1[1]; - k1 = (ens == 2) ? sl_beg[2] : 0; - k2 = (ens == 2) ? sl_end[2] : d1[2]; - l1 = (ens == 3) ? sl_beg[3] : 0; - l2 = (ens == 3) ? sl_end[3] : d1[3]; - for (int i = i1; i < i2; i++) { - for (int j = j1; j < j2; j++) { - for (int k = k1; k < k2; k++) { - for (int l = l1; l < l2; l++) { - REQUIRE(v4d_h(i, j, k, l) == sv_h(i - i1, j - j1, k - k1, l - l1)); - } - } - } - } - } - } - - // Dynamic Subfields - SECTION ("dynamic_subfield") { - const int vec_dim = 10; - std::vector t1 = {COL,CMP,CMP,LEV}; - std::vector d1 = {3,vec_dim,2,24}; - - FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); - - Field f1(fid1); - f1.allocate_view(); - randomize(f1,engine,pdf); - - const int idim = 1; - const int ivar = 0; - - auto f2 = f1.subfield(idim,ivar,/* dynamic = */ true); - - // Cannot reset subview idx of non-subfield fields - REQUIRE_THROWS(f1.get_header().get_alloc_properties().reset_subview_idx(0)); - - // subview idx out of bounds - auto& f2_ap = f2.get_header().get_alloc_properties(); - REQUIRE_THROWS(f2_ap.reset_subview_idx(-1)); - REQUIRE_THROWS(f2_ap.reset_subview_idx(vec_dim)); - - // Fill f1 with random numbers, and verify corresponding subviews get same values - randomize(f1,engine,pdf); - - for (int ivar_dyn=0; ivar_dyn(); - auto v3d_h = f2.get_view(); - for (int i=0; i tags_2 = {COL,CMP,LEV}; - std::vector dims_2 = {3,2,24}; - - FieldIdentifier fid_2("vec_3d",{tags_2,dims_2},m/s,"some_grid"); - - Field f_vec(fid_2); - f_vec.allocate_view(); - - auto f0 = f_vec.get_component(0); - auto f1 = f_vec.get_component(1); - - // No 3rd component - REQUIRE_THROWS(f_vec.get_component(2)); - - // f0 is scalar, no vector dimension - REQUIRE_THROWS(f0.get_component(0)); - - f0.deep_copy(1.0); - f1.deep_copy(2.0); - - f_vec.sync_to_host(); - - auto v = f_vec.get_view(); - for (int col=0; col<3; ++col) { - for (int lev=0; lev<24; ++lev) { - REQUIRE (v(col,0,lev)==1.0); - REQUIRE (v(col,1,lev)==2.0); - } - } - } - SECTION ("host_view") { Field f(fid); diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp new file mode 100644 index 000000000000..28e7ddf61298 --- /dev/null +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -0,0 +1,401 @@ +#include +#include + +#include "ekat/kokkos/ekat_subview_utils.hpp" +#include "share/field/field_identifier.hpp" +#include "share/field/field_header.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" +#include "share/field/field_utils.hpp" +#include "share/util/scream_setup_random_test.hpp" + +#include "share/grid/point_grid.hpp" + +#include "ekat/ekat_pack.hpp" +#include "ekat/ekat_pack_utils.hpp" +#include "ekat/util/ekat_test_utils.hpp" + +namespace { + + +TEST_CASE("field", "") { + using namespace scream; + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + auto engine = setup_random_test (); + using RPDF = std::uniform_real_distribution; + RPDF pdf(0.01,0.99); + + std::vector tags = {COL,LEV}; + std::vector dims = {3,24}; + + FieldIdentifier fid ("field_1", {tags,dims}, m/s,"some_grid"); + + // Subfields + SECTION ("subfield") { + std::vector t1 = {COL,CMP,CMP,LEV}; + std::vector d1 = {3,10,2,24}; + + FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); + + Field f1(fid1); + f1.allocate_view(); + randomize(f1,engine,pdf); + + const int idim = 1; + const int ivar = 2; + + auto f2 = f1.subfield(idim,ivar); + + // Wrong rank for the subfield f2 + REQUIRE_THROWS(f2.get_view()); + + auto v4d_h = f1.get_view(); + auto v3d_h = f2.get_view(); + for (int i=0; i t1 = {COL}; + std::vector d1 = {11}; + FieldIdentifier fid1("1d", {t1, d1}, m/s, "some_grid"); + + Field f1(fid1); + f1.allocate_view(); + randomize(f1, engine, pdf); + + const int idim = {0}; + const int sl_beg = {3}; + const int sl_end = {7}; + + auto v1d_h = f1.get_view(); + + // error on idx_end > v.extent(idim) + REQUIRE_THROWS(f1.subfield(idim, sl_beg, d1[0] + 1)); + // error on idx_beg negative + REQUIRE_THROWS(f1.subfield(idim, -1, sl_end)); + // error on idim > v.rank() + REQUIRE_THROWS(f1.subfield(1, sl_beg, sl_end)); + + auto sf = f1.subfield(idim, sl_beg, sl_end); + + REQUIRE_THROWS(f1.get_view()); + + auto sv_h = sf.get_multi_sliced_view(); + for (int i = sl_beg; i < sl_end; i++) { + REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); + } + } + SECTION ("2D multi-slice") { + std::vector t2 = {COL, CMP}; + std::vector d2 = {5, 10}; + FieldIdentifier fid2("2d", {t2, d2}, m/s, "some_grid"); + + Field f2(fid2); + f2.allocate_view(); + randomize(f2, engine, pdf); + + const int idim[2] = {0, 1}; + const int sl_beg[2] = {0, 3}; + const int sl_end[2] = {4, 6}; + + auto v2d_h = f2.get_view(); + int i1, i2, j1, j2; + + for (int ens = 0; ens < 2; ens++) { + auto sf = f2.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + auto sv_h = sf.get_multi_sliced_view(); + i1 = (ens == 0) ? sl_beg[0] : 0; + i2 = (ens == 0) ? sl_end[0] : d2[0]; + j1 = (ens == 1) ? sl_beg[1] : 0; + j2 = (ens == 1) ? sl_end[1] : d2[1]; + for (int i = i1; i < i2; i++) { + for (int j = j1; j < j2; j++) { + REQUIRE(v2d_h(i, j) == sv_h(i - i1, j - j1)); + } + } + } + } + SECTION ("3D multi-slice") { + // ============== + /* Rank-3 view */ + // ============== + std::vector t3 = {COL, CMP, LEV}; + std::vector d3 = {5, 10, 2}; + FieldIdentifier fid3("3d", {t3, d3}, m/s, "some_grid"); + + Field f3(fid3); + f3.allocate_view(); + randomize(f3, engine, pdf); + + const int idim[3] = {0, 1, 2}; + const int sl_beg[3] = {2, 3, 0}; + const int sl_end[3] = {4, 6, 1}; + + auto v3d_h = f3.get_view(); + int i1, i2, j1, j2, k1, k2; + + for (int ens = 0; ens < 3; ens++) { + auto sf = f3.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + auto sv_h = sf.get_multi_sliced_view(); + i1 = (ens == 0) ? sl_beg[0] : 0; + i2 = (ens == 0) ? sl_end[0] : d3[0]; + j1 = (ens == 1) ? sl_beg[1] : 0; + j2 = (ens == 1) ? sl_end[1] : d3[1]; + k1 = (ens == 2) ? sl_beg[2] : 0; + k2 = (ens == 2) ? sl_end[2] : d3[2]; + for (int i = i1; i < i2; i++) { + for (int j = j1; j < j2; j++) { + for (int k = k1; k < k2; k++) { + REQUIRE(v3d_h(i, j, k) == sv_h(i - i1, j - j1, k - k1)); + } + } + } + } + // ====================== + /* get_component test */ + // ====================== + auto cmp3 = f3.get_component(sl_beg[1], sl_end[1]); + REQUIRE_THROWS(f3.get_component(sl_beg[1], d3[1] + 1)); + REQUIRE_THROWS(f3.get_component(-1, sl_end[1] + 1)); + + auto svc_h = cmp3.get_multi_sliced_view(); + for (int i = 0; i < d3[0]; i++) { + for (int j = sl_beg[1]; j < sl_end[1]; j++) { + for (int k = 0; k < d3[2]; k++) { + REQUIRE(v3d_h(i, j, k) == svc_h(i, j - sl_beg[1], k)); + } + } + } + + } + SECTION ("4D multi-slice") { + // ============== + /* Rank-4 view */ + // ============== + std::vector t4 = {COL, CMP, CMP, LEV}; + std::vector d4 = {5, 10, 2, 23}; + FieldIdentifier fid4("4d", {t4, d4}, m/s, "some_grid"); + + Field f4(fid4); + f4.allocate_view(); + randomize(f4, engine, pdf); + + const int idim[4] = {0, 1, 2, 3}; + const int sl_beg[4] = {2, 3, 0, 9}; + const int sl_end[4] = {4, 6, 1, 15}; + + auto v4d_h = f4.get_view(); + int i1, i2, j1, j2, k1, k2, l1, l2; + + for (int ens = 0; ens < 4; ens++) { + auto sf = f4.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + auto sv_h = sf.get_multi_sliced_view(); + i1 = (ens == 0) ? sl_beg[0] : 0; + i2 = (ens == 0) ? sl_end[0] : d4[0]; + j1 = (ens == 1) ? sl_beg[1] : 0; + j2 = (ens == 1) ? sl_end[1] : d4[1]; + k1 = (ens == 2) ? sl_beg[2] : 0; + k2 = (ens == 2) ? sl_end[2] : d4[2]; + l1 = (ens == 3) ? sl_beg[3] : 0; + l2 = (ens == 3) ? sl_end[3] : d4[3]; + for (int i = i1; i < i2; i++) { + for (int j = j1; j < j2; j++) { + for (int k = k1; k < k2; k++) { + for (int l = l1; l < l2; l++) { + REQUIRE(v4d_h(i, j, k, l) == sv_h(i - i1, j - j1, k - k1, l - l1)); + } + } + } + } + } + } + SECTION ("5D multi-slice") { + // ============== + /* Rank-5 view */ + // ============== + std::vector t5 = {EL, CMP, GP, GP, LEV}; + std::vector d5 = {5, 10, 4, 2, 23}; + FieldIdentifier fid5("5d", {t5, d5}, m/s, "some_grid"); + + Field f5(fid5); + f5.allocate_view(); + randomize(f5, engine, pdf); + + const int idim[5] = {0, 1, 2, 3, 4}; + const int sl_beg[5] = {2, 3, 1, 0, 9}; + const int sl_end[5] = {4, 6, 3, 1, 15}; + + auto v5d_h = f5.get_view(); + int i1, i2, j1, j2, k1, k2, l1, l2, m1, m2; + + for (int ens = 0; ens < 5; ens++) { + auto sf = f5.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + auto sv_h = sf.get_multi_sliced_view(); + i1 = (ens == 0) ? sl_beg[0] : 0; + i2 = (ens == 0) ? sl_end[0] : d5[0]; + j1 = (ens == 1) ? sl_beg[1] : 0; + j2 = (ens == 1) ? sl_end[1] : d5[1]; + k1 = (ens == 2) ? sl_beg[2] : 0; + k2 = (ens == 2) ? sl_end[2] : d5[2]; + l1 = (ens == 3) ? sl_beg[3] : 0; + l2 = (ens == 3) ? sl_end[3] : d5[3]; + m1 = (ens == 4) ? sl_beg[4] : 0; + m2 = (ens == 4) ? sl_end[4] : d5[4]; + for (int i = i1; i < i2; i++) { + for (int j = j1; j < j2; j++) { + for (int k = k1; k < k2; k++) { + for (int l = l1; l < l2; l++) { + for (int m = m1; m < m2; m++) { + REQUIRE(v5d_h(i, j, k, l, m) == sv_h(i - i1, j - j1, k - k1, + l - l1, m - m1)); + } + } + } + } + } + } + } + SECTION ("6D multi-slice") { + // ============== + /* Rank-6 view */ + // ============== + std::vector t6 = {EL, TL, CMP, GP, GP, LEV}; + std::vector d6 = {5, 10, 4, 2, 9, 23}; + FieldIdentifier fid6("6d", {t6, d6}, m/s, "some_grid"); + + Field f6(fid6); + f6.allocate_view(); + randomize(f6, engine, pdf); + + const int idim[6] = {0, 1, 2, 3, 4, 5}; + const int sl_beg[6] = {2, 3, 1, 0, 5, 9}; + const int sl_end[6] = {4, 6, 3, 1, 8, 15}; + + auto v6d_h = f6.get_view(); + int i1, i2, j1, j2, k1, k2, l1, l2, m1, m2, n1, n2; + + for (int ens = 0; ens < 6; ens++) { + auto sf = f6.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + auto sv_h = sf.get_multi_sliced_view(); + i1 = (ens == 0) ? sl_beg[0] : 0; + i2 = (ens == 0) ? sl_end[0] : d6[0]; + j1 = (ens == 1) ? sl_beg[1] : 0; + j2 = (ens == 1) ? sl_end[1] : d6[1]; + k1 = (ens == 2) ? sl_beg[2] : 0; + k2 = (ens == 2) ? sl_end[2] : d6[2]; + l1 = (ens == 3) ? sl_beg[3] : 0; + l2 = (ens == 3) ? sl_end[3] : d6[3]; + m1 = (ens == 4) ? sl_beg[4] : 0; + m2 = (ens == 4) ? sl_end[4] : d6[4]; + n1 = (ens == 5) ? sl_beg[5] : 0; + n2 = (ens == 5) ? sl_end[5] : d6[5]; + for (int i = i1; i < i2; i++) { + for (int j = j1; j < j2; j++) { + for (int k = k1; k < k2; k++) { + for (int l = l1; l < l2; l++) { + for (int m = m1; m < m2; m++) { + for (int n = n1; n < n2; n++) { + REQUIRE(v6d_h(i, j, k, l, m, n) == + sv_h(i - i1, j - j1, k - k1, + l - l1, m - m1, n - n1)); + } + } + } + } + } + } + } + } + } + + // Dynamic Subfields + SECTION ("dynamic_subfield") { + const int vec_dim = 10; + std::vector t1 = {COL,CMP,CMP,LEV}; + std::vector d1 = {3,vec_dim,2,24}; + + FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); + + Field f1(fid1); + f1.allocate_view(); + randomize(f1,engine,pdf); + + const int idim = 1; + const int ivar = 0; + + auto f2 = f1.subfield(idim,ivar,/* dynamic = */ true); + + // Cannot reset subview idx of non-subfield fields + REQUIRE_THROWS(f1.get_header().get_alloc_properties().reset_subview_idx(0)); + + // subview idx out of bounds + auto& f2_ap = f2.get_header().get_alloc_properties(); + REQUIRE_THROWS(f2_ap.reset_subview_idx(-1)); + REQUIRE_THROWS(f2_ap.reset_subview_idx(vec_dim)); + + // Fill f1 with random numbers, and verify corresponding subviews get same values + randomize(f1,engine,pdf); + + for (int ivar_dyn=0; ivar_dyn(); + auto v3d_h = f2.get_view(); + for (int i=0; i tags_2 = {COL,CMP,LEV}; + std::vector dims_2 = {3,2,24}; + + FieldIdentifier fid_2("vec_3d",{tags_2,dims_2},m/s,"some_grid"); + + Field f_vec(fid_2); + f_vec.allocate_view(); + + auto f0 = f_vec.get_component(0); + auto f1 = f_vec.get_component(1); + + // No 3rd component + REQUIRE_THROWS(f_vec.get_component(2)); + + // f0 is scalar, no vector dimension + REQUIRE_THROWS(f0.get_component(0)); + + f0.deep_copy(1.0); + f1.deep_copy(2.0); + + f_vec.sync_to_host(); + + auto v = f_vec.get_view(); + for (int col=0; col<3; ++col) { + for (int lev=0; lev<24; ++lev) { + REQUIRE (v(col,0,lev)==1.0); + REQUIRE (v(col,1,lev)==2.0); + } + } + } +} + + + + + +} // anonymous namespace From b1a19c1bdbd9d117b19f09e272507f20e39bb996 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 10 Apr 2024 22:16:06 -0600 Subject: [PATCH 218/476] address PR comments --- .../eamxx/src/share/field/field_impl.hpp | 39 ++++++++++--------- .../eamxx/src/share/tests/subfield_tests.cpp | 2 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index b0854569b248..91c196bc465c 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -206,10 +206,9 @@ auto Field::get_strided_view () const // NOTE: multi-slicing a view is only supported for strided view return type // TODO: N doesn't actually need to be passed in, since the subview must have // the same rank as its parent -template -auto Field::get_multi_sliced_view () const - -> get_strided_view_type, HD> -{ +template +auto Field::get_multi_sliced_view() const + -> get_strided_view_type, HD> { // The destination view type on correct mem space using DstView = get_strided_view_type, HD>; // The dst value types @@ -220,18 +219,22 @@ auto Field::get_multi_sliced_view () const const auto& fl = m_header->get_identifier().get_layout(); // Checks - EKAT_REQUIRE_MSG (N == fl.rank(), - "Error! Input Rank must be equal to parent view's rank for multi-sliced subview.\n"); - EKAT_REQUIRE_MSG(is_allocated(), + EKAT_REQUIRE_MSG(N == fl.rank(), "Error! Input Rank must be equal to parent " + "view's rank for multi-sliced subview.\n"); + EKAT_REQUIRE_MSG( + is_allocated(), "Error! Cannot extract a field's view before allocation happens.\n"); - EKAT_REQUIRE_MSG (not m_is_read_only || std::is_const::value, - "Error! Cannot get a view to non-const data if the field is read-only.\n"); + EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, + "Error! Cannot get a view to non-const data if the field is " + "read-only.\n"); EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), - "Error! Source field allocation is not compatible with the requested value type.\n"); + "Error! Source field allocation is not compatible with the " + "requested value type.\n"); // get parent header and check if this field is a subview of another field const auto parent = m_header->get_parent().lock(); - EKAT_REQUIRE_MSG(parent != nullptr, + EKAT_REQUIRE_MSG( + parent != nullptr, "Error! Multi-sliced subview is unavailable for non-subfields.\n"); Field f; // create new field with the header/data from the parent @@ -242,14 +245,12 @@ auto Field::get_multi_sliced_view () const // Now we can subview v_np1 at the correct slice const auto& info = m_header->get_alloc_properties().get_subview_info(); - const int idim = info.dim_idx; - const int k_beg = info.slice_idx; - const int k_end = info.slice_idx_end; - - // this version of ekat::subview overloaded conveniently, so only the single - // version of the call is required here - return DstView(ekat::subview(v_fullsize, - Kokkos::make_pair(k_beg, k_end), idim)); + const int idim = info.dim_idx; + const int k_beg = info.slice_idx; + const int k_end = info.slice_idx_end; + + return DstView(ekat::subview( + v_fullsize, Kokkos::make_pair(k_beg, k_end), idim)); } template diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp index 28e7ddf61298..24950388fa5c 100644 --- a/components/eamxx/src/share/tests/subfield_tests.cpp +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -88,7 +88,7 @@ TEST_CASE("field", "") { auto sf = f1.subfield(idim, sl_beg, sl_end); - REQUIRE_THROWS(f1.get_view()); + REQUIRE_THROWS(f1.get_multi_sliced_view()); auto sv_h = sf.get_multi_sliced_view(); for (int i = sl_beg; i < sl_end; i++) { From e57794738560417c4f8566dfba960e5a91a63e6b Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 11 Apr 2024 15:57:02 -0600 Subject: [PATCH 219/476] remove require_throws() from subfield tests, and point ekat to master after merge --- .../eamxx/src/share/tests/subfield_tests.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp index 24950388fa5c..fd4203feefb9 100644 --- a/components/eamxx/src/share/tests/subfield_tests.cpp +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -78,19 +78,9 @@ TEST_CASE("field", "") { const int sl_end = {7}; auto v1d_h = f1.get_view(); - - // error on idx_end > v.extent(idim) - REQUIRE_THROWS(f1.subfield(idim, sl_beg, d1[0] + 1)); - // error on idx_beg negative - REQUIRE_THROWS(f1.subfield(idim, -1, sl_end)); - // error on idim > v.rank() - REQUIRE_THROWS(f1.subfield(1, sl_beg, sl_end)); - - auto sf = f1.subfield(idim, sl_beg, sl_end); - - REQUIRE_THROWS(f1.get_multi_sliced_view()); - + auto sf = f1.subfield(idim, sl_beg, sl_end auto sv_h = sf.get_multi_sliced_view(); + for (int i = sl_beg; i < sl_end; i++) { REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); } @@ -165,8 +155,6 @@ TEST_CASE("field", "") { /* get_component test */ // ====================== auto cmp3 = f3.get_component(sl_beg[1], sl_end[1]); - REQUIRE_THROWS(f3.get_component(sl_beg[1], d3[1] + 1)); - REQUIRE_THROWS(f3.get_component(-1, sl_end[1] + 1)); auto svc_h = cmp3.get_multi_sliced_view(); for (int i = 0; i < d3[0]; i++) { From f7ee547c3048e37bde63b1cdc351f1bac0d7d3e1 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Tue, 16 Apr 2024 15:33:46 -0600 Subject: [PATCH 220/476] fix typo --- components/eamxx/src/share/tests/subfield_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp index fd4203feefb9..5c649f9a272c 100644 --- a/components/eamxx/src/share/tests/subfield_tests.cpp +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -78,7 +78,7 @@ TEST_CASE("field", "") { const int sl_end = {7}; auto v1d_h = f1.get_view(); - auto sf = f1.subfield(idim, sl_beg, sl_end + auto sf = f1.subfield(idim, sl_beg, sl_end); auto sv_h = sf.get_multi_sliced_view(); for (int i = sl_beg; i < sl_end; i++) { From 6d14e3a90e170a5fdc1077c084731b20e9b4f6bb Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 17 Apr 2024 15:13:01 -0600 Subject: [PATCH 221/476] edits in response to Luca review --- components/eamxx/src/share/field/field.cpp | 27 ++- components/eamxx/src/share/field/field.hpp | 13 +- .../src/share/field/field_alloc_prop.cpp | 15 +- .../src/share/field/field_alloc_prop.hpp | 28 ++-- .../eamxx/src/share/field/field_header.cpp | 4 +- .../eamxx/src/share/field/field_header.hpp | 6 +- .../eamxx/src/share/tests/subfield_tests.cpp | 154 ++++++++---------- 7 files changed, 112 insertions(+), 135 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 1e790ae3ceac..f56b3b83d933 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -82,7 +82,6 @@ Field Field:: subfield (const std::string& sf_name, const ekat::units::Units& sf_units, const int idim, const int index, const bool dynamic) const { - // const auto& id = m_header->get_identifier(); const auto& lt = id.get_layout(); @@ -119,8 +118,7 @@ subfield (const int idim, const int index, const bool dynamic) const { // slice at index idim, entries \in [index_beg, index_end] Field Field::subfield(const std::string& sf_name, const ekat::units::Units& sf_units, const int idim, - const int index_beg, const int index_end, - const bool dynamic) const { + const int index_beg, const int index_end) const { const auto& id = m_header->get_identifier(); const auto& lt = id.get_layout(); @@ -129,11 +127,9 @@ Field Field::subfield(const std::string& sf_name, EKAT_REQUIRE_MSG( is_allocated(), "Error! Input field must be allocated in order to subview it.\n"); - // EKAT_REQUIRE_MSG(idim == 0 || idim == 1, - // "Error! Subview dimension index must be either 0 or 1.\n"); auto sf_layout = - lt.clone_with_different_extent(idim, index_end - index_beg + 1); + lt.clone_with_different_extent(idim, index_end - index_beg); // Create identifier for subfield FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name()); @@ -141,23 +137,22 @@ Field Field::subfield(const std::string& sf_name, // Note: we can access protected members, since it's the same type Field sf; sf.m_header = create_subfield_header(sf_id, m_header, idim, index_beg, - index_end, dynamic); + index_end); sf.m_data = m_data; return sf; } Field Field::subfield(const std::string& sf_name, const int idim, - const int index_beg, const int index_end, - const bool dynamic) const { + const int index_beg, const int index_end) const { const auto& id = m_header->get_identifier(); - return subfield(sf_name, id.get_units(), idim, index_beg, index_end, dynamic); + return subfield(sf_name, id.get_units(), idim, index_beg, index_end); } -Field Field::subfield(const int idim, const int index_beg, const int index_end, - const bool dynamic) const { - return subfield(m_header->get_identifier().name(), idim, index_beg, index_end, - dynamic); +Field Field::subfield(const int idim, const int index_beg, + const int index_end) const { + return subfield(m_header->get_identifier().name(), idim, index_beg, + index_end); } Field Field:: @@ -177,7 +172,7 @@ get_component (const int i, const bool dynamic) { return subfield (fname + "_" + std::to_string(i),idim,i,dynamic); } -Field Field::get_component(const int i1, const int i2, const bool dynamic) { +Field Field::get_components(const int i1, const int i2) { const auto& layout = get_header().get_identifier().get_layout(); const auto& fname = get_header().get_identifier().name(); EKAT_REQUIRE_MSG(layout.is_vector_layout(), @@ -195,7 +190,7 @@ Field Field::get_component(const int i1, const int i2, const bool dynamic) { // Add _$i1-$i2 to the field name, to avoid issues if the subfield is stored // in some structure that requires unique names (e.g., a remapper) return subfield(fname + "_" + std::to_string(i1) + "-" + std::to_string(i2), - idim, i1, i2, dynamic); + idim, i1, i2); } bool Field::equivalent(const Field& rhs) const diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 98169ece28d4..f4ad81997265 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -247,23 +247,20 @@ class Field { Field subfield (const std::string& sf_name, const int idim, const int index, const bool dynamic = false) const; Field subfield (const int idim, const int k, const bool dynamic = false) const; - // subfield fxn to extract multiple slices in a continuous range of indices + // extracts a subfield composed of multiple slices in a continuous range of indices // e.g., (in matlab syntax) subf = f.subfield(:, 1:3, :) // but NOT subf = f.subfield(:, [1, 3, 4], :) Field subfield (const std::string& sf_name, const ekat::units::Units& sf_units, - const int idim, const int index_beg, const int index_end, - const bool dynamic = false) const; + const int idim, const int index_beg, const int index_end) const; Field subfield (const std::string& sf_name, const int idim, - const int index_beg, const int index_end, - const bool dynamic = false) const; - Field subfield (const int idim, const int k_beg, const int k_end, - const bool dynamic = false) const; + const int index_beg, const int index_end) const; + Field subfield (const int idim, const int k_beg, const int k_end) const; // If this field is a vector field, get a subfield for the ith component. // If dynamic = true, it is possible to "reset" the component index at runtime. // Note: throws if this is not a vector field. Field get_component (const int i, const bool dynamic = false); // version for slicing across multiple, contiguous indices - Field get_component (const int i1, const int i2, const bool dynamic = false); + Field get_components (const int i1, const int i2); // Checks whether the underlying view has been already allocated. bool is_allocated () const { return m_data.d_view.data()!=nullptr; } diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index 08d850a9325f..635e62bae19c 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -79,12 +79,11 @@ subview (const int idim, const int k, const bool dynamic) const { FieldAllocProp FieldAllocProp::subview(const int idim, const int k_beg, - const int k_end, - const bool dynamic) const { + const int k_end) const { EKAT_REQUIRE_MSG( is_committed(), "Error! Subview requires alloc properties to be committed.\n"); - EKAT_REQUIRE_MSG(idim <= m_layout.rank(), + EKAT_REQUIRE_MSG(idim < m_layout.rank(), "Error! Dimension index out of bounds.\n"); EKAT_REQUIRE_MSG(k_beg < k_end, "Error! Slice indices are invalid (non-increasing).\n"); @@ -97,19 +96,17 @@ FieldAllocProp FieldAllocProp::subview(const int idim, props.m_committed = true; props.m_scalar_type_size = m_scalar_type_size; props.m_layout = - m_layout.clone_with_different_extent(idim, k_end - k_beg + 1); + m_layout.clone_with_different_extent(idim, k_end - k_beg); // Output is contiguous if either // - this->m_contiguous=true AND idim==0 // - m_layout.dim(i)==1 for all i create_subfield_header(const FieldIdentifier& id, std::shared_ptr parent, const int idim, - const int k_beg, const int k_end, const bool dynamic) { + const int k_beg, const int k_end) { // Sanity checks EKAT_REQUIRE_MSG(parent != nullptr, "Error! Invalid pointer for parent header.\n"); @@ -92,7 +92,7 @@ create_subfield_header(const FieldIdentifier& id, // Create alloc props fh->m_alloc_prop = std::make_shared( - parent->get_alloc_properties().subview(idim, k_beg, k_end, dynamic)); + parent->get_alloc_properties().subview(idim, k_beg, k_end)); return fh; } diff --git a/components/eamxx/src/share/field/field_header.hpp b/components/eamxx/src/share/field/field_header.hpp index 842842720137..fce98965e2fb 100644 --- a/components/eamxx/src/share/field/field_header.hpp +++ b/components/eamxx/src/share/field/field_header.hpp @@ -95,8 +95,7 @@ class FieldHeader : public FamilyTracking { friend std::shared_ptr create_subfield_header (const FieldIdentifier&, std::shared_ptr, - const int idim, const int k_beg, const int k_end, - const bool dynamic); + const int idim, const int k_beg, const int k_end); // NOTE: the identifier *cannot* be a shared_ptr, b/c we // don't foresee sharing an identifier between two @@ -178,8 +177,7 @@ create_subfield_header (const FieldIdentifier& id, std::shared_ptr create_subfield_header (const FieldIdentifier& id, std::shared_ptr parent, - const int idim, const int k_beg, const int k_end, - const bool dynamic); + const int idim, const int k_beg, const int k_end); } // namespace scream diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp index 5c649f9a272c..b68e62b4fddd 100644 --- a/components/eamxx/src/share/tests/subfield_tests.cpp +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -1,73 +1,59 @@ #include #include -#include "ekat/kokkos/ekat_subview_utils.hpp" -#include "share/field/field_identifier.hpp" -#include "share/field/field_header.hpp" #include "share/field/field.hpp" -#include "share/field/field_manager.hpp" #include "share/field/field_utils.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/grid/point_grid.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/ekat_pack_utils.hpp" #include "ekat/util/ekat_test_utils.hpp" namespace { - TEST_CASE("field", "") { using namespace scream; using namespace ShortFieldTagsNames; using namespace ekat::units; - auto engine = setup_random_test (); + auto engine = setup_random_test(); using RPDF = std::uniform_real_distribution; - RPDF pdf(0.01,0.99); - - std::vector tags = {COL,LEV}; - std::vector dims = {3,24}; - - FieldIdentifier fid ("field_1", {tags,dims}, m/s,"some_grid"); + RPDF pdf(0.01, 0.99); // Subfields - SECTION ("subfield") { - std::vector t1 = {COL,CMP,CMP,LEV}; - std::vector d1 = {3,10,2,24}; + SECTION("subfield") { + std::vector t1 = {COL, CMP, CMP, LEV}; + std::vector d1 = {3, 10, 2, 24}; - FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); + FieldIdentifier fid1("4d", {t1, d1}, m / s, "some_grid"); Field f1(fid1); f1.allocate_view(); - randomize(f1,engine,pdf); + randomize(f1, engine, pdf); const int idim = 1; const int ivar = 2; - auto f2 = f1.subfield(idim,ivar); + auto f2 = f1.subfield(idim, ivar); // Wrong rank for the subfield f2 REQUIRE_THROWS(f2.get_view()); - auto v4d_h = f1.get_view(); - auto v3d_h = f2.get_view(); - for (int i=0; i(); + auto v3d_h = f2.get_view(); + for (int i = 0; i < d1[0]; ++i) + for (int j = 0; j < d1[2]; ++j) + for (int k = 0; k < d1[3]; ++k) { + REQUIRE(v4d_h(i, ivar, j, k) == v3d_h(i, j, k)); } } - SECTION ("multi-sliced subfield") { - SECTION ("1D multi-slice") { + SECTION("multi-sliced subfield") { + SECTION("1D multi-slice") { // ============== /* Rank-1 view */ // ============== std::vector t1 = {COL}; std::vector d1 = {11}; - FieldIdentifier fid1("1d", {t1, d1}, m/s, "some_grid"); + FieldIdentifier fid1("1d", {t1, d1}, m / s, "some_grid"); Field f1(fid1); f1.allocate_view(); @@ -81,14 +67,17 @@ TEST_CASE("field", "") { auto sf = f1.subfield(idim, sl_beg, sl_end); auto sv_h = sf.get_multi_sliced_view(); + REQUIRE(sv_h.extent_int(idim) == (sl_end - sl_beg)); + for (int i = sl_beg; i < sl_end; i++) { REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); } } - SECTION ("2D multi-slice") { + + SECTION("2D multi-slice") { std::vector t2 = {COL, CMP}; std::vector d2 = {5, 10}; - FieldIdentifier fid2("2d", {t2, d2}, m/s, "some_grid"); + FieldIdentifier fid2("2d", {t2, d2}, m / s, "some_grid"); Field f2(fid2); f2.allocate_view(); @@ -115,13 +104,13 @@ TEST_CASE("field", "") { } } } - SECTION ("3D multi-slice") { + SECTION("3D multi-slice") { // ============== /* Rank-3 view */ // ============== std::vector t3 = {COL, CMP, LEV}; std::vector d3 = {5, 10, 2}; - FieldIdentifier fid3("3d", {t3, d3}, m/s, "some_grid"); + FieldIdentifier fid3("3d", {t3, d3}, m / s, "some_grid"); Field f3(fid3); f3.allocate_view(); @@ -152,9 +141,9 @@ TEST_CASE("field", "") { } } // ====================== - /* get_component test */ + /* get_components test */ // ====================== - auto cmp3 = f3.get_component(sl_beg[1], sl_end[1]); + auto cmp3 = f3.get_components(sl_beg[1], sl_end[1]); auto svc_h = cmp3.get_multi_sliced_view(); for (int i = 0; i < d3[0]; i++) { @@ -164,15 +153,14 @@ TEST_CASE("field", "") { } } } - } - SECTION ("4D multi-slice") { + SECTION("4D multi-slice") { // ============== /* Rank-4 view */ // ============== std::vector t4 = {COL, CMP, CMP, LEV}; std::vector d4 = {5, 10, 2, 23}; - FieldIdentifier fid4("4d", {t4, d4}, m/s, "some_grid"); + FieldIdentifier fid4("4d", {t4, d4}, m / s, "some_grid"); Field f4(fid4); f4.allocate_view(); @@ -200,20 +188,21 @@ TEST_CASE("field", "") { for (int j = j1; j < j2; j++) { for (int k = k1; k < k2; k++) { for (int l = l1; l < l2; l++) { - REQUIRE(v4d_h(i, j, k, l) == sv_h(i - i1, j - j1, k - k1, l - l1)); + REQUIRE(v4d_h(i, j, k, l) == + sv_h(i - i1, j - j1, k - k1, l - l1)); } } } } } } - SECTION ("5D multi-slice") { + SECTION("5D multi-slice") { // ============== /* Rank-5 view */ // ============== std::vector t5 = {EL, CMP, GP, GP, LEV}; std::vector d5 = {5, 10, 4, 2, 23}; - FieldIdentifier fid5("5d", {t5, d5}, m/s, "some_grid"); + FieldIdentifier fid5("5d", {t5, d5}, m / s, "some_grid"); Field f5(fid5); f5.allocate_view(); @@ -244,8 +233,8 @@ TEST_CASE("field", "") { for (int k = k1; k < k2; k++) { for (int l = l1; l < l2; l++) { for (int m = m1; m < m2; m++) { - REQUIRE(v5d_h(i, j, k, l, m) == sv_h(i - i1, j - j1, k - k1, - l - l1, m - m1)); + REQUIRE(v5d_h(i, j, k, l, m) == + sv_h(i - i1, j - j1, k - k1, l - l1, m - m1)); } } } @@ -253,13 +242,13 @@ TEST_CASE("field", "") { } } } - SECTION ("6D multi-slice") { + SECTION("6D multi-slice") { // ============== /* Rank-6 view */ // ============== std::vector t6 = {EL, TL, CMP, GP, GP, LEV}; std::vector d6 = {5, 10, 4, 2, 9, 23}; - FieldIdentifier fid6("6d", {t6, d6}, m/s, "some_grid"); + FieldIdentifier fid6("6d", {t6, d6}, m / s, "some_grid"); Field f6(fid6); f6.allocate_view(); @@ -293,9 +282,15 @@ TEST_CASE("field", "") { for (int l = l1; l < l2; l++) { for (int m = m1; m < m2; m++) { for (int n = n1; n < n2; n++) { - REQUIRE(v6d_h(i, j, k, l, m, n) == - sv_h(i - i1, j - j1, k - k1, - l - l1, m - m1, n - n1)); + REQUIRE(v6d_h(i, j, k, l, m, n) == sv_h(i - i1, j - j1, + k - k1, l - l1, + m - m1, n - n1)); + REQUIRE((sv_h.extent_int(0) == (i2 - i1) && + sv_h.extent_int(1) == (j2 - j1) && + sv_h.extent_int(2) == (k2 - k1) && + sv_h.extent_int(3) == (l2 - l1) && + sv_h.extent_int(4) == (m2 - m1) && + sv_h.extent_int(5) == (n2 - n1))); } } } @@ -305,23 +300,22 @@ TEST_CASE("field", "") { } } } - // Dynamic Subfields - SECTION ("dynamic_subfield") { + SECTION("dynamic_subfield") { const int vec_dim = 10; - std::vector t1 = {COL,CMP,CMP,LEV}; - std::vector d1 = {3,vec_dim,2,24}; + std::vector t1 = {COL, CMP, CMP, LEV}; + std::vector d1 = {3, vec_dim, 2, 24}; - FieldIdentifier fid1("4d",{t1,d1},m/s,"some_grid"); + FieldIdentifier fid1("4d", {t1, d1}, m / s, "some_grid"); Field f1(fid1); f1.allocate_view(); - randomize(f1,engine,pdf); + randomize(f1, engine, pdf); const int idim = 1; const int ivar = 0; - auto f2 = f1.subfield(idim,ivar,/* dynamic = */ true); + auto f2 = f1.subfield(idim, ivar, /* dynamic = */ true); // Cannot reset subview idx of non-subfield fields REQUIRE_THROWS(f1.get_header().get_alloc_properties().reset_subview_idx(0)); @@ -331,29 +325,30 @@ TEST_CASE("field", "") { REQUIRE_THROWS(f2_ap.reset_subview_idx(-1)); REQUIRE_THROWS(f2_ap.reset_subview_idx(vec_dim)); - // Fill f1 with random numbers, and verify corresponding subviews get same values - randomize(f1,engine,pdf); + // Fill f1 with random numbers, and verify corresponding subviews get same + // values + randomize(f1, engine, pdf); - for (int ivar_dyn=0; ivar_dyn(); - auto v3d_h = f2.get_view(); - for (int i=0; i(); + auto v3d_h = f2.get_view(); + for (int i = 0; i < d1[0]; ++i) + for (int j = 0; j < d1[2]; ++j) + for (int k = 0; k < d1[3]; ++k) { + REQUIRE(v4d_h(i, ivar_dyn, j, k) == v3d_h(i, j, k)); } } } - SECTION ("vector_component") { - std::vector tags_2 = {COL,CMP,LEV}; - std::vector dims_2 = {3,2,24}; + SECTION("vector_component") { + std::vector tags_2 = {COL, CMP, LEV}; + std::vector dims_2 = {3, 2, 24}; - FieldIdentifier fid_2("vec_3d",{tags_2,dims_2},m/s,"some_grid"); + FieldIdentifier fid_2("vec_3d", {tags_2, dims_2}, m / s, "some_grid"); Field f_vec(fid_2); f_vec.allocate_view(); @@ -372,18 +367,13 @@ TEST_CASE("field", "") { f_vec.sync_to_host(); - auto v = f_vec.get_view(); - for (int col=0; col<3; ++col) { - for (int lev=0; lev<24; ++lev) { - REQUIRE (v(col,0,lev)==1.0); - REQUIRE (v(col,1,lev)==2.0); + auto v = f_vec.get_view(); + for (int col = 0; col < 3; ++col) { + for (int lev = 0; lev < 24; ++lev) { + REQUIRE(v(col, 0, lev) == 1.0); + REQUIRE(v(col, 1, lev) == 2.0); } } } } - - - - - } // anonymous namespace From 8c9384d74085684ee8929700a1fe7b6fd0b3130b Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 24 Apr 2024 13:10:13 -0600 Subject: [PATCH 222/476] wip on unifying subview --- .../eamxx/src/share/field/field_impl.hpp | 194 ++++++++++-------- .../eamxx/src/share/tests/subfield_tests.cpp | 54 ++--- 2 files changed, 136 insertions(+), 112 deletions(-) diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 91c196bc465c..c2fdce5d5cf5 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -141,117 +141,141 @@ auto Field::get_view () const return DstView(view_ND); } -template -auto Field::get_strided_view () const - -> get_strided_view_type -{ +template +auto Field::get_strided_view() const -> +get_strided_view_type { // The destination view type on correct mem space - using DstView = get_strided_view_type; + using DstView = get_strided_view_type; // The dst value types using DstValueType = typename DstView::traits::value_type; // We only allow to reshape to a view of the correct rank constexpr int DstRank = DstView::rank; - constexpr int DstRankDynamic= DstView::rank_dynamic; + constexpr int DstRankDynamic = DstView::rank_dynamic; // Get src details const auto& alloc_prop = m_header->get_alloc_properties(); const auto& fl = m_header->get_identifier().get_layout(); // Checks - EKAT_REQUIRE_MSG (DstRank==1 && fl.rank()==1, - "Error! Strided view only available for rank-1 fields.\n"); - EKAT_REQUIRE_MSG (DstRankDynamic==1, - "Error! Strided view not allowed with compile-time dimensions.\n"); - EKAT_REQUIRE_MSG(is_allocated(), + EKAT_REQUIRE_MSG( + is_allocated(), "Error! Cannot extract a field's view before allocation happens.\n"); - EKAT_REQUIRE_MSG (not m_is_read_only || std::is_const::value, - "Error! Cannot get a view to non-const data if the field is read-only.\n"); + EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, + "Error! Cannot get a view to non-const data if the field is " + "read-only.\n"); EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), - "Error! Source field allocation is not compatible with the requested value type.\n"); + "Error! Source field allocation is not compatible with the " + "requested value type.\n"); // Check if this field is a subview of another field const auto parent = m_header->get_parent().lock(); - if (parent!=nullptr) { + if (parent != nullptr) { // Parent field has correct layout to reinterpret the view into N+1-dim view - // So create the parent field on the fly, use it to get the N+1-dim view, then subview it. - // NOTE: we can set protected members, since f is the same type of this class. + // So create the parent field on the fly, use it to get the N+1-dim view, + // then subview it. NOTE: we can set protected members, since f is the same + // type of this class. Field f; f.m_header = parent; - f.m_data = m_data; - - // Take 2 dimensional view with normal LayoutRight - auto v_np1 = f.get_ND_view(); + f.m_data = m_data; - // Now we can subview v_np1 at the correct slice - const auto& info = m_header->get_alloc_properties().get_subview_info(); + // get subview info to determine whether we are single- or multi-slicing + const auto& sv_alloc_prop = m_header->get_alloc_properties(); + const auto& info = sv_alloc_prop.get_subview_info(); const int idim = info.dim_idx; - const int k = info.slice_idx; - - // So far we can only subview at first or second dimension. - EKAT_REQUIRE_MSG (idim==0 || idim==1, - "Error! Subview dimension index is out of bounds.\n"); - - // Use correct subview utility - if (idim==0) { - return DstView(ekat::subview(v_np1,k)); - } else { - return DstView(ekat::subview_1(v_np1,k)); + const int k = info.slice_idx; + const int k_end = info.slice_idx_end; + + std::cout << "k_end = " << k_end << "\n"; + std::cout << "sv_alloc_prop.contiguous() = " << sv_alloc_prop.contiguous() << "\n"; + + if (k_end == -1) { + EKAT_REQUIRE_MSG( + DstRank == 1 && fl.rank() == 1, + "Error! Single-slice, strided view requires destination subview" + " rank to be equal to parent field rank.\n"); + EKAT_REQUIRE_MSG( + DstRankDynamic == 1, + "Error! Strided view not allowed with compile-time dimensions.\n"); + + // Take 2 dimensional view with normal LayoutRight + auto v_np1 = f.get_ND_view(); + + // As of now, we can only single-slice subview at first or second dimension. + EKAT_REQUIRE_MSG(idim == 0 || idim == 1, + "Error! Subview dimension index is out of bounds.\n"); + + // Use correct subview utility + if (idim == 0) { + return DstView(ekat::subview(v_np1, k)); + } else { + return DstView(ekat::subview_1(v_np1, k)); + } + // k_end has been set, so we're multi-slicing, and not contiguous, so we + } else if (k_end > 0) { + EKAT_REQUIRE_MSG(DstRank == fl.rank(), + "Error! Destination view rank must be equal to parent " + "field's rank for multi-sliced subview.\n"); + auto v_fullsize = f.get_ND_view(); + + return DstView(ekat::subview( + v_fullsize, Kokkos::make_pair(k, k_end), idim)); } } - // Not a subfield, so stride=1, and we can create the strided view from the LayoutRight 1d view. - return DstView(get_ND_view()); + // Not a subfield, so stride=1, and we can create the strided view from the + // LayoutRight 1d view. + return DstView(get_ND_view()); } -// NOTE: multi-slicing a view is only supported for strided view return type -// TODO: N doesn't actually need to be passed in, since the subview must have -// the same rank as its parent -template -auto Field::get_multi_sliced_view() const - -> get_strided_view_type, HD> { - // The destination view type on correct mem space - using DstView = get_strided_view_type, HD>; - // The dst value types - using DstValueType = typename DstView::traits::value_type; - - // Get src details - const auto& alloc_prop = m_header->get_alloc_properties(); - const auto& fl = m_header->get_identifier().get_layout(); - - // Checks - EKAT_REQUIRE_MSG(N == fl.rank(), "Error! Input Rank must be equal to parent " - "view's rank for multi-sliced subview.\n"); - EKAT_REQUIRE_MSG( - is_allocated(), - "Error! Cannot extract a field's view before allocation happens.\n"); - EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, - "Error! Cannot get a view to non-const data if the field is " - "read-only.\n"); - EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), - "Error! Source field allocation is not compatible with the " - "requested value type.\n"); - - // get parent header and check if this field is a subview of another field - const auto parent = m_header->get_parent().lock(); - EKAT_REQUIRE_MSG( - parent != nullptr, - "Error! Multi-sliced subview is unavailable for non-subfields.\n"); - Field f; - // create new field with the header/data from the parent - f.m_header = parent; - f.m_data = m_data; - - auto v_fullsize = f.get_ND_view(); - - // Now we can subview v_np1 at the correct slice - const auto& info = m_header->get_alloc_properties().get_subview_info(); - const int idim = info.dim_idx; - const int k_beg = info.slice_idx; - const int k_end = info.slice_idx_end; - - return DstView(ekat::subview( - v_fullsize, Kokkos::make_pair(k_beg, k_end), idim)); -} +// // NOTE: multi-slicing a view is only supported for strided view return type +// // TODO: N doesn't actually need to be passed in, since the subview must have +// // the same rank as its parent +// template +// auto Field::get_multi_sliced_view() const +// -> get_strided_view_type, HD> { +// // The destination view type on correct mem space +// using DstView = get_strided_view_type, HD>; +// // The dst value types +// using DstValueType = typename DstView::traits::value_type; + +// // Get src details +// const auto& alloc_prop = m_header->get_alloc_properties(); +// const auto& fl = m_header->get_identifier().get_layout(); + +// // Checks +// EKAT_REQUIRE_MSG(N == fl.rank(), "Error! Input Rank must be equal to parent " +// "view's rank for multi-sliced subview.\n"); +// EKAT_REQUIRE_MSG( +// is_allocated(), +// "Error! Cannot extract a field's view before allocation happens.\n"); +// EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, +// "Error! Cannot get a view to non-const data if the field is " +// "read-only.\n"); +// EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), +// "Error! Source field allocation is not compatible with the " +// "requested value type.\n"); + +// // get parent header and check if this field is a subview of another field +// const auto parent = m_header->get_parent().lock(); +// EKAT_REQUIRE_MSG( +// parent != nullptr, +// "Error! Multi-sliced subview is unavailable for non-subfields.\n"); +// Field f; +// // create new field with the header/data from the parent +// f.m_header = parent; +// f.m_data = m_data; + +// auto v_fullsize = f.get_ND_view(); + +// // Now we can subview v_np1 at the correct slice +// const auto& info = m_header->get_alloc_properties().get_subview_info(); +// const int idim = info.dim_idx; +// const int k_beg = info.slice_idx; +// const int k_end = info.slice_idx_end; + +// return DstView(ekat::subview( +// v_fullsize, Kokkos::make_pair(k_beg, k_end), idim)); +// } template void Field:: diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp index b68e62b4fddd..a20a945f6516 100644 --- a/components/eamxx/src/share/tests/subfield_tests.cpp +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -47,32 +47,32 @@ TEST_CASE("field", "") { } SECTION("multi-sliced subfield") { - SECTION("1D multi-slice") { - // ============== - /* Rank-1 view */ - // ============== - std::vector t1 = {COL}; - std::vector d1 = {11}; - FieldIdentifier fid1("1d", {t1, d1}, m / s, "some_grid"); + // SECTION("1D multi-slice") { + // // ============== + // /* Rank-1 view */ + // // ============== + // std::vector t1 = {COL}; + // std::vector d1 = {11}; + // FieldIdentifier fid1("1d", {t1, d1}, m / s, "some_grid"); - Field f1(fid1); - f1.allocate_view(); - randomize(f1, engine, pdf); + // Field f1(fid1); + // f1.allocate_view(); + // randomize(f1, engine, pdf); - const int idim = {0}; - const int sl_beg = {3}; - const int sl_end = {7}; + // const int idim = {0}; + // const int sl_beg = {3}; + // const int sl_end = {7}; - auto v1d_h = f1.get_view(); - auto sf = f1.subfield(idim, sl_beg, sl_end); - auto sv_h = sf.get_multi_sliced_view(); + // auto v1d_h = f1.get_view(); + // auto sf = f1.subfield(idim, sl_beg, sl_end); + // auto sv_h = sf.get_strided_view(); - REQUIRE(sv_h.extent_int(idim) == (sl_end - sl_beg)); + // REQUIRE(sv_h.extent_int(idim) == (sl_end - sl_beg)); - for (int i = sl_beg; i < sl_end; i++) { - REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); - } - } + // for (int i = sl_beg; i < sl_end; i++) { + // REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); + // } + // } SECTION("2D multi-slice") { std::vector t2 = {COL, CMP}; @@ -92,7 +92,7 @@ TEST_CASE("field", "") { for (int ens = 0; ens < 2; ens++) { auto sf = f2.subfield(idim[ens], sl_beg[ens], sl_end[ens]); - auto sv_h = sf.get_multi_sliced_view(); + auto sv_h = sf.get_strided_view(); i1 = (ens == 0) ? sl_beg[0] : 0; i2 = (ens == 0) ? sl_end[0] : d2[0]; j1 = (ens == 1) ? sl_beg[1] : 0; @@ -125,7 +125,7 @@ TEST_CASE("field", "") { for (int ens = 0; ens < 3; ens++) { auto sf = f3.subfield(idim[ens], sl_beg[ens], sl_end[ens]); - auto sv_h = sf.get_multi_sliced_view(); + auto sv_h = sf.get_strided_view(); i1 = (ens == 0) ? sl_beg[0] : 0; i2 = (ens == 0) ? sl_end[0] : d3[0]; j1 = (ens == 1) ? sl_beg[1] : 0; @@ -145,7 +145,7 @@ TEST_CASE("field", "") { // ====================== auto cmp3 = f3.get_components(sl_beg[1], sl_end[1]); - auto svc_h = cmp3.get_multi_sliced_view(); + auto svc_h = cmp3.get_strided_view(); for (int i = 0; i < d3[0]; i++) { for (int j = sl_beg[1]; j < sl_end[1]; j++) { for (int k = 0; k < d3[2]; k++) { @@ -175,7 +175,7 @@ TEST_CASE("field", "") { for (int ens = 0; ens < 4; ens++) { auto sf = f4.subfield(idim[ens], sl_beg[ens], sl_end[ens]); - auto sv_h = sf.get_multi_sliced_view(); + auto sv_h = sf.get_strided_view(); i1 = (ens == 0) ? sl_beg[0] : 0; i2 = (ens == 0) ? sl_end[0] : d4[0]; j1 = (ens == 1) ? sl_beg[1] : 0; @@ -217,7 +217,7 @@ TEST_CASE("field", "") { for (int ens = 0; ens < 5; ens++) { auto sf = f5.subfield(idim[ens], sl_beg[ens], sl_end[ens]); - auto sv_h = sf.get_multi_sliced_view(); + auto sv_h = sf.get_strided_view(); i1 = (ens == 0) ? sl_beg[0] : 0; i2 = (ens == 0) ? sl_end[0] : d5[0]; j1 = (ens == 1) ? sl_beg[1] : 0; @@ -263,7 +263,7 @@ TEST_CASE("field", "") { for (int ens = 0; ens < 6; ens++) { auto sf = f6.subfield(idim[ens], sl_beg[ens], sl_end[ens]); - auto sv_h = sf.get_multi_sliced_view(); + auto sv_h = sf.get_strided_view(); i1 = (ens == 0) ? sl_beg[0] : 0; i2 = (ens == 0) ? sl_end[0] : d6[0]; j1 = (ens == 1) ? sl_beg[1] : 0; From 950e19b7ff7adb34a1e774cdfb400a7185a7600f Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 25 Apr 2024 13:18:38 -0600 Subject: [PATCH 223/476] unify multi-slice subviews with single-slice under get_strided_view() --- components/eamxx/src/share/field/field.hpp | 25 +++-- .../eamxx/src/share/field/field_impl.hpp | 96 +++++++------------ .../src/share/field/field_utils_impl.hpp | 33 ++++--- .../eamxx/src/share/tests/subfield_tests.cpp | 55 +++++------ externals/ekat | 2 +- 5 files changed, 101 insertions(+), 110 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index f4ad81997265..5b3935fba79f 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -318,25 +318,38 @@ class Field { if_t<(N<=2), get_view_type,HD>> get_subview_1 (const get_view_type,HD>&, const int) const { - EKAT_ERROR_MSG ("Error! Cannot subview a rank2 view along the second dimension without losing LayoutRight.\n"); + EKAT_ERROR_MSG ("Error! Cannot subview a rank2 view along the second " + "dimension without losing LayoutRight.\n"); } template auto get_ND_view () const - -> if_t,HD>>; + -> if_t<((N < MaxRank) and (N > 0)), get_view_type,HD>>; template auto get_ND_view () const - -> if_t<(N,HD>>; + -> if_t,HD>>; + + // NOTE: DO NOT USE--this circumvents compile-time issues with + // subview slicing in get_strided_view() + template + auto get_ND_view () const + -> if_t,HD>>; + + // NOTE: DO NOT USE--it only returns an error and is here to protect + // against compiler errors related to sliced subviews in get_strided_view() + template + auto get_ND_view () const + -> if_t<(N >= MaxRank + 1), get_view_type,HD>>; // Metadata (name, rank, dims, customer/providers, time stamp, ...) - std::shared_ptr m_header; + std::shared_ptr m_header; // Actual data. - dual_view_t m_data; + dual_view_t m_data; // Whether this field is read-only - bool m_is_read_only = false; + bool m_is_read_only = false; }; // We use this to find a Field in a std container. diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index c2fdce5d5cf5..f93a49c809e6 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -90,7 +90,8 @@ auto Field::get_view () const using DstView = get_view_type; // The dst value types using DstValueType = typename DstView::traits::value_type; - // The ViewDimension object from the Dst View (used to check validity of possible compile-time extents) + // The ViewDimension object from the Dst View (used to check validity of + // possible compile-time extents) using dims_type = typename DstView::traits::dimension; // We only allow to reshape to a view of the correct rank constexpr int DstRank = DstView::rank; @@ -185,9 +186,7 @@ get_strided_view_type { const int k = info.slice_idx; const int k_end = info.slice_idx_end; - std::cout << "k_end = " << k_end << "\n"; - std::cout << "sv_alloc_prop.contiguous() = " << sv_alloc_prop.contiguous() << "\n"; - + // k_end has not been set by a multi-slice subfield function if (k_end == -1) { EKAT_REQUIRE_MSG( DstRank == 1 && fl.rank() == 1, @@ -197,8 +196,9 @@ get_strided_view_type { DstRankDynamic == 1, "Error! Strided view not allowed with compile-time dimensions.\n"); - // Take 2 dimensional view with normal LayoutRight - auto v_np1 = f.get_ND_view(); + // Take an (n + 1)-dimensional == DstRank (== 2D, in practice) view + // with normal LayoutRight + auto v_np1 = f.get_ND_view(); // As of now, we can only single-slice subview at first or second dimension. EKAT_REQUIRE_MSG(idim == 0 || idim == 1, @@ -210,7 +210,7 @@ get_strided_view_type { } else { return DstView(ekat::subview_1(v_np1, k)); } - // k_end has been set, so we're multi-slicing, and not contiguous, so we + // k_end has been set, so we're multi-slicing } else if (k_end > 0) { EKAT_REQUIRE_MSG(DstRank == fl.rank(), "Error! Destination view rank must be equal to parent " @@ -221,62 +221,11 @@ get_strided_view_type { v_fullsize, Kokkos::make_pair(k, k_end), idim)); } } - - // Not a subfield, so stride=1, and we can create the strided view from the + // Not a subfield, so stride == 1, and we can create the strided view from the // LayoutRight 1d view. - return DstView(get_ND_view()); + return DstView(get_ND_view()); } -// // NOTE: multi-slicing a view is only supported for strided view return type -// // TODO: N doesn't actually need to be passed in, since the subview must have -// // the same rank as its parent -// template -// auto Field::get_multi_sliced_view() const -// -> get_strided_view_type, HD> { -// // The destination view type on correct mem space -// using DstView = get_strided_view_type, HD>; -// // The dst value types -// using DstValueType = typename DstView::traits::value_type; - -// // Get src details -// const auto& alloc_prop = m_header->get_alloc_properties(); -// const auto& fl = m_header->get_identifier().get_layout(); - -// // Checks -// EKAT_REQUIRE_MSG(N == fl.rank(), "Error! Input Rank must be equal to parent " -// "view's rank for multi-sliced subview.\n"); -// EKAT_REQUIRE_MSG( -// is_allocated(), -// "Error! Cannot extract a field's view before allocation happens.\n"); -// EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, -// "Error! Cannot get a view to non-const data if the field is " -// "read-only.\n"); -// EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), -// "Error! Source field allocation is not compatible with the " -// "requested value type.\n"); - -// // get parent header and check if this field is a subview of another field -// const auto parent = m_header->get_parent().lock(); -// EKAT_REQUIRE_MSG( -// parent != nullptr, -// "Error! Multi-sliced subview is unavailable for non-subfields.\n"); -// Field f; -// // create new field with the header/data from the parent -// f.m_header = parent; -// f.m_data = m_data; - -// auto v_fullsize = f.get_ND_view(); - -// // Now we can subview v_np1 at the correct slice -// const auto& info = m_header->get_alloc_properties().get_subview_info(); -// const int idim = info.dim_idx; -// const int k_beg = info.slice_idx; -// const int k_end = info.slice_idx_end; - -// return DstView(ekat::subview( -// v_fullsize, Kokkos::make_pair(k_beg, k_end), idim)); -// } - template void Field:: deep_copy (const Field& src) { @@ -343,7 +292,7 @@ deep_copy_impl (const Field& src) { const auto& layout_src = src.get_header().get_identifier().get_layout(); EKAT_REQUIRE_MSG(layout==layout_src, "ERROR: Unable to copy field " + src.get_header().get_identifier().name() + - " to field " + get_header().get_identifier().name() + ". Layouts don't match."); + " to field " + get_header().get_identifier().name() + ". Layouts don't match.\n"); const auto rank = layout.rank(); // For rank 0 view, we only need to copy a single value and return @@ -778,7 +727,7 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) template auto Field::get_ND_view () const -> - if_t<(N,HD>> + if_t<((N < MaxRank) and (N > 0)),get_view_type,HD>> { const auto& fl = m_header->get_identifier().get_layout(); EKAT_REQUIRE_MSG (N==1 || N==fl.rank(), @@ -863,6 +812,29 @@ auto Field::get_ND_view () const -> return ret_type (ptr,kl); } +// NOTE: DO NOT USE--this circumvents compile-time issues with +// subview slicing in get_strided_view() +template +auto Field::get_ND_view () const -> + if_t,HD>> +{ + EKAT_ERROR_MSG("Error! Cannot call get_ND_view for rank == 0 (scalar).\n" + "This should never be called at run time.\n" + "Please contact developer if this functionality is required\n"); +} + +// NOTE: DO NOT USE--this circumvents compile-time issues with +// subview slicing in get_strided_view() +template +auto Field::get_ND_view () const -> + if_t<(N >= MaxRank + 1),get_view_type,HD>> +{ + EKAT_ERROR_MSG("Error! Cannot call get_ND_view for rank greater than " + "MaxRank = 6.\n" + "This should never be called at run time.\n" + "Please contact developer if this functionality is required\n"); +} + } // namespace scream #endif // SCREAM_FIELD_IMPL_HPP diff --git a/components/eamxx/src/share/field/field_utils_impl.hpp b/components/eamxx/src/share/field/field_utils_impl.hpp index 222ff1bb49ed..d6ed0b13d44d 100644 --- a/components/eamxx/src/share/field/field_utils_impl.hpp +++ b/components/eamxx/src/share/field/field_utils_impl.hpp @@ -29,6 +29,8 @@ bool views_are_equal(const Field& f1, const Field& f2, const ekat::Comm* comm) f2.sync_to_host(); // Reshape based on the rank, then loop over all entries. + // NOTE: because views_are_equal() is only used for testing, we generalize by + // always calling get_strided_view(), even if it could be contiguous bool same_locally = true; const auto& dims = l1.dims(); switch (l1.rank()) { @@ -46,8 +48,8 @@ bool views_are_equal(const Field& f1, const Field& f2, const ekat::Comm* comm) break; case 2: { - auto v1 = f1.template get_view(); - auto v2 = f2.template get_view(); + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); for (int i=0; same_locally && i(); - auto v2 = f2.template get_view(); + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); for (int i=0; same_locally && i(); - auto v2 = f2.template get_view(); + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); for (int i=0; same_locally && i(); - auto v2 = f2.template get_view(); + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); for (int i=0; same_locally && i(); - auto v2 = f2.template get_view(); + auto v1 = f1.template get_strided_view(); + auto v2 = f2.template get_strided_view(); for (int i=0; same_locally && i()() << ", \n"; + out << "\n " << f.get_strided_view()() << ", \n"; break; } case 1: @@ -769,7 +774,7 @@ void print_field_hyperslab (const Field& f, case 2: { dims_str[dims_left[1]] = ":"; - auto v = f.get_view(); + auto v = f.get_strided_view(); for (int i=0; i(); + auto v = f.get_strided_view(); for (int i=0; i(); + auto v = f.get_strided_view(); for (int i=0; i t1 = {COL}; - // std::vector d1 = {11}; - // FieldIdentifier fid1("1d", {t1, d1}, m / s, "some_grid"); - - // Field f1(fid1); - // f1.allocate_view(); - // randomize(f1, engine, pdf); - - // const int idim = {0}; - // const int sl_beg = {3}; - // const int sl_end = {7}; - - // auto v1d_h = f1.get_view(); - // auto sf = f1.subfield(idim, sl_beg, sl_end); - // auto sv_h = sf.get_strided_view(); - - // REQUIRE(sv_h.extent_int(idim) == (sl_end - sl_beg)); - - // for (int i = sl_beg; i < sl_end; i++) { - // REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); - // } - // } + SECTION("1D multi-slice") { + // ============== + /* Rank-1 view */ + // ============== + std::vector t1 = {COL}; + std::vector d1 = {11}; + FieldIdentifier fid1("1d", {t1, d1}, m / s, "some_grid"); + + Field f1(fid1); + f1.allocate_view(); + randomize(f1, engine, pdf); + + const int idim = {0}; + const int sl_beg = {3}; + const int sl_end = {7}; + + auto v1d_h = f1.get_view(); + auto sf = f1.subfield(idim, sl_beg, sl_end); + REQUIRE(sf.get_header().get_alloc_properties().contiguous() == false); + auto sv_h = sf.get_strided_view(); + REQUIRE(sv_h.extent_int(idim) == (sl_end - sl_beg)); + + for (int i = sl_beg; i < sl_end; i++) { + REQUIRE(v1d_h(i) == sv_h(i - sl_beg)); + } + } SECTION("2D multi-slice") { std::vector t2 = {COL, CMP}; @@ -145,7 +145,7 @@ TEST_CASE("field", "") { // ====================== auto cmp3 = f3.get_components(sl_beg[1], sl_end[1]); - auto svc_h = cmp3.get_strided_view(); + auto svc_h = cmp3.get_strided_view(); for (int i = 0; i < d3[0]; i++) { for (int j = sl_beg[1]; j < sl_end[1]; j++) { for (int k = 0; k < d3[2]; k++) { @@ -263,6 +263,7 @@ TEST_CASE("field", "") { for (int ens = 0; ens < 6; ens++) { auto sf = f6.subfield(idim[ens], sl_beg[ens], sl_end[ens]); + REQUIRE(sf.get_header().get_alloc_properties().contiguous() == false); auto sv_h = sf.get_strided_view(); i1 = (ens == 0) ? sl_beg[0] : 0; i2 = (ens == 0) ? sl_end[0] : d6[0]; diff --git a/externals/ekat b/externals/ekat index eed9ddbefe68..bb3ebcdfd5f6 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit eed9ddbefe685ea084853fa9c9262c861a5a7f39 +Subproject commit bb3ebcdfd5f600c8e75420eaed167d9b268b100b From 51dae5b253c148731cc64d8a256e6f5bd77cdb0e Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Fri, 26 Apr 2024 00:40:01 -0600 Subject: [PATCH 224/476] fix other tests in share/ --- components/eamxx/src/share/field/field.hpp | 8 +-- .../eamxx/src/share/field/field_impl.hpp | 21 +----- .../src/share/field/field_utils_impl.hpp | 64 +++++++++---------- .../share/property_checks/field_nan_check.cpp | 14 ++-- .../eamxx/src/share/tests/field_utils.cpp | 26 ++++---- .../eamxx/src/share/tests/property_checks.cpp | 12 ++-- externals/ekat | 2 +- 7 files changed, 61 insertions(+), 86 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 5b3935fba79f..a3ed9076a2a0 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -324,18 +324,12 @@ class Field { template auto get_ND_view () const - -> if_t<((N < MaxRank) and (N > 0)), get_view_type,HD>>; + -> if_t<(N < MaxRank), get_view_type,HD>>; template auto get_ND_view () const -> if_t,HD>>; - // NOTE: DO NOT USE--this circumvents compile-time issues with - // subview slicing in get_strided_view() - template - auto get_ND_view () const - -> if_t,HD>>; - // NOTE: DO NOT USE--it only returns an error and is here to protect // against compiler errors related to sliced subviews in get_strided_view() template diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index f93a49c809e6..fcfae128ad9c 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -188,14 +188,6 @@ get_strided_view_type { // k_end has not been set by a multi-slice subfield function if (k_end == -1) { - EKAT_REQUIRE_MSG( - DstRank == 1 && fl.rank() == 1, - "Error! Single-slice, strided view requires destination subview" - " rank to be equal to parent field rank.\n"); - EKAT_REQUIRE_MSG( - DstRankDynamic == 1, - "Error! Strided view not allowed with compile-time dimensions.\n"); - // Take an (n + 1)-dimensional == DstRank (== 2D, in practice) view // with normal LayoutRight auto v_np1 = f.get_ND_view(); @@ -727,7 +719,7 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) template auto Field::get_ND_view () const -> - if_t<((N < MaxRank) and (N > 0)),get_view_type,HD>> + if_t<(N < MaxRank), get_view_type,HD>> { const auto& fl = m_header->get_identifier().get_layout(); EKAT_REQUIRE_MSG (N==1 || N==fl.rank(), @@ -812,17 +804,6 @@ auto Field::get_ND_view () const -> return ret_type (ptr,kl); } -// NOTE: DO NOT USE--this circumvents compile-time issues with -// subview slicing in get_strided_view() -template -auto Field::get_ND_view () const -> - if_t,HD>> -{ - EKAT_ERROR_MSG("Error! Cannot call get_ND_view for rank == 0 (scalar).\n" - "This should never be called at run time.\n" - "Please contact developer if this functionality is required\n"); -} - // NOTE: DO NOT USE--this circumvents compile-time issues with // subview slicing in get_strided_view() template diff --git a/components/eamxx/src/share/field/field_utils_impl.hpp b/components/eamxx/src/share/field/field_utils_impl.hpp index d6ed0b13d44d..e12f6debbc10 100644 --- a/components/eamxx/src/share/field/field_utils_impl.hpp +++ b/components/eamxx/src/share/field/field_utils_impl.hpp @@ -141,13 +141,13 @@ void randomize (const Field& f, Engine& engine, PDF&& pdf) switch (fl.rank()) { case 0: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); v() = pdf(engine); } break; case 1: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + const auto gids = dof_gids.get_strided_view(); // Create a field to store perturbation values with layout // the same as f, but stripped of column and level dimension. @@ -296,7 +296,7 @@ ST frobenius_norm(const Field& f, const ekat::Comm* comm) switch (fl.rank()) { case 1: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); for (int i=0; i(); + auto v = f.template get_strided_view(); Kokkos::parallel_reduce(size, KOKKOS_LAMBDA(int i, int& result) { if (ekat::is_invalid(v(i))) { result = i; @@ -57,7 +57,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { break; case 2: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); Kokkos::parallel_reduce(size, KOKKOS_LAMBDA(int idx, int& result) { int i,j; unflatten_idx(idx,extents,i,j); @@ -69,7 +69,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { break; case 3: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); Kokkos::parallel_reduce(size, KOKKOS_LAMBDA(int idx, int& result) { int i,j,k; unflatten_idx(idx,extents,i,j,k); @@ -81,7 +81,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { break; case 4: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); Kokkos::parallel_reduce(size, KOKKOS_LAMBDA(int idx, int& result) { int i,j,k,l; unflatten_idx(idx,extents,i,j,k,l); @@ -93,7 +93,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { break; case 5: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); Kokkos::parallel_reduce(size, KOKKOS_LAMBDA(int idx, int& result) { int i,j,k,l,m; unflatten_idx(idx,extents,i,j,k,l,m); @@ -105,7 +105,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { break; case 6: { - auto v = f.template get_view(); + auto v = f.template get_strided_view(); Kokkos::parallel_reduce(size, KOKKOS_LAMBDA(int idx, int& result) { int i,j,k,l,m,n; unflatten_idx(idx,extents,i,j,k,l,m,n); @@ -136,7 +136,7 @@ PropertyCheck::ResultAndMsg FieldNaNCheck::check_impl() const { if (m_grid) { // We are storing grid info, and the field is over columns. Get col id and coords. col_lid = indices[0]; - auto gids = m_grid->get_dofs_gids().get_view(); + auto gids = m_grid->get_dofs_gids().get_strided_view(); res_and_msg.msg += " - indices (w/ global column index): (" + std::to_string(gids(col_lid)); for (size_t i=1; i(); - auto v2 = f2.get_view(); + auto v1 = f1.get_strided_view(); + auto v2 = f2.get_strided_view(); auto dim0 = fid.get_layout().dim(0); auto dim1 = fid.get_layout().dim(1); auto am_i_root = comm.am_i_root(); @@ -80,7 +80,7 @@ TEST_CASE("utils") { SECTION ("sum") { - auto v1 = f1.get_view(); + auto v1 = f1.get_strided_view(); auto dim0 = fid.get_layout().dim(0); auto dim1 = fid.get_layout().dim(1); auto lsize = fid.get_layout().size(); @@ -103,7 +103,7 @@ TEST_CASE("utils") { SECTION ("frobenius") { - auto v1 = f1.get_view(); + auto v1 = f1.get_strided_view(); auto dim0 = fid.get_layout().dim(0); auto dim1 = fid.get_layout().dim(1); auto lsize = fid.get_layout().size(); @@ -130,7 +130,7 @@ TEST_CASE("utils") { SECTION ("max") { - auto v1 = f1.get_view(); + auto v1 = f1.get_strided_view(); auto dim0 = fid.get_layout().dim(0); auto dim1 = fid.get_layout().dim(1); auto lsize = fid.get_layout().size(); @@ -153,7 +153,7 @@ TEST_CASE("utils") { SECTION ("min") { - auto v1 = f1.get_view(); + auto v1 = f1.get_strided_view(); auto dim0 = fid.get_layout().dim(0); auto dim1 = fid.get_layout().dim(1); auto lsize = fid.get_layout().size(); @@ -221,10 +221,10 @@ TEST_CASE("utils") { // Sync to host for checks f1.sync_to_host(), f2a.sync_to_host(), f2b.sync_to_host(), f3.sync_to_host(); - const auto v1 = f1.get_view (); - const auto v2a = f2a.get_view(); - const auto v2b = f2b.get_view(); - const auto v3 = f3.get_view (); + const auto v1 = f1.get_strided_view (); + const auto v2a = f2a.get_strided_view(); + const auto v2b = f2b.get_strided_view(); + const auto v3 = f3.get_strided_view (); // Check that all field values are 1 for all but last 3 levels and between [2,3] otherwise. auto check_level = [&] (const int ilev, const Real val) { @@ -248,8 +248,8 @@ TEST_CASE("utils") { perturb(f3_alt, engine, pdf, base_seed_alt, mask_lambda, gids); f1_alt.sync_to_host(), f3_alt.sync_to_host(); - const auto v1_alt = f1_alt.get_view(); - const auto v3_alt = f3_alt.get_view(); + const auto v1_alt = f1_alt.get_strided_view(); + const auto v3_alt = f3_alt.get_strided_view(); auto check_diff = [&] (const int ilev, const Real val1, const Real val2) { if (ilev < nlevs-3) REQUIRE(val1==val2); @@ -318,7 +318,7 @@ TEST_CASE ("print_field_hyperslab") { f.allocate_view(); randomize (f,engine,pdf); - auto v = f.get_view(); + auto v = f.get_strided_view(); SECTION ("slice_0") { std::vector loc_tags = {EL,CMP}; diff --git a/components/eamxx/src/share/tests/property_checks.cpp b/components/eamxx/src/share/tests/property_checks.cpp index 1cb5e64369ed..e0da898a8416 100644 --- a/components/eamxx/src/share/tests/property_checks.cpp +++ b/components/eamxx/src/share/tests/property_checks.cpp @@ -69,10 +69,10 @@ TEST_CASE("property_checks", "") { const auto units = ekat::units::Units::nondimensional(); const auto& lat = grid->create_geometry_data("lat",layout,units); const auto& lon = grid->create_geometry_data("lon",layout,units); - auto lat_h = lat.get_view(); - auto lon_h = lon.get_view(); + auto lat_h = lat.get_strided_view(); + auto lon_h = lon.get_strided_view(); auto dofs = grid->get_dofs_gids(); - auto dofs_h = dofs.get_view(); + auto dofs_h = dofs.get_strided_view(); for (int i=0; iget_num_local_dofs(); ++i) { lat_h(i) = i; lon_h(i) = -i; @@ -110,12 +110,12 @@ TEST_CASE("property_checks", "") { REQUIRE(res_and_msg.result==CheckResult::Pass); // Assign a NaN value to the field, make sure it fails the check, - auto f_view = f.get_view(); + auto f_view = f.get_strided_view(); f_view(1,2,3) = std::numeric_limits::quiet_NaN(); f.sync_to_dev(); res_and_msg = nan_check->check(); REQUIRE(res_and_msg.result==CheckResult::Fail); - + std::string expected_msg = "FieldNaNCheck failed.\n" " - field id: " + fid.get_id_string() + "\n" @@ -149,7 +149,7 @@ TEST_CASE("property_checks", "") { // Assign out-of-bounds values to the field, make sure it fails the check, // and then repair the field so it passes. f.deep_copy(0.5); - auto f_view = f.get_view(); + auto f_view = f.get_strided_view(); f_view(1,2,3) = 2.0; f_view(0,1,2) = 0.0; f.sync_to_dev(); diff --git a/externals/ekat b/externals/ekat index bb3ebcdfd5f6..c5c05e626648 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit bb3ebcdfd5f600c8e75420eaed167d9b268b100b +Subproject commit c5c05e626648fa62fff8d99c51457986acdbc0c4 From acf6214ddeb7b84767c720fb63048721c6ffcc02 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Tue, 21 May 2024 00:17:47 -0600 Subject: [PATCH 225/476] small changes to fix rebase --- components/eamxx/src/share/field/field.cpp | 7 +++---- components/eamxx/src/share/field/field.hpp | 6 +++++- components/eamxx/src/share/field/field_alloc_prop.cpp | 4 ++-- components/eamxx/src/share/field/field_impl.hpp | 9 ++++++++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index f56b3b83d933..e8a3ab4a0581 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -128,8 +128,8 @@ Field Field::subfield(const std::string& sf_name, is_allocated(), "Error! Input field must be allocated in order to subview it.\n"); - auto sf_layout = - lt.clone_with_different_extent(idim, index_end - index_beg); + auto sf_layout = lt.clone(); + sf_layout.reset_dim(idim, index_end - index_beg); // Create identifier for subfield FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name()); @@ -178,8 +178,7 @@ Field Field::get_components(const int i1, const int i2) { EKAT_REQUIRE_MSG(layout.is_vector_layout(), "Error! 'get_component' available only for vector fields.\n" " Layout of '" + - fname + "': " + e2str(get_layout_type(layout.tags())) + - "\n"); + fname + "': " + e2str(layout.type()) + "\n"); const int idim = layout.get_vector_component_idx(); EKAT_REQUIRE_MSG(i1 >= 0 && i2 < layout.dim(idim), diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index a3ed9076a2a0..c0a84b101309 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -324,7 +324,11 @@ class Field { template auto get_ND_view () const - -> if_t<(N < MaxRank), get_view_type,HD>>; + -> if_t, HD>>; + + template + auto get_ND_view () const + -> if_t<(N > 0) and (N < MaxRank), get_view_type,HD>>; template auto get_ND_view () const diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index 635e62bae19c..48796a826bd9 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -95,8 +95,8 @@ FieldAllocProp FieldAllocProp::subview(const int idim, FieldAllocProp props(m_scalar_type_size); props.m_committed = true; props.m_scalar_type_size = m_scalar_type_size; - props.m_layout = - m_layout.clone_with_different_extent(idim, k_end - k_beg); + props.m_layout = m_layout.clone(); + props.m_layout.reset_dim(idim, k_end - k_beg); // Output is contiguous if either // - this->m_contiguous=true AND idim==0 diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index fcfae128ad9c..97cb2c6b36ae 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -717,9 +717,16 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) Kokkos::fence(); } +template +auto get_ND_view () const + -> if_t,HD>> +{ + EKAT_ERROR_MSG("Error! Cannot take an ND view of 0 dimensional field.\n"); +} + template auto Field::get_ND_view () const -> - if_t<(N < MaxRank), get_view_type,HD>> + if_t<(N > 0) and (N < MaxRank), get_view_type, HD>> { const auto& fl = m_header->get_identifier().get_layout(); EKAT_REQUIRE_MSG (N==1 || N==fl.rank(), From ca0b2347ff53d9b363fe2d23335cb5aaba8414e6 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 22 May 2024 17:56:03 -0600 Subject: [PATCH 226/476] discard changes on ekat and point to current commit on master --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index c5c05e626648..eed9ddbefe68 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit c5c05e626648fa62fff8d99c51457986acdbc0c4 +Subproject commit eed9ddbefe685ea084853fa9c9262c861a5a7f39 From 0a18d7e123bc3185c5621110132aa77da5393af6 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Wed, 22 May 2024 18:12:48 -0600 Subject: [PATCH 227/476] update get_strided_view() approach--all appears to be working --- components/eamxx/src/share/field/field.hpp | 12 +- .../eamxx/src/share/field/field_impl.hpp | 148 +++++++++--------- 2 files changed, 74 insertions(+), 86 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index c0a84b101309..858c542fd4ab 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -136,12 +136,6 @@ class Field { get_strided_view_type get_strided_view () const; - // this is the same as above but for a multi-sliced view - // which is to say, also a strided view - template - auto get_multi_sliced_view () const - -> get_strided_view_type, HD>; - // These two getters are convenience function for commonly accessed metadata. // The same info can be extracted from the metadata stored in the FieldHeader DataType data_type () const { return get_header().get_identifier().data_type(); } @@ -324,11 +318,7 @@ class Field { template auto get_ND_view () const - -> if_t, HD>>; - - template - auto get_ND_view () const - -> if_t<(N > 0) and (N < MaxRank), get_view_type,HD>>; + -> if_t<(N < MaxRank), get_view_type,HD>>; template auto get_ND_view () const diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 97cb2c6b36ae..420d90492e72 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -151,70 +151,74 @@ get_strided_view_type { using DstValueType = typename DstView::traits::value_type; // We only allow to reshape to a view of the correct rank constexpr int DstRank = DstView::rank; - constexpr int DstRankDynamic = DstView::rank_dynamic; - // Get src details - const auto& alloc_prop = m_header->get_alloc_properties(); - const auto& fl = m_header->get_identifier().get_layout(); - - // Checks - EKAT_REQUIRE_MSG( - is_allocated(), - "Error! Cannot extract a field's view before allocation happens.\n"); - EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, - "Error! Cannot get a view to non-const data if the field is " - "read-only.\n"); - EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), - "Error! Source field allocation is not compatible with the " - "requested value type.\n"); - - // Check if this field is a subview of another field - const auto parent = m_header->get_parent().lock(); - if (parent != nullptr) { - // Parent field has correct layout to reinterpret the view into N+1-dim view - // So create the parent field on the fly, use it to get the N+1-dim view, - // then subview it. NOTE: we can set protected members, since f is the same - // type of this class. - Field f; - f.m_header = parent; - f.m_data = m_data; - - // get subview info to determine whether we are single- or multi-slicing - const auto& sv_alloc_prop = m_header->get_alloc_properties(); - const auto& info = sv_alloc_prop.get_subview_info(); - const int idim = info.dim_idx; - const int k = info.slice_idx; - const int k_end = info.slice_idx_end; - - // k_end has not been set by a multi-slice subfield function - if (k_end == -1) { - // Take an (n + 1)-dimensional == DstRank (== 2D, in practice) view - // with normal LayoutRight - auto v_np1 = f.get_ND_view(); - - // As of now, we can only single-slice subview at first or second dimension. - EKAT_REQUIRE_MSG(idim == 0 || idim == 1, - "Error! Subview dimension index is out of bounds.\n"); - - // Use correct subview utility - if (idim == 0) { - return DstView(ekat::subview(v_np1, k)); - } else { - return DstView(ekat::subview_1(v_np1, k)); + if constexpr (DstRank > 0) { + // Get src details + const auto& alloc_prop = m_header->get_alloc_properties(); + const auto& fl = m_header->get_identifier().get_layout(); + + // Checks + EKAT_REQUIRE_MSG( + is_allocated(), + "Error! Cannot extract a field's view before allocation happens.\n"); + EKAT_REQUIRE_MSG(not m_is_read_only || std::is_const::value, + "Error! Cannot get a view to non-const data if the field is " + "read-only.\n"); + EKAT_REQUIRE_MSG(alloc_prop.template is_compatible(), + "Error! Source field allocation is not compatible with the " + "requested value type.\n"); + + // Check if this field is a subview of another field + const auto parent = m_header->get_parent().lock(); + if (parent != nullptr) { + // Parent field has correct layout to reinterpret the view into N+1-dim view, + // for single-slice subfield, and N-dim view for multi-slice subfield. + // So create the parent field on the fly, use it to get the N+{1,0}-dim view, + // then subview it. NOTE: we can set protected members, since f is the same + // type of this class. + Field f; + f.m_header = parent; + f.m_data = m_data; + + // get subview info to determine whether we are single- or multi-slicing + const auto& sv_alloc_prop = m_header->get_alloc_properties(); + const auto& info = sv_alloc_prop.get_subview_info(); + const int idim = info.dim_idx; + const int k = info.slice_idx; + const int k_end = info.slice_idx_end; + + // k_end has not been set by a multi-slice subfield function + if (k_end == -1) { + // Take an (n + 1)-dimensional == DstRank (== 2D, in practice) view + // with normal LayoutRight + auto v_np1 = f.get_ND_view(); + + // As of now, we can only single-slice subview at first or second dimension. + EKAT_REQUIRE_MSG(idim == 0 || idim == 1, + "Error! Subview dimension index is out of bounds.\n"); + + // Use correct subview utility + if (idim == 0) { + return DstView(ekat::subview(v_np1, k)); + } else { + return DstView(ekat::subview_1(v_np1, k)); + } + // k_end has been set, so we're multi-slicing + } else if (k_end > 0) { + // rank doesn't change for multi-slice + EKAT_REQUIRE_MSG(DstRank == fl.rank(), + "Error! Destination view rank must be equal to parent " + "field's rank for multi-sliced subview.\n"); + auto v_fullsize = f.get_ND_view(); + + return DstView(ekat::subview( + v_fullsize, Kokkos::make_pair(k, k_end), idim)); } - // k_end has been set, so we're multi-slicing - } else if (k_end > 0) { - EKAT_REQUIRE_MSG(DstRank == fl.rank(), - "Error! Destination view rank must be equal to parent " - "field's rank for multi-sliced subview.\n"); - auto v_fullsize = f.get_ND_view(); - - return DstView(ekat::subview( - v_fullsize, Kokkos::make_pair(k, k_end), idim)); } } - // Not a subfield, so stride == 1, and we can create the strided view from the - // LayoutRight 1d view. + // Either not a subfield or requesting a zero-D view from a + // 0D or 1D subfield, so stride == 1, and we can create the + // strided view from the LayoutRight 1d view. return DstView(get_ND_view()); } @@ -717,16 +721,9 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) Kokkos::fence(); } -template -auto get_ND_view () const - -> if_t,HD>> -{ - EKAT_ERROR_MSG("Error! Cannot take an ND view of 0 dimensional field.\n"); -} - -template -auto Field::get_ND_view () const -> - if_t<(N > 0) and (N < MaxRank), get_view_type, HD>> +template +auto Field::get_ND_view () const + -> if_t<(N < MaxRank), get_view_type, HD>> { const auto& fl = m_header->get_identifier().get_layout(); EKAT_REQUIRE_MSG (N==1 || N==fl.rank(), @@ -754,7 +751,8 @@ auto Field::get_ND_view () const -> "Error! Subview dimension index is out of bounds.\n"); EKAT_REQUIRE_MSG (idim==0 || N>1, - "Error! Cannot subview a rank-2 (or less) view along 2nd dimension without losing LayoutRight.\n"); + "Error! Cannot subview a rank-2 (or less) view along 2nd dimension " + "without losing LayoutRight.\n"); // Use SFINAE-ed get_subview helper function to pick correct // subview impl. If N+1<=2 and idim!=0, the code craps out in the check above. @@ -785,8 +783,8 @@ auto Field::get_ND_view () const -> } template -auto Field::get_ND_view () const -> - if_t,HD>> +auto Field::get_ND_view () const + -> if_t,HD>> { const auto& fl = m_header->get_identifier().get_layout(); EKAT_REQUIRE_MSG (N==1 || N==fl.rank(), @@ -814,8 +812,8 @@ auto Field::get_ND_view () const -> // NOTE: DO NOT USE--this circumvents compile-time issues with // subview slicing in get_strided_view() template -auto Field::get_ND_view () const -> - if_t<(N >= MaxRank + 1),get_view_type,HD>> +auto Field::get_ND_view () const + -> if_t<(N >= MaxRank + 1),get_view_type,HD>> { EKAT_ERROR_MSG("Error! Cannot call get_ND_view for rank greater than " "MaxRank = 6.\n" From 96903c18a8e0ba0549fbbb0dcc05de34e9cf475e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 23 May 2024 13:48:14 -0600 Subject: [PATCH 228/476] EAMxx: add valgrind suppression for mappy For some reason, pnetcdf issues an uninitialized bytes error, which is not due to us. We can ignore it --- .../eamxx/scripts/jenkins/valgrind/mappy.supp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/components/eamxx/scripts/jenkins/valgrind/mappy.supp b/components/eamxx/scripts/jenkins/valgrind/mappy.supp index b8d02e9906c3..7516dbe44054 100644 --- a/components/eamxx/scripts/jenkins/valgrind/mappy.supp +++ b/components/eamxx/scripts/jenkins/valgrind/mappy.supp @@ -550,3 +550,21 @@ fun:start_thread fun:clone } + +{ + + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:mca_fbtl_posix_pwritev + fun:write_init.constprop.0 + fun:mca_fcoll_vulcan_file_write_all + fun:mca_common_ompio_file_write_at_all + fun:mca_io_ompio_file_write_at_all + fun:PMPI_File_write_at_all + fun:ncmpio_read_write + fun:wait_getput + fun:ncmpio_wait + fun:ncmpi_wait_all + fun:flush_output_buffer +} From 382f3bfc238c8fdf528aaefd94e62ec3446c8a6a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 23 May 2024 13:49:20 -0600 Subject: [PATCH 229/476] EAMxx: fix compilation error of query-cf-database when yaml-cpp is found and not built --- components/eamxx/scripts/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/scripts/CMakeLists.txt b/components/eamxx/scripts/CMakeLists.txt index 3c7ab6eba5ff..377452229b9d 100644 --- a/components/eamxx/scripts/CMakeLists.txt +++ b/components/eamxx/scripts/CMakeLists.txt @@ -9,4 +9,6 @@ add_executable(query-cf-database query-cf-database.cpp) target_compile_definitions(query-cf-database PUBLIC CF_STANDARD_NAME_FILE=${CF_STANDARD_NAME_FILE} CF_SCREAM_NAME_FILE=${CF_SCREAM_NAME_FILE}) + +find_package (yaml-cpp HINTS ${YAML_CPP_ROOT}) target_link_libraries(query-cf-database ekat yaml-cpp) From 3ccde551e76ab316afe4f0d7e0f53ebad8d76dbd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 23 May 2024 13:51:10 -0600 Subject: [PATCH 230/476] EAMxx: make scorpio interface test not hard-coded to 1-4 ranks --- .../eamxx/src/share/io/tests/scorpio_interface_tests.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp b/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp index ea2999eef575..d5dcafd839a7 100644 --- a/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp +++ b/components/eamxx/src/share/io/tests/scorpio_interface_tests.cpp @@ -20,18 +20,14 @@ TEST_CASE ("io_subsystem") { TEST_CASE ("write_and_read") { ekat::Comm comm (MPI_COMM_WORLD); - EKAT_REQUIRE_MSG (comm.size()<=4, - "Error! This test is tailored for an MPI_Comm of size 1, 2, 3, or 4.\n" - " - MPI_Comm size: " + std::to_string(comm.size()) + "\n"); - init_subsystem (comm); std::string filename = "scorpio_interface_write_test_np" + std::to_string(comm.size()) + ".nc"; const int dim1 = 2; const int dim2 = 4; - const int dim3 = 12; - const int ldim3 = dim3 / comm.size(); + const int ldim3 = 3; + const int dim3 = ldim3 * comm.size(); // Offsets for dim3 decomp owned by this rank std::vector my_offsets; From d2e3fe1b1d9a2d2790f47a3dcb3470efd058a933 Mon Sep 17 00:00:00 2001 From: Peter Bogenschutz Date: Thu, 23 May 2024 13:32:13 -0700 Subject: [PATCH 231/476] Fix where minimum TKE was being clipped an order of magnitude higher than intended in the SHOC interface --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 099794adafce..4fbd89994b12 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -27,6 +27,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess using PF = scream::PhysicsFunctions; using C = physics::Constants; using KT = ekat::KokkosTypes; + using SC = scream::shoc::Constants; using Spack = typename SHF::Spack; using IntSmallPack = typename SHF::IntSmallPack; @@ -79,6 +80,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const Real cpair = C::Cpair; const Real ggr = C::gravit; const Real inv_ggr = 1/ggr; + const Real mintke = SC::mintke; const int nlev_packs = ekat::npack(nlev); @@ -93,7 +95,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess EKAT_KERNEL_ASSERT((nonzero || !in_nlev_range).all()); inv_exner(i,k).set(nonzero, 1/exner); - tke(i,k) = ekat::max(sp(0.004), tke(i,k)); + tke(i,k) = ekat::max(mintke, tke(i,k)); // Tracers are updated as a group. The tracers tke and qc act as separate inputs to shoc_main() // and are therefore updated differently to the bundled tracers. Here, we make a copy if each From f6246e12226bd0a933f218eeaddfe33d4039e323 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 4 Nov 2023 17:46:38 -0700 Subject: [PATCH 232/476] Started with MAMOptics and changed to MAMAci --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- .../mam/eamxx_mam_aci_process_interface.cpp | 107 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 81 +++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 53eb4049f11b..cfd33e88d0b2 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -42,7 +42,8 @@ add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/ma # EAMxx mam4xx-based atmospheric processes add_library(mam eamxx_mam_microphysics_process_interface.cpp - eamxx_mam_optics_process_interface.cpp) + eamxx_mam_optics_process_interface.cpp + eamxx_mam_aci_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp new file mode 100644 index 000000000000..fc27eafcf9d2 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -0,0 +1,107 @@ +#include +#include +#include + +#include "scream_config.h" // for SCREAM_CIME_BUILD + +#include + +namespace scream +{ + +MAMAci::MAMAci( + const ekat::Comm& comm, + const ekat::ParameterList& params) + : AtmosphereProcess(comm, params), + aero_config_() { +} + +AtmosphereProcessType MAMAci::type() const { + return AtmosphereProcessType::Physics; +} + +std::string MAMAci::name() const { + return "mam4_optics"; +} + +void MAMAci::set_grids(const std::shared_ptr grids_manager) { + using namespace ekat::units; + + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column + nswbands_ = 14; // number of shortwave bands + nlwbands_ = 16; // number of longwave bands + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Define aerosol optics fields computed by this process. + auto nondim = Units::nondimensional(); + FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; + FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; + + // shortwave aerosol scattering asymmetry parameter [-] + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol single-scattering albedo [-] + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol optical depth [-] + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + // longwave aerosol optical depth [-] + add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); + + // FIXME: this field doesn't belong here, but this is a convenient place to + // FIXME: put it for now. + // number mixing ratio for CCN + using Spack = ekat::Pack; + using Pack = ekat::Pack; + constexpr int ps = Pack::n; + FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; + add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); +} + +void MAMAci::initialize_impl(const RunType run_type) { +} + +void MAMAci::run_impl(const double dt) { + + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + + // get the aerosol optics fields + auto aero_g_sw = get_field_out("aero_g_sw").get_view(); + auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); + auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); + auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); + + auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this + + // Compute optical properties on all local columns. + // (Strictly speaking, we don't need this parallel_for here yet, but we leave + // it in anticipation of column-specific aerosol optics to come.) + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { + const Int icol = team.league_rank(); // column index + + auto g_sw = ekat::subview(aero_g_sw, icol); + auto ssa_sw = ekat::subview(aero_ssa_sw, icol); + auto tau_sw = ekat::subview(aero_tau_sw, icol); + auto tau_lw = ekat::subview(aero_tau_lw, icol); + + // populate these fields with reasonable representative values + Kokkos::deep_copy(g_sw, 0.5); + Kokkos::deep_copy(ssa_sw, 0.7); + Kokkos::deep_copy(tau_sw, 0.0); + Kokkos::deep_copy(tau_lw, 0.0); + + // FIXME: Get rid of this + auto nccn = ekat::subview(aero_nccn, icol); + Kokkos::deep_copy(nccn, 50.0); + }); +} + +void MAMAci::finalize_impl() +{ +} + +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp new file mode 100644 index 000000000000..0dbcbf753665 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -0,0 +1,81 @@ +#ifndef EAMXX_MAM_ACI_HPP +#define EAMXX_MAM_ACI_HPP + +#include +#include +#include + +#include +#include +#include + +#include + +#ifndef KOKKOS_ENABLE_CUDA +#define protected_except_cuda public +#define private_except_cuda public +#else +#define protected_except_cuda protected +#define private_except_cuda private +#endif + +namespace scream +{ + +// The process responsible for handling MAM4 aerosol optical properties. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMAci final : public scream::AtmosphereProcess { + using PF = scream::PhysicsFunctions; + using KT = ekat::KokkosTypes; + + // a quantity stored in a single vertical column with a single index + using ColumnView = mam4::ColumnView; + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + +public: + + // Constructor + MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); + +protected_except_cuda: + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // process metadata + AtmosphereProcessType type() const override; + std::string name() const override; + + // grid + void set_grids(const std::shared_ptr grids_manager) override; + + // process behavior + void initialize_impl(const RunType run_type) override; + void run_impl(const double dt) override; + void finalize_impl() override; + +private_except_cuda: + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // number of shortwave and longwave radiation bands + int nswbands_, nlwbands_; + + // MAM4 aerosol particle size description + mam4::AeroConfig aero_config_; + + // aerosol processes + //std::unique_ptr optics_; + + // physics grid for column information + std::shared_ptr grid_; +}; // MAMAci + +} // namespace scream + + +#endif // EAMXX_MAM_ACI_HPP From 3da2c97b7b347787a9f5bf767ad4e5b60e591492 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 6 Nov 2023 11:09:50 -0800 Subject: [PATCH 233/476] Minimal additions for a MAMAci class with a constructor --- .../mam/eamxx_mam_aci_process_interface.cpp | 97 +------------------ .../mam/eamxx_mam_aci_process_interface.hpp | 62 ------------ 2 files changed, 1 insertion(+), 158 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index fc27eafcf9d2..e550ad55fe7e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,10 +1,4 @@ #include -#include -#include - -#include "scream_config.h" // for SCREAM_CIME_BUILD - -#include namespace scream { @@ -12,96 +6,7 @@ namespace scream MAMAci::MAMAci( const ekat::Comm& comm, const ekat::ParameterList& params) - : AtmosphereProcess(comm, params), - aero_config_() { -} - -AtmosphereProcessType MAMAci::type() const { - return AtmosphereProcessType::Physics; -} - -std::string MAMAci::name() const { - return "mam4_optics"; -} - -void MAMAci::set_grids(const std::shared_ptr grids_manager) { - using namespace ekat::units; - - grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); - - ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // number of levels per column - nswbands_ = 14; // number of shortwave bands - nlwbands_ = 16; // number of longwave bands - - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - - // Define aerosol optics fields computed by this process. - auto nondim = Units::nondimensional(); - FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; - FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; - - // shortwave aerosol scattering asymmetry parameter [-] - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol single-scattering albedo [-] - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol optical depth [-] - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); - // longwave aerosol optical depth [-] - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); - - // FIXME: this field doesn't belong here, but this is a convenient place to - // FIXME: put it for now. - // number mixing ratio for CCN - using Spack = ekat::Pack; - using Pack = ekat::Pack; - constexpr int ps = Pack::n; - FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); -} - -void MAMAci::initialize_impl(const RunType run_type) { -} - -void MAMAci::run_impl(const double dt) { - - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - - // get the aerosol optics fields - auto aero_g_sw = get_field_out("aero_g_sw").get_view(); - auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); - auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); - auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); - - auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this - - // Compute optical properties on all local columns. - // (Strictly speaking, we don't need this parallel_for here yet, but we leave - // it in anticipation of column-specific aerosol optics to come.) - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const Int icol = team.league_rank(); // column index - - auto g_sw = ekat::subview(aero_g_sw, icol); - auto ssa_sw = ekat::subview(aero_ssa_sw, icol); - auto tau_sw = ekat::subview(aero_tau_sw, icol); - auto tau_lw = ekat::subview(aero_tau_lw, icol); - - // populate these fields with reasonable representative values - Kokkos::deep_copy(g_sw, 0.5); - Kokkos::deep_copy(ssa_sw, 0.7); - Kokkos::deep_copy(tau_sw, 0.0); - Kokkos::deep_copy(tau_lw, 0.0); - - // FIXME: Get rid of this - auto nccn = ekat::subview(aero_nccn, icol); - Kokkos::deep_copy(nccn, 50.0); - }); -} - -void MAMAci::finalize_impl() -{ + : AtmosphereProcess(comm, params){ } } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0dbcbf753665..e54c2e1400ae 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -2,77 +2,15 @@ #define EAMXX_MAM_ACI_HPP #include -#include -#include - -#include -#include -#include - -#include - -#ifndef KOKKOS_ENABLE_CUDA -#define protected_except_cuda public -#define private_except_cuda public -#else -#define protected_except_cuda protected -#define private_except_cuda private -#endif namespace scream { -// The process responsible for handling MAM4 aerosol optical properties. The AD -// stores exactly ONE instance of this class in its list of subcomponents. class MAMAci final : public scream::AtmosphereProcess { - using PF = scream::PhysicsFunctions; - using KT = ekat::KokkosTypes; - - // a quantity stored in a single vertical column with a single index - using ColumnView = mam4::ColumnView; - - // a thread team dispatched to a single vertical column - using ThreadTeam = mam4::ThreadTeam; public: - // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); - -protected_except_cuda: - - // -------------------------------------------------------------------------- - // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) - // -------------------------------------------------------------------------- - - // process metadata - AtmosphereProcessType type() const override; - std::string name() const override; - - // grid - void set_grids(const std::shared_ptr grids_manager) override; - - // process behavior - void initialize_impl(const RunType run_type) override; - void run_impl(const double dt) override; - void finalize_impl() override; - -private_except_cuda: - - // number of horizontal columns and vertical levels - int ncol_, nlev_; - - // number of shortwave and longwave radiation bands - int nswbands_, nlwbands_; - - // MAM4 aerosol particle size description - mam4::AeroConfig aero_config_; - - // aerosol processes - //std::unique_ptr optics_; - - // physics grid for column information - std::shared_ptr grid_; }; // MAMAci } // namespace scream From 13861452a9187458405d80197802286a98cdb0bb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 7 Nov 2023 14:59:50 -0800 Subject: [PATCH 234/476] Model compiles after registering the process, added extra stuff, will remove in the next commit --- .../mam/eamxx_mam_aci_process_interface.cpp | 29 +++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 23 +++++++++++++++ .../eamxx/src/physics/register_physics.hpp | 2 ++ 3 files changed, 54 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e550ad55fe7e..458244807283 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,11 @@ #include +#include "scream_config.h" // for SCREAM_CIME_BUILD +#include +#include + +#include "scream_config.h" // for SCREAM_CIME_BUILD + +#include namespace scream { @@ -9,4 +16,26 @@ MAMAci::MAMAci( : AtmosphereProcess(comm, params){ } +AtmosphereProcessType MAMAci::type() const { + return AtmosphereProcessType::Physics; +} +//return name of the process +std::string MAMAci::name() const{ + return "mam4_aci"; + } + + +//grid +void MAMAci::set_grids(const std::shared_ptr grids_manager) { +} + +void MAMAci::initialize_impl(const RunType run_type) { +} + +void MAMAci::run_impl(const double dt) { +} + +void MAMAci::finalize_impl(){ +} + } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index e54c2e1400ae..ca89537db96c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -2,6 +2,14 @@ #define EAMXX_MAM_ACI_HPP #include +#include +#include + +#include +#include +#include + +#include namespace scream { @@ -11,8 +19,23 @@ class MAMAci final : public scream::AtmosphereProcess { public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); +// process metadata + AtmosphereProcessType type() const override; + std::string name() const override; + + // grid + void set_grids(const std::shared_ptr grids_manager) override; + +// process behavior + void initialize_impl(const RunType run_type) override; + void run_impl(const double dt) override; + void finalize_impl() override; + + + }; // MAMAci + } // namespace scream diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 329b7022756c..36afabf8de97 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,6 +26,7 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" +#include "physics/mam/eamxx_mam_aci_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -62,6 +63,7 @@ inline void register_physics () { #ifdef EAMXX_HAS_MAM proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); + proc_factory.register_product("mam4_aci",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 10f5790efdf84f6cd0f81a95fa294de4bbbcf572 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 8 Nov 2023 08:19:08 -0800 Subject: [PATCH 235/476] Removed un-needed code, minimal code for invoking and printing init,set grid, run and finalize aci process --- .../mam/eamxx_mam_aci_process_interface.cpp | 20 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 16 ++------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 458244807283..b753e3d7ee8f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,41 +1,39 @@ #include -#include "scream_config.h" // for SCREAM_CIME_BUILD -#include -#include - -#include "scream_config.h" // for SCREAM_CIME_BUILD - -#include namespace scream { - MAMAci::MAMAci( const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params){ } + +//Return type of the process AtmosphereProcessType MAMAci::type() const { return AtmosphereProcessType::Physics; } + //return name of the process std::string MAMAci::name() const{ return "mam4_aci"; } - -//grid +//set grid for all the inputs and outputs void MAMAci::set_grids(const std::shared_ptr grids_manager) { + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI set grid = "); } void MAMAci::initialize_impl(const RunType run_type) { + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); } void MAMAci::run_impl(const double dt) { + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI run = "); } void MAMAci::finalize_impl(){ + m_atm_logger->log(ekat::logger::LogLevel::info,"ACI final = "); } -} // namespace scream +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index ca89537db96c..0d361ba9f415 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -2,14 +2,6 @@ #define EAMXX_MAM_ACI_HPP #include -#include -#include - -#include -#include -#include - -#include namespace scream { @@ -19,23 +11,19 @@ class MAMAci final : public scream::AtmosphereProcess { public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); -// process metadata + // process metadata AtmosphereProcessType type() const override; std::string name() const override; // grid void set_grids(const std::shared_ptr grids_manager) override; -// process behavior + // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; void finalize_impl() override; - - - }; // MAMAci - } // namespace scream From 4a59acad24d55aec3f81996f2fcae3f9342db347 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 Nov 2023 08:29:34 -0800 Subject: [PATCH 236/476] Minimal code for adding a test for aci with two outputs --- .../mam/eamxx_mam_aci_process_interface.cpp | 17 +++++ .../mam/eamxx_mam_aci_process_interface.hpp | 9 +++ .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 34 +++++++++ .../tests/uncoupled/mam4/mam4_aci_output.yaml | 12 ++++ .../uncoupled/mam4/mam4_aci_standalone.cpp | 69 +++++++++++++++++++ 5 files changed, 141 insertions(+) create mode 100644 components/eamxx/tests/uncoupled/mam4/input_aci.yaml create mode 100644 components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml create mode 100644 components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b753e3d7ee8f..c72ddfd43e03 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -22,10 +22,27 @@ std::string MAMAci::name() const{ //set grid for all the inputs and outputs void MAMAci::set_grids(const std::shared_ptr grids_manager) { m_atm_logger->log(ekat::logger::LogLevel::info,"ACI set grid = "); + + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variables + FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; + + using namespace ekat::units; + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure } void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); + std::cout<<"BALLI==============================="< grid_; }; // MAMAci } // namespace scream diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml new file mode 100644 index 000000000000..b254d95a7350 --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -0,0 +1,34 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_aci] + mam4_micro: + compute_tendencies: [] + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + T_mid: 273.0 + p_mid: 1.e5 + qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["mam4_aci_output.yaml"] +... diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml b/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml new file mode 100644 index 000000000000..0087223d1d6a --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml @@ -0,0 +1,12 @@ +%YAML 1.1 +--- +filename_prefix: mam4_aci_standalone_output +Averaging Type: Instant +Field Names: + - T_mid + - p_mid +output_control: + Frequency: 1 + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp new file mode 100644 index 000000000000..8100fd37a1fc --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp @@ -0,0 +1,69 @@ +#include + +#include "control/atmosphere_driver.hpp" +#include "diagnostics/register_diagnostics.hpp" + +#include "physics/register_physics.hpp" +#include "physics/mam/eamxx_mam_aci_process_interface.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "ekat/ekat_parse_yaml_file.hpp" +#include "ekat/logging/ekat_logger.hpp" + +#include + +namespace scream { + +TEST_CASE("mam4-aci-standalone", "") { + using namespace scream; + using namespace scream::control; + + // Create a comm + ekat::Comm atm_comm (MPI_COMM_WORLD); + ekat::logger::Logger<> logger("mam4-aci", + ekat::logger::LogLevel::debug, atm_comm); + + + // Load ad parameter list + std::string fname = "input_aci.yaml"; + ekat::ParameterList ad_params("Atmosphere Driver"); + parse_yaml_file(fname,ad_params); + logger.debug("aci yaml parsed."); + + // Time stepping parameters + const auto& ts = ad_params.sublist("time_stepping"); + const auto dt = ts.get("time_step"); + const auto nsteps = ts.get("number_of_steps"); + const auto t0_str = ts.get("run_t0"); + const auto t0 = util::str_to_time_stamp(t0_str); + + logger.info("running MAMAci standalone test with dt = {} for {} steps.", dt, nsteps); + + // Need to register products in the factory *before* we create any atm process or grids manager. + register_physics(); + register_mesh_free_grids_manager(); + register_diagnostics(); + logger.debug("products registered."); + + // Create the driver + AtmosphereDriver ad; + logger.debug("driver created."); + + // Init and run + ad.initialize(atm_comm,ad_params,t0); + logger.debug("driver initialized."); + + logger.info("Start time stepping loop ... [0%]"); + for (int i=0; i Date: Tue, 14 Nov 2023 15:03:35 -0800 Subject: [PATCH 237/476] Adds some required inputs for the aci process --- .../mam/eamxx_mam_aci_process_interface.cpp | 34 +++++++++++++++++-- .../uncoupled/mam4/mam4_aci_standalone.cpp | 1 - 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index c72ddfd43e03..808dfc75d83b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -33,11 +33,39 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) using namespace ShortFieldTagsNames; // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; + FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points + FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces using namespace ekat::units; - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // total pressure + auto q_unit = kg/kg; // units of mass mixing ratios of tracers + auto n_unit = kg/kg; // units of number mixing ratios of tracers + + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid mass mixing ratio [kg/kg] + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints + add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints + add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios + /*for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } + } + }*/ + /*NOTE on other inputs for the aci process: + 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density + 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using + pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). + qv_mid can be obtained from "get_field_in" call*/ + } void MAMAci::initialize_impl(const RunType run_type) { diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp index 8100fd37a1fc..dd3ed62ec15b 100644 --- a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp +++ b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp @@ -24,7 +24,6 @@ TEST_CASE("mam4-aci-standalone", "") { ekat::logger::Logger<> logger("mam4-aci", ekat::logger::LogLevel::debug, atm_comm); - // Load ad parameter list std::string fname = "input_aci.yaml"; ekat::ParameterList ad_params("Atmosphere Driver"); From 3dad80470927ca400560c69322c80842924d557e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 Nov 2023 15:56:02 -0800 Subject: [PATCH 238/476] Fixes unit for the number mixing ratio --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 808dfc75d83b..52a282ab5696 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -38,7 +38,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) using namespace ekat::units; auto q_unit = kg/kg; // units of mass mixing ratios of tracers - auto n_unit = kg/kg; // units of number mixing ratios of tracers + auto n_unit = 1/kg; // units of number mixing ratios of tracers add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] From 1557547f53ae273ebce8f1b7f981a58a1c6494a3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 Nov 2023 18:08:30 -0800 Subject: [PATCH 239/476] Adds Commented out code to stage next changes --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index dc788898073c..357740598508 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,6 +1,8 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP +//#include +//#include #include namespace scream From f9539b1b3b4993274b1845d07baec5a650f661a8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 17 Nov 2023 14:32:08 -0800 Subject: [PATCH 240/476] Adds aerosol fields as required and also in the input yaml --- .../mam/eamxx_mam_aci_process_interface.cpp | 7 +++--- .../mam/eamxx_mam_aci_process_interface.hpp | 3 +-- .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 25 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 52a282ab5696..cae35dc2b93f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -50,7 +50,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios - /*for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { @@ -59,7 +59,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } - }*/ + } + /*NOTE on other inputs for the aci process: 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using @@ -81,4 +82,4 @@ void MAMAci::finalize_impl(){ m_atm_logger->log(ekat::logger::LogLevel::info,"ACI final = "); } -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 357740598508..07e553ce7570 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,8 +1,7 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP -//#include -//#include +#include #include namespace scream diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index b254d95a7350..7927b2418350 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -26,6 +26,31 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} T_mid: 273.0 p_mid: 1.e5 + num_a1: 1.e5 + bc_a1: 1.e-5 + num_a2: 1e5 + num_a3: 1e5 + num_a4: 1e5 + soa_a1: 1e-5 + so4_a1: 1e-5 + pom_a1: 1e-5 + nacl_a1: 1e-5 + dst_a1: 1e-5 + mom_a1: 1e-5 + soa_a2: 1e-5 + so4_a2: 1e-5 + nacl_a2: 1e-5 + mom_a2: 1e-5 + soa_a3: 1e-5 + so4_a3: 1e-5 + pom_a3: 1e-5 + bc_a3: 1e-5 + nacl_a3: 1e-5 + dst_a3: 1e-5 + mom_a3: 1e-5 + pom_a4: 1e-5 + bc_a4: 1e-5 + mom_a4: 1e-5 qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae # The parameters for I/O control From 13333316db6e3d577e578a98b927858bd381b3fc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 17 Nov 2023 15:13:39 -0800 Subject: [PATCH 241/476] Adds cloud borne aerosols; fixes a minor bug in mam coupling --- .../mam/eamxx_mam_aci_process_interface.cpp | 8 +++++++ .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index cae35dc2b93f..f848e6dd8f5a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -59,6 +59,14 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } + for (int c = 0; c < mam_coupling::num_aero_species(); ++c) { + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, c); + std::cout< 0) { + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + } + } } /*NOTE on other inputs for the aci process: diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index 7927b2418350..3704235bdf97 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -51,6 +51,28 @@ initial_conditions: pom_a4: 1e-5 bc_a4: 1e-5 mom_a4: 1e-5 + #cloud borne aerosols + soa_c1: 1e-5 + so4_c1: 1e-5 + pom_c1: 1e-5 + bc_c1: 1e-5 + nacl_c1: 1e-5 + dst_c1: 1e-5 + mom_c1: 1e-5 + soa_c2: 1e-5 + so4_c2: 1e-5 + nacl_c2: 1e-5 + mom_c2: 1e-5 + soa_c3: 1e-5 + so4_c3: 1e-5 + pom_c3: 1e-5 + bc_c3: 1e-5 + nacl_c3: 1e-5 + dst_c3: 1e-5 + mom_c3: 1e-5 + pom_c4: 1e-5 + bc_c4: 1e-5 + mom_c4: 1e-5 qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae # The parameters for I/O control From 25d45f0820475a29f0fb79e82588531a5c4e7c61 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 20 Nov 2023 19:43:01 -0800 Subject: [PATCH 242/476] Adds all input output variables in set grid --- .../mam/eamxx_mam_aci_process_interface.cpp | 21 +++++++++++++++++-- .../mam/eamxx_mam_aci_process_interface.hpp | 4 ++++ .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 6 +++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f848e6dd8f5a..9818160f914a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -37,8 +37,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces using namespace ekat::units; - auto q_unit = kg/kg; // units of mass mixing ratios of tracers - auto n_unit = 1/kg; // units of number mixing ratios of tracers + const auto q_unit = kg/kg; // units of mass mixing ratios of tracers + const auto n_unit = 1/kg; // units of number mixing ratios of tracers add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] @@ -69,6 +69,23 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } } + //input for aci codes that existed in PBUF in EAM + //These outputs should come from the cloud macrophysics process (e.g., SHOC) + const auto m2 = m*m; + const auto s2 = s*s; + //FIXME BALLI: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time + add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + + auto nondim = Units::nondimensional(); + add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints + add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints + add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_modes_} }; // mid points + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + + /*NOTE on other inputs for the aci process: 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 07e553ce7570..232bcf95abcb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -30,6 +30,10 @@ class MAMAci final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; + // number of aerosol modes + int num_modes_; + + // physics grid for column information std::shared_ptr grid_; }; // MAMAci diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index 3704235bdf97..a14ec51d0368 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -74,7 +74,11 @@ initial_conditions: bc_c4: 1e-5 mom_c4: 1e-5 qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae - + w_sec: 1e-2 + strat_cld_frac: 0.1 + liq_strat_cld_frac: 0.1 + kvh: 0.1 + dgnum: 1e-3 # The parameters for I/O control Scorpio: output_yaml_files: ["mam4_aci_output.yaml"] From 7f9d5a3db5bf4bc3d884caeb2904553e50b9ade5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 25 Nov 2023 10:58:42 -0800 Subject: [PATCH 243/476] Adds commented out Fortran code with comments, some cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 209 +++++++++++++++++- .../mam/eamxx_mam_aci_process_interface.hpp | 8 +- .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 4 + 3 files changed, 208 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9818160f914a..7e35ef923139 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -2,6 +2,10 @@ namespace scream { + + //FIXME: The following variables are namelist variables + Real wsubmin = 1; + MAMAci::MAMAci( const ekat::Comm& comm, const ekat::ParameterList& params) @@ -37,9 +41,12 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces using namespace ekat::units; - const auto q_unit = kg/kg; // units of mass mixing ratios of tracers - const auto n_unit = 1/kg; // units of number mixing ratios of tracers + auto q_unit = kg/kg; // units of mass mixing ratios of tracers + q_unit.set_string("kg/kg"); + auto n_unit = 1/kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid mass mixing ratio [kg/kg] @@ -49,19 +56,27 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + //interstitial aerosol tracers of interest: number (n) mixing ratios const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + //cloudborne aerosol tracers of interest: number (n) mixing ratios + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - } - for (int c = 0; c < mam_coupling::num_aero_species(); ++c) { - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, c); - std::cout< 0) { //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); @@ -69,10 +84,13 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } } - //input for aci codes that existed in PBUF in EAM + //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM //These outputs should come from the cloud macrophysics process (e.g., SHOC) - const auto m2 = m*m; - const auto s2 = s*s; + auto m2 = m*m; + m2.set_string("m^2"); + auto s2 = s*s; + s2.set_string("s^2"); + //FIXME BALLI: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints @@ -82,7 +100,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_modes_} }; // mid points + num_aero_modes_ = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols @@ -96,11 +115,177 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); - std::cout<<"BALLI==============================="<log(ekat::logger::LogLevel::info,"ACI run = "); + + //---------------------------------------------------------- + // Convert from omega to w (vertical velocity) + // Negative omega means rising motion + //--------------------------------------------------------- + + //Get physical constants + using C = physics::Constants; + + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + + + /* + NOTE: All the inputs are available to compute w0 + Fortran code: + w0 = 0 + rho(:,:) = -999.0_r8 + do kk = top_lev, pver + do icol = 1, ncol + rho(icol,kk) = pmid(icol,kk)/(rair*temperature(icol,kk)) + w0(icol,kk) = -1._r8*omega(icol,kk)/(rho(icol,kk)*gravit) + enddo + enddo */ + + //--------------------------------------------------------- + //Compute TKE using "w_sec" + //--------------------------------------------------------- + + /* + NOTE: All the inputs are available to compute tke + Fortran code: + tke(:ncol,:) = (3._r8/2._r8)*w_sec(:ncol,:) + */ + //--------------------------------------------------------- + // Compute subgrid scale velocities(wsub, wsig and wsubice) + //--------------------------------------------------------- + + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + /* + NOTE: All the inputs are available to compute wsub, wsig and wsubice + "min_max_bound" is present in MAM4xx utils + + Fortran code: + wsub(:ncol,:top_lev-1) = wsubmin + wsubice(:ncol,:top_lev-1) = 0.001_r8 + wsig(:ncol,:top_lev-1) = 0.001_r8 + + do kk = top_lev, pver + do icol = 1, ncol + wsub(icol,kk) = sqrt(0.5_r8*(tke(icol,kk) + tke(icol,kk+1))*(2._r8/3._r8)) + wsig(icol,kk) = min_max_bound(0.001_r8, 10._r8, wsub(icol,kk)) + wsubice(icol,kk) = min_max_bound(0.2_r8, 10._r8, wsub(icol,kk)) + wsub(icol,kk) = max(wsubmin, wsub(icol,kk)) + end do + end do + */ + + + //--------------------------------------------------------- + // Compute subgrid mean updraft velocity (w2) + //--------------------------------------------------------- + + /* + NOTE: All inputs are available. "subgrid_mean_updraft" is not ported yet but it is a very small routine + Fortran code: + w2(1:ncol,1:pver) = 0._r8 + call subgrid_mean_updraft(ncol, w0, wsig, &!in + w2) !out + */ + + //------------------------------------------------------------- + // Get number of activated aerosol for ice nucleation (naai) + // from ice nucleation + //------------------------------------------------------------- + + /* + NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". + Only output we care about is "naai", "naai_hom" is never used anywhere + + Fortran code: + call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input + rho, wsubice, strat_cld_frac, dgnum, & ! input + naai, naai_hom) ! output + */ + + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + //------------------------------------------------------------- + + static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) + + /* + MUST FIXME NOTE: We need old and new liquid cloud fractions here. + We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to + store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make + a note of it and use the new cloud fraction for the + old cloud fraction. + + Fortran code: + liqcldf(:ncol,:pver) = liq_strat_cld_frac(:ncol,:pver) + lcldn = 0._r8 + lcldo = 0._r8 + do kk = top_lev, pver + do icol = 1, ncol + qcld = qc(icol,kk) + qi(icol,kk) + if (qcld > qsmall) then + lcldn(icol,kk)=liqcldf(icol,kk) + lcldo(icol,kk)=liqcldfo(icol,kk) + end if + end do + end do + */ + + //------------------------------------------------------------- + // Save cloud borne aerosols to be used in the heterozenous + // freezing before they are changed by the droplet activation + // process. This is only a select subset of cloud borne + // aerosols, not all the cloud borne aerosols. + //------------------------------------------------------------- + + /*NOTE: We probably need to store indices for the select few + cloud borne aerosols + + Fortran code: + lchnk_zb = lchnk - begchunk + ! save copy of cloud borne aerosols for use in heterogeneous freezing before + !we change it in dropmixnuc + do ispec = 1, ncnst + call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) + aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) + aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) + enddo + */ + //------------------------------------------------------------- + // Compute activated fraction of aerosols + //------------------------------------------------------------- + + + /* + NOTE: "deltain" is the model time step. "state_q" is a combination of tracers + fields with "int_mmr_field_name" and "int_nmr_field_name". "z_mid" is computed. + "qqcw" is the combination of cld_mmr_field_name and cld_nmr_field_name. + The output "ptend" will have tendencies for interstitial and cloud borne aerosols. + + Fortan code: + call dropmixnuc(lchnk, ncol, deltatin, T_mid, p_mid, p_int, p_del, rpdel, z_mid, & ! in + state_q, nc, kvh, wsub, lcldn, lcldo, & ! in + qqcw, & ! inout + ptend, nctend_mixnuc, factnum) !out + */ + + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + + /* + call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, & ! in + qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in + frzimm, frzcnt, frzdep) + */ + } void MAMAci::finalize_impl(){ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 232bcf95abcb..21a8caeee2a6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,9 +1,15 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP +//For MAM4 aerosol configuration #include + +//For declaring ACI class derived from atm process class #include +//For physical constants +#include "physics/share/physics_constants.hpp" + namespace scream { @@ -31,7 +37,7 @@ class MAMAci final : public scream::AtmosphereProcess { int ncol_, nlev_; // number of aerosol modes - int num_modes_; + int num_aero_modes_; // physics grid for column information diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index a14ec51d0368..a7103ad01a88 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -27,10 +27,14 @@ initial_conditions: T_mid: 273.0 p_mid: 1.e5 num_a1: 1.e5 + num_c1: 1.e5 bc_a1: 1.e-5 num_a2: 1e5 num_a3: 1e5 num_a4: 1e5 + num_c2: 1e5 + num_c3: 1e5 + num_c4: 1e5 soa_a1: 1e-5 so4_a1: 1e-5 pom_a1: 1e-5 From 6ebbf3a70cb6a80ceb6d3ac4963754ea7ce97b89 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 28 Nov 2023 14:58:03 -0800 Subject: [PATCH 244/476] Cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 32 ++++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7e35ef923139..7fccef52f054 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -14,7 +14,7 @@ MAMAci::MAMAci( //Return type of the process -AtmosphereProcessType MAMAci::type() const { +AtmosphereProcessType MAMAci::type() const{ return AtmosphereProcessType::Physics; } @@ -25,9 +25,9 @@ std::string MAMAci::name() const{ //set grid for all the inputs and outputs void MAMAci::set_grids(const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info,"ACI set grid = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); //Use physics grid const auto& grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -56,6 +56,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { //interstitial aerosol tracers of interest: number (n) mixing ratios @@ -91,7 +93,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) auto s2 = s*s; s2.set_string("s^2"); - //FIXME BALLI: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time + //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints auto nondim = Units::nondimensional(); @@ -114,11 +116,25 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info,"ACI init = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); + /* + NOTE: All other inputs should follow the way "pseudo_density" is initialized + */ + const auto& p_del = get_field_in("pseudo_density").get_view(); + + // set atmosphere state data + pdel_ = p_del; + + /* + NOTE: All derived variables (like rpdel and geopotential height) should be computed in + preprocess struct + */ + //preprocess_.set_variables(pdel_); + } void MAMAci::run_impl(const double dt) { - m_atm_logger->log(ekat::logger::LogLevel::info,"ACI run = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI run"); //---------------------------------------------------------- // Convert from omega to w (vertical velocity) @@ -259,7 +275,7 @@ void MAMAci::run_impl(const double dt) { // Compute activated fraction of aerosols //------------------------------------------------------------- - + std::cout<<"pdel_ in run_impl is:"<log(ekat::logger::LogLevel::info,"ACI final = "); + m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI final"); } } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 21a8caeee2a6..7b63eb83fdd8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -15,6 +15,13 @@ namespace scream class MAMAci final : public scream::AtmosphereProcess { + + using KT = ekat::KokkosTypes; + + // views for single- and multi-column data + using const_view_2d = typename KT::template view_2d; + + public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); @@ -38,6 +45,31 @@ class MAMAci final : public scream::AtmosphereProcess { // number of aerosol modes int num_aero_modes_; + + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + + //const_view_2d pdel_; // hydrostatic "pressure thickness" at grid + // interfaces [Pa] + + // assigns local variables + void set_variables(const const_view_2d& pdel) { + //p1del_ = pdel; + } // set_variables + }; // MAMAci::Preprocess + + + // pre- and postprocessing scratch pads + Preprocess preprocess_; + + // local atmospheric state column variables + const_view_2d pdel_; // hydrostatic "pressure thickness" at grid + // interfaces [Pa] // physics grid for column information From a05b9753a9cd86fd3fbd988865174b6e13b81960 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 30 Nov 2023 14:25:06 -0700 Subject: [PATCH 245/476] Add some scratch views to store density and velocity. --- .../mam/eamxx_mam_aci_process_interface.cpp | 21 ++++++++++++++++--- .../mam/eamxx_mam_aci_process_interface.hpp | 16 +++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7fccef52f054..83c3da5f2d46 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -33,6 +33,9 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -120,10 +123,11 @@ void MAMAci::initialize_impl(const RunType run_type) { /* NOTE: All other inputs should follow the way "pseudo_density" is initialized */ - const auto& p_del = get_field_in("pseudo_density").get_view(); - // set atmosphere state data - pdel_ = p_del; + pdel_ = get_field_in("pseudo_density").get_view(); + omega_ = get_field_in("omega").get_view(); + p_mid_ = get_field_in("p_mid").get_view(); + T_mid_ = get_field_in("T_mid").get_view(); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in @@ -147,6 +151,17 @@ void MAMAci::run_impl(const double dt) { static constexpr auto gravit = C::gravit; // Gravity [m/s2] static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + // NOTE: All the inputs are available to compute w0 + for (int icol = 0; icol < ncol_; ++icol) { + for (int kk = 0; kk< top_lev_; ++kk) { + w0_(icol,kk) = 0; + rho_(icol, kk) = -999.0; + } + for (int kk = top_lev_; kk < nlev_; ++kk) { + rho_(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); + w0_(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); + } + } /* NOTE: All the inputs are available to compute w0 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 7b63eb83fdd8..fe90d9991956 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -20,7 +20,15 @@ class MAMAci final : public scream::AtmosphereProcess { // views for single- and multi-column data using const_view_2d = typename KT::template view_2d; - + using view_2d = typename KT::template view_2d; + // rho is air density [kg/m3] + view_2d rho_; + + // w0_ is large scale velocity (m/s) + view_2d w0_; + + // Top level for troposphere cloud physics + const int top_lev_ = 6; public: // Constructor @@ -70,8 +78,10 @@ class MAMAci final : public scream::AtmosphereProcess { // local atmospheric state column variables const_view_2d pdel_; // hydrostatic "pressure thickness" at grid // interfaces [Pa] - - + const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints + const_view_2d p_mid_; // Total pressure [Pa] at midpoints + const_view_2d T_mid_; // Temperature[K] at midpoints + // physics grid for column information std::shared_ptr grid_; }; // MAMAci From 82eb4d87446b95a6c613667f4bde5d9d84d59304 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 5 Dec 2023 09:05:25 -0700 Subject: [PATCH 246/476] Add subgrid_mean_updraft implementation. --- components/eamxx/CMakeLists.txt | 2 +- .../mam/eamxx_mam_aci_process_interface.cpp | 155 +++++++++++------- .../mam/eamxx_mam_aci_process_interface.hpp | 9 +- 3 files changed, 107 insertions(+), 59 deletions(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 8567512abf2b..f0afd1c30451 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -546,7 +546,7 @@ if (SCREAM_DOUBLE_PRECISION) set(SCREAM_Fortran_FLAGS -real-size 64) elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -s default32 -eZ) + set(SCREAM_Fortran_FLAGS -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 83c3da5f2d46..de5478873b68 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,8 +1,57 @@ #include - namespace scream { +KOKKOS_INLINE_FUNCTION +Real subgrid_mean_updraft(const Real w0, const Real wsig) +{ +/* --------------------------------------------------------------------------------- + Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the + vertical velocity distribution is Gaussian and peaks at the + GCM resolved large-scale vertical velocity. + When icenul_wsub_scheme = 2, the model uses the mean updraft velocity as the + characteristic updraft velocity to calculate the ice nucleation rate. + Author: Kai Zhang (kai.zhang@pnnl.gov) + Last Modified: Oct, 2015 +-------------------------------------------------------------------------------- */ + +// interface + +// in :: wsig standard deviation (m/s) +// in :: w0 large scale vertical velocity (m/s) +// out:: mean updraft velocity(m/s) -> characteristic w* + + // FIXME should nbin be a user parameter? + const int nbin = 50; + + using C = physics::Constants; + constexpr Real pi = C::Pi; + Real zz[nbin], wa = 0, ww = 0; + int kp = 0; + + const Real sigma = haero::max(0.001, wsig); + const Real wlarge = w0; + + const Real xx = 6.0 * sigma / nbin; + + for (int ibin = 0; ibin < nbin; ++ibin) { + Real yy = wlarge - 3.0*sigma + 0.5*xx; + yy += (ibin-1)*xx; + // wbar = integrator < w * f(w) * dw > + zz[ibin] = yy * haero::exp(-1.0*haero::square(yy-wlarge)/(2*sigma*sigma))/(sigma*haero::sqrt(2*pi))*xx; + } + for (int ibin = 0; ibin < nbin; ++ibin) { + if (zz[ibin] > 0) ++kp, wa += zz[ibin]; + } + if (kp) { + // wbar = integrator < w * f(w) * dw > + ww = wa; + } else { + ww = 0.001; + } + return ww; +} + //FIXME: The following variables are namelist variables Real wsubmin = 1; @@ -35,7 +84,12 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); - + Kokkos::resize(tke_, ncol_, nlev_+1); + Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(wsubice_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -128,6 +182,7 @@ void MAMAci::initialize_impl(const RunType run_type) { omega_ = get_field_in("omega").get_view(); p_mid_ = get_field_in("p_mid").get_view(); T_mid_ = get_field_in("T_mid").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in @@ -152,75 +207,61 @@ void MAMAci::run_impl(const double dt) { static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] // NOTE: All the inputs are available to compute w0 - for (int icol = 0; icol < ncol_; ++icol) { - for (int kk = 0; kk< top_lev_; ++kk) { + auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { w0_(icol,kk) = 0; rho_(icol, kk) = -999.0; - } - for (int kk = top_lev_; kk < nlev_; ++kk) { + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { rho_(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); w0_(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); - } - } - - /* - NOTE: All the inputs are available to compute w0 - Fortran code: - w0 = 0 - rho(:,:) = -999.0_r8 - do kk = top_lev, pver - do icol = 1, ncol - rho(icol,kk) = pmid(icol,kk)/(rair*temperature(icol,kk)) - w0(icol,kk) = -1._r8*omega(icol,kk)/(rho(icol,kk)*gravit) - enddo - enddo */ + }); + }); - //--------------------------------------------------------- - //Compute TKE using "w_sec" - //--------------------------------------------------------- - - /* - NOTE: All the inputs are available to compute tke - Fortran code: - tke(:ncol,:) = (3._r8/2._r8)*w_sec(:ncol,:) - */ + //--------------------------------------------------------- + //Compute TKE using "w_sec" + //--------------------------------------------------------- + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // FIXME Is this the correct boundary condition for tke at the surface? + tke_(icol,nlev_) = 0; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { + tke_(icol,kk) = (3.0/2.0)*w_sec_(icol,kk); + }); + }); //--------------------------------------------------------- // Compute subgrid scale velocities(wsub, wsig and wsubice) //--------------------------------------------------------- - // More refined computation of sub-grid vertical velocity - // Set to be zero at the surface by initialization. - /* - NOTE: All the inputs are available to compute wsub, wsig and wsubice - "min_max_bound" is present in MAM4xx utils - - Fortran code: - wsub(:ncol,:top_lev-1) = wsubmin - wsubice(:ncol,:top_lev-1) = 0.001_r8 - wsig(:ncol,:top_lev-1) = 0.001_r8 - - do kk = top_lev, pver - do icol = 1, ncol - wsub(icol,kk) = sqrt(0.5_r8*(tke(icol,kk) + tke(icol,kk+1))*(2._r8/3._r8)) - wsig(icol,kk) = min_max_bound(0.001_r8, 10._r8, wsub(icol,kk)) - wsubice(icol,kk) = min_max_bound(0.2_r8, 10._r8, wsub(icol,kk)) - wsub(icol,kk) = max(wsubmin, wsub(icol,kk)) - end do - end do - */ - + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { + wsub_(icol,kk) = wsubmin; + wsubice_(icol,kk) = 0.001; + wsig_(icol,kk) = 0.001; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { + wsub_(icol,kk) = haero::sqrt(0.5*(tke_(icol,kk) + tke_(icol,kk+1))*(2.0/3.0)); + wsig_(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub_(icol,kk)); + wsubice_(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub_(icol,kk)); + wsub_(icol,kk) = haero::max(wsubmin, wsub_(icol,kk)); + }); + }); //--------------------------------------------------------- // Compute subgrid mean updraft velocity (w2) //--------------------------------------------------------- - /* - NOTE: All inputs are available. "subgrid_mean_updraft" is not ported yet but it is a very small routine - Fortran code: - w2(1:ncol,1:pver) = 0._r8 - call subgrid_mean_updraft(ncol, w0, wsig, &!in - w2) !out - */ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { + w2_(icol,kk) = subgrid_mean_updraft(w0_(icol,kk), wsig_(icol,kk)); + }); + }); //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index fe90d9991956..8c0decf0925d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -27,7 +27,13 @@ class MAMAci final : public scream::AtmosphereProcess { // w0_ is large scale velocity (m/s) view_2d w0_; + // turbulent kinetic energy [m^2/s^2] + view_2d tke_; + + // Subgrid scale velocities + view_2d wsub_, wsubice_, wsig_, w2_; // Top level for troposphere cloud physics + // FIXME: This should be read in to make user selectable. const int top_lev_ = 6; public: @@ -81,7 +87,8 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints const_view_2d p_mid_; // Total pressure [Pa] at midpoints const_view_2d T_mid_; // Temperature[K] at midpoints - + const_view_2d w_sec_; // Vertical velocity variance + // physics grid for column information std::shared_ptr grid_; }; // MAMAci From 512818d389e55014eb6225a4e3fbb6c8bc8fd421 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 7 Dec 2023 08:32:47 -0700 Subject: [PATCH 247/476] Add call to nucleate_ice_ This required a number input fields many of which are not in the regression test input file. Will need to find a way to add them. Next is the dropmixnuc function call which is also has quite a few input fields. --- .../mam/eamxx_mam_aci_process_interface.cpp | 239 +++++++++++++++--- .../mam/eamxx_mam_aci_process_interface.hpp | 34 +++ 2 files changed, 234 insertions(+), 39 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index de5478873b68..7a4368d327fd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,6 @@ #include +#include "ekat/util/ekat_units.hpp" +#include "mam4xx/aero_config.hpp" namespace scream { @@ -52,8 +54,8 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) return ww; } - //FIXME: The following variables are namelist variables - Real wsubmin = 1; +//FIXME: The following variables are namelist variables +const Real wsubmin = 1; MAMAci::MAMAci( const ekat::Comm& comm, @@ -89,6 +91,9 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) Kokkos::resize(wsubice_, ncol_, nlev_); Kokkos::resize(wsig_, ncol_, nlev_); Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(lcldn_, ncol_, nlev_); + Kokkos::resize(lcldo_, ncol_, nlev_); + Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -102,7 +107,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) q_unit.set_string("kg/kg"); auto n_unit = 1/kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); - + auto nondim = Units::nondimensional(); // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] @@ -111,7 +116,26 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces + add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] + //add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] + //add_field("q_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] + //add_field("q_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] + //add_field("n_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] + //add_field("n_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] + + add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] + add_field("niim",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] + add_field("nidep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + add_field("nimey",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + add_field("naai_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] + add_field("naai",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry @@ -125,7 +149,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -151,9 +175,8 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) s2.set_string("s^2"); //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time - add_field("w_sec", scalar3d_layout_mid, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints - auto nondim = Units::nondimensional(); add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat @@ -161,7 +184,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) // Layout for 4D (2d horiz X 1d vertical x number of modes) variables num_aero_modes_ = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols /*NOTE on other inputs for the aci process: @@ -183,6 +206,34 @@ void MAMAci::initialize_impl(const RunType run_type) { p_mid_ = get_field_in("p_mid").get_view(); T_mid_ = get_field_in("T_mid").get_view(); w_sec_ = get_field_in("w_sec").get_view(); + qv_dry_ = get_field_in("qv").get_view(); + cldfrac_ = get_field_in("cldfrac_tot").get_view(); + Kokkos::resize(w_updraft_, ncol_, nlev_); // w_updraft_ = get_field_in("w_updraft").get_view(); + Kokkos::resize(q_coarse_dst_, ncol_, nlev_); // q_coarse_dst_ = get_field_in("q_coarse_dst").get_view(); + Kokkos::resize(q_coarse_nacl_, ncol_, nlev_); // q_coarse_nacl_ = get_field_in("q_coarse_nacl").get_view(); + Kokkos::resize(q_coarse_so4_, ncol_, nlev_); // q_coarse_so4_ = get_field_in("q_coarse_so4").get_view(); + Kokkos::resize(q_coarse_mom_, ncol_, nlev_); // q_coarse_mom_ = get_field_in("q_coarse_mom").get_view(); + Kokkos::resize(q_coarse_bc_, ncol_, nlev_); // q_coarse_bc_ = get_field_in("q_coarse_bc").get_view(); + Kokkos::resize(q_coarse_pom_, ncol_, nlev_); // q_coarse_pom_ = get_field_in("q_coarse_pom").get_view(); + Kokkos::resize(q_coarse_soa_, ncol_, nlev_); // q_coarse_soa_ = get_field_in("q_coarse_soa").get_view(); + Kokkos::resize(n_coarse_, ncol_, nlev_); // n_coarse_ = get_field_in("n_coarse").get_view(); + Kokkos::resize(n_aitken_, ncol_, nlev_); // n_aitken_ = get_field_in("n_aitken").get_view(); + dgnum_ = get_field_out("dgnum").get_view(); + nihf_ = get_field_out("nihf").get_view(); + niim_ = get_field_out("niim").get_view(); + nidep_ = get_field_out("nidep").get_view(); + nimey_ = get_field_out("nimey").get_view(); + naai_hom_ = get_field_out("naai_hom").get_view(); + naai_ = get_field_out("naai").get_view(); + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + qc_ = get_field_in("qc").get_view(); + qi_ = get_field_in("qi").get_view(); + + + // configure the nucleation parameterization + mam4::NucleateIce::Config config; + mam4::AeroConfig aero_config; + nucleate_ice_.init(aero_config, config); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in @@ -206,17 +257,57 @@ void MAMAci::run_impl(const double dt) { static constexpr auto gravit = C::gravit; // Gravity [m/s2] static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + // Alias member variables + auto w0 = w0_; + auto w2 = w2_; + auto rho = rho_; + auto tke = tke_; + auto T_mid = T_mid_; + auto p_mid = p_mid_; + auto w_sec = w_sec_; + auto wsub = wsub_; + auto wsubice = wsubice_; + auto wsig = wsig_; + auto qv_dry = qv_dry_; + auto cldfrac = cldfrac_; + auto w_updraft = w_updraft_; + const auto nlev = nlev_; + const auto top_lev = top_lev_; + + auto q_coarse_dst = q_coarse_dst_; + auto q_coarse_nacl = q_coarse_nacl_; + auto q_coarse_so4 = q_coarse_so4_; + auto q_coarse_mom = q_coarse_mom_; + auto q_coarse_bc = q_coarse_bc_; + auto q_coarse_pom = q_coarse_pom_; + auto q_coarse_soa = q_coarse_soa_; + auto n_coarse = n_coarse_; + auto n_aitken = n_aitken_; + auto dgnum = dgnum_; + auto nihf = nihf_; + auto niim = niim_; + auto nidep = nidep_; + auto nimey = nimey_; + auto naai_hom = naai_hom_; + auto naai = naai_; + auto liqcldf = liqcldf_; + auto qc = qc_; + auto qi = qi_; + auto lcldn = lcldn_; + auto lcldo = lcldo_; + auto aitken_dry_dia = aitken_dry_dia_; + // NOTE: All the inputs are available to compute w0 auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { - w0_(icol,kk) = 0; - rho_(icol, kk) = -999.0; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol,kk) = 0; + rho(icol, kk) = -999.0; }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { - rho_(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); - w0_(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol,kk) = p_mid_(icol,kk)/(rair*T_mid_(icol,kk)); + w0(icol,kk) = -1.0*omega_(icol,kk)/(rho_(icol,kk)*gravit); }); }); @@ -226,9 +317,10 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // FIXME Is this the correct boundary condition for tke at the surface? - tke_(icol,nlev_) = 0; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { - tke_(icol,kk) = (3.0/2.0)*w_sec_(icol,kk); + // TKE seems to be at interfaces but w_sec is at cell centers so this + // descrepensy needs to be worked out. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { + tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); }); }); //--------------------------------------------------------- @@ -239,16 +331,16 @@ void MAMAci::run_impl(const double dt) { // Set to be zero at the surface by initialization. Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev_), KOKKOS_LAMBDA(int kk) { - wsub_(icol,kk) = wsubmin; - wsubice_(icol,kk) = 0.001; - wsig_(icol,kk) = 0.001; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = wsubmin; + wsubice(icol,kk) = 0.001; + wsig(icol,kk) = 0.001; }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev_, nlev_), KOKKOS_LAMBDA(int kk) { - wsub_(icol,kk) = haero::sqrt(0.5*(tke_(icol,kk) + tke_(icol,kk+1))*(2.0/3.0)); - wsig_(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub_(icol,kk)); - wsubice_(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub_(icol,kk)); - wsub_(icol,kk) = haero::max(wsubmin, wsub_(icol,kk)); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = haero::sqrt(0.5*(tke_(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); + wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); + wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); + wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); }); }); @@ -258,26 +350,84 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev_), KOKKOS_LAMBDA(int kk) { - w2_(icol,kk) = subgrid_mean_updraft(w0_(icol,kk), wsig_(icol,kk)); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); }); }); - //------------------------------------------------------------- - // Get number of activated aerosol for ice nucleation (naai) - // from ice nucleation - //------------------------------------------------------------- + //------------------------------------------------------------- + // Get number of activated aerosol for ice nucleation (naai) + // from ice nucleation + //------------------------------------------------------------- + using view_1d = typename KokkosTypes::template view_1d; + view_1d dummy("DummyView", nlev); + + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); + }); + }); + + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, + ekat::subview(T_mid, icol), + ekat::subview(p_mid, icol), + ekat::subview(qv_dry, icol), + dummy, dummy, dummy, dummy, dummy, dummy, + ekat::subview(cldfrac, icol), + ekat::subview(w_updraft, icol), pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs(nlev); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(q_coarse_dst, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(q_coarse_nacl, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(q_coarse_so4, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(q_coarse_mom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(q_coarse_bc, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(q_coarse_pom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(q_coarse_soa, icol); + progs.n_mode_i[coarse_idx] = ekat::subview(n_coarse, icol); + progs.n_mode_i[aitken_idx] = ekat::subview(n_aitken, icol); + + // nucleation doesn't use any diagnostics, so it's okay to leave this alone + // for now + mam4::Diagnostics diags(nlev); + diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t=0, dt=0; + nucleate_ice_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); /* - NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". - Only output we care about is "naai", "naai_hom" is never used anywhere + NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". + Only output we care about is "naai", "naai_hom" is never used anywhere - Fortran code: - call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input - rho, wsubice, strat_cld_frac, dgnum, & ! input - naai, naai_hom) ! output + Fortran code: + call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input + rho, wsubice, strat_cld_frac, dgnum, & ! input + naai, naai_hom) ! output */ - + }); //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value @@ -291,7 +441,18 @@ void MAMAci::run_impl(const double dt) { store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make a note of it and use the new cloud fraction for the old cloud fraction. - + */ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const Real qcld = qc(icol,kk) + qi(icol,kk); + if (qcld > qsmall) { + lcldn(icol,kk)=liqcldf(icol,kk); + lcldo(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + } + }); + }); + /* Fortran code: liqcldf(:ncol,:pver) = liq_strat_cld_frac(:ncol,:pver) lcldn = 0._r8 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 8c0decf0925d..b959990d9cd1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -18,9 +18,15 @@ class MAMAci final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; + mam4::NucleateIce nucleate_ice_; + // views for single- and multi-column data + using view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; using view_2d = typename KT::template view_2d; + using view_3d = typename KT::template view_3d; + using const_view_3d = typename KT::template view_3d; + // rho is air density [kg/m3] view_2d rho_; @@ -30,6 +36,34 @@ class MAMAci final : public scream::AtmosphereProcess { // turbulent kinetic energy [m^2/s^2] view_2d tke_; + const_view_2d qv_dry_; + const_view_2d cldfrac_; + //const_view_2d w_updraft_; + view_2d w_updraft_; + + view_2d aitken_dry_dia_; + view_2d q_coarse_dst_; + view_2d q_coarse_nacl_; + view_2d q_coarse_so4_; + view_2d q_coarse_mom_; + view_2d q_coarse_bc_; + view_2d q_coarse_pom_; + view_2d q_coarse_soa_; + view_2d n_coarse_; + view_2d n_aitken_; + view_3d dgnum_; + view_2d nihf_; + view_2d niim_; + view_2d nidep_; + view_2d nimey_; + view_2d naai_hom_; + view_2d naai_; + const_view_2d liqcldf_; + const_view_2d qc_; + const_view_2d qi_; + view_2d lcldn_; + view_2d lcldo_; + // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; // Top level for troposphere cloud physics From c1a663507929c2c6b93b2490e737943b7ed396ce Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 19 Dec 2023 15:23:26 -0700 Subject: [PATCH 248/476] Update eamxx_mam_aci_process_interface to call dropmixnuc. Since dropmixnuc takes about 75 input arguments, this is no trivial task. Now that the inputs are defined and allocated, need to clean this up with a few helper classes like Jeff Johnson has used for other cases. --- .../mam/eamxx_mam_aci_process_interface.cpp | 278 +++++++++++++++++- .../mam/eamxx_mam_aci_process_interface.hpp | 44 ++- 2 files changed, 306 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7a4368d327fd..9e70dc8a4776 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,6 +1,7 @@ #include #include "ekat/util/ekat_units.hpp" #include "mam4xx/aero_config.hpp" +#include "mam4xx/ndrop.hpp" namespace scream { @@ -25,6 +26,7 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) // FIXME should nbin be a user parameter? const int nbin = 50; + using C = physics::Constants; constexpr Real pi = C::Pi; @@ -84,17 +86,21 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_+1); - Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_+1); + Kokkos::resize(wsub_, ncol_, nlev_); Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(lcldn_, ncol_, nlev_); - Kokkos::resize(lcldo_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(lcldn_, ncol_, nlev_); + Kokkos::resize(lcldo_, ncol_, nlev_); Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); + for (int i=0; i<15; ++i) { + Kokkos::resize(scratch_mem_[i],ncol_, nlev_); + } // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -118,7 +124,9 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] + + + // FIXME: These should all come from the intput file. Might need to be added to the input file? //add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] //add_field("q_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] //add_field("q_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] @@ -129,6 +137,12 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) //add_field("q_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] //add_field("n_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] //add_field("n_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] + //add_field("zm", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) + //add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] + //add_field("ncldwtr", , n_unit, grid_name); // initial droplet number mixing ratio [#/kg] + //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] + //add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] add_field("niim",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] @@ -136,6 +150,22 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("nimey",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] add_field("naai_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] add_field("naai",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] + add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] + add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] + add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] + add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::AeroConfig::num_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] + add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] + add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output + add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency + + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry @@ -179,7 +209,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints - add_field("kvh", scalar3d_layout_mid, m2/s, grid_name); // Eddy diffusivity for heat + add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables num_aero_modes_ = mam_coupling::num_aero_modes(); @@ -204,10 +234,11 @@ void MAMAci::initialize_impl(const RunType run_type) { pdel_ = get_field_in("pseudo_density").get_view(); omega_ = get_field_in("omega").get_view(); p_mid_ = get_field_in("p_mid").get_view(); + p_int_ = get_field_in("p_int").get_view(); T_mid_ = get_field_in("T_mid").get_view(); w_sec_ = get_field_in("w_sec").get_view(); qv_dry_ = get_field_in("qv").get_view(); - cldfrac_ = get_field_in("cldfrac_tot").get_view(); + // FIXME: These should all come from the intput file. Might need to be added to the input file? Kokkos::resize(w_updraft_, ncol_, nlev_); // w_updraft_ = get_field_in("w_updraft").get_view(); Kokkos::resize(q_coarse_dst_, ncol_, nlev_); // q_coarse_dst_ = get_field_in("q_coarse_dst").get_view(); Kokkos::resize(q_coarse_nacl_, ncol_, nlev_); // q_coarse_nacl_ = get_field_in("q_coarse_nacl").get_view(); @@ -218,6 +249,31 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(q_coarse_soa_, ncol_, nlev_); // q_coarse_soa_ = get_field_in("q_coarse_soa").get_view(); Kokkos::resize(n_coarse_, ncol_, nlev_); // n_coarse_ = get_field_in("n_coarse").get_view(); Kokkos::resize(n_aitken_, ncol_, nlev_); // n_aitken_ = get_field_in("n_aitken").get_view(); + Kokkos::resize(zm_, ncol_, nlev_); // zm_ = get_field_in("zm").get_view(); + Kokkos::resize(state_q_, ncol_, nlev_, mam4::ndrop::nvars); // state_q_ = get_field_in("state_q").get_view(); + Kokkos::resize(ncldwtr_, ncol_, nlev_); // ncldwtr_ = get_field_in("ncldwtr").get_view(); + Kokkos::resize(cldo_, ncol_, nlev_); // cldo_ = get_field_in("cldo").get_view(); + Kokkos::resize(qqcw_inp_, ncol_, nlev_, mam4::ndrop::ncnst_tot); // qqcw_inp_ = get_field_in("qqcw").get_view(); + + for (int i=0; i(); dgnum_ = get_field_out("dgnum").get_view(); nihf_ = get_field_out("nihf").get_view(); niim_ = get_field_out("niim").get_view(); @@ -225,9 +281,24 @@ void MAMAci::initialize_impl(const RunType run_type) { nimey_ = get_field_out("nimey").get_view(); naai_hom_ = get_field_out("naai_hom").get_view(); naai_ = get_field_out("naai").get_view(); + qcld_ = get_field_out("qcld").get_view(); + ptend_q_inp_ = get_field_out("ptend_q").get_view(); + tendnd_ = get_field_out("tendnd").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + ccn_ = get_field_out("ccn").get_view(); + coltend_outp_ = get_field_out("coltend").get_view(); + coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + + + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); qc_ = get_field_in("qc").get_view(); qi_ = get_field_in("qi").get_view(); + kvh_ = get_field_in("kvh").get_view(); // configure the nucleation parameterization @@ -264,6 +335,8 @@ void MAMAci::run_impl(const double dt) { auto tke = tke_; auto T_mid = T_mid_; auto p_mid = p_mid_; + auto p_int = p_int_; + auto pdel = pdel_; auto w_sec = w_sec_; auto wsub = wsub_; auto wsubice = wsubice_; @@ -296,9 +369,67 @@ void MAMAci::run_impl(const double dt) { auto lcldn = lcldn_; auto lcldo = lcldo_; auto aitken_dry_dia = aitken_dry_dia_; + auto rpdel = rpdel_; + auto zm = zm_; + auto state_q = state_q_; + auto ncldwtr = ncldwtr_; + auto kvh = kvh_; + auto qcld = qcld_; + auto ptend_q_inp = ptend_q_inp_; + auto tendnd = tendnd_; + auto factnum = factnum_; + auto ndropcol = ndropcol_ ; + auto ndropmix = ndropmix_ ; + auto nsource = nsource_ ; + auto wtke = wtke_ ; + auto cldo = cldo_; + auto qqcw_inp = qqcw_inp_; + auto ccn = ccn_; + auto coltend_outp = coltend_outp_; + auto coltend_cw_outp = coltend_cw_outp_; + auto nact = nact_; + auto mact = mact_; + auto eddy_diff = scratch_mem_[0]; + auto zn = scratch_mem_[1]; + auto csbot = scratch_mem_[2]; + auto zs = scratch_mem_[3]; + auto overlapp = scratch_mem_[4]; + auto overlapm = scratch_mem_[5]; + auto eddy_diff_kp = scratch_mem_[6]; + auto eddy_diff_km = scratch_mem_[7]; + auto qncld = scratch_mem_[8]; + auto srcn = scratch_mem_[9]; + auto source = scratch_mem_[10]; + auto dz = scratch_mem_[11]; + auto csbot_cscen = scratch_mem_[12]; + auto raertend = scratch_mem_[13]; + auto qqcwtend = scratch_mem_[14]; - // NOTE: All the inputs are available to compute w0 auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); + + // NOTE: All the inputs are available to compute w0 + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i; using const_view_3d = typename KT::template view_3d; + template + using view_4d = KT::view; + + // FIXME the time step for microphysics [s] need to get from the input + const Real dtmicro_ = .0001; + // rho is air density [kg/m3] view_2d rho_; @@ -37,7 +43,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d tke_; const_view_2d qv_dry_; - const_view_2d cldfrac_; + view_2d cldfrac_; //const_view_2d w_updraft_; view_2d w_updraft_; @@ -60,9 +66,38 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_; const_view_2d liqcldf_; const_view_2d qc_; - const_view_2d qi_; + const_view_2d qi_; + const_view_2d kvh_; + view_2d cldo_; + + view_2d zm_; + view_3d state_q_; + view_2d ncldwtr_; + view_2d lcldn_; view_2d lcldo_; + view_2d qcld_; + view_2d tendnd_; + view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; + view_3d ptend_q_inp_; + view_3d factnum_; + view_3d qqcw_inp_; + view_2d qqcw_[mam4::ndrop::ncnst_tot]; + view_2d ndropcol_; + view_2d ndropmix_; + view_2d nsource_; + view_2d wtke_; + view_3d ccn_; + view_3d coltend_outp_; + view_2d coltend_[mam4::ndrop::ncnst_tot]; + view_3d coltend_cw_outp_; + view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; + view_2d raercol_cw_[mam4::ndrop::pver][2]; + view_2d raercol_[mam4::ndrop::pver][2]; + view_3d nact_; + view_3d mact_; + view_2d scratch_mem_[15]; + // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; @@ -116,11 +151,12 @@ class MAMAci final : public scream::AtmosphereProcess { Preprocess preprocess_; // local atmospheric state column variables - const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints const_view_2d p_mid_; // Total pressure [Pa] at midpoints + const_view_2d p_int_; // Total pressure [Pa] at interfaces const_view_2d T_mid_; // Temperature[K] at midpoints + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ const_view_2d w_sec_; // Vertical velocity variance // physics grid for column information From 7a5aaecd0ec27f897c94a59e1806cd63dcba5ebd Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 23 Jan 2024 13:21:48 -0700 Subject: [PATCH 249/476] Just stashing a working version. --- .../mam/eamxx_mam_aci_process_interface.cpp | 602 +++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 71 ++- 2 files changed, 489 insertions(+), 184 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9e70dc8a4776..1fb52659ac5f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -2,9 +2,99 @@ #include "ekat/util/ekat_units.hpp" #include "mam4xx/aero_config.hpp" #include "mam4xx/ndrop.hpp" + namespace scream { +namespace +{ +void copy_scream_array_to_mam4xx( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d mam4xx_view[], + MAMAci::const_view_3d scream_view, + const int nlev, + const int num_views) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol,kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); + w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); + }); + }); +} +void compute_tke_using_w_sec( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d tke, + MAMAci::const_view_2d w_sec, + const int nlev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // FIXME Is this the correct boundary condition for tke at the surface? + // TKE seems to be at interfaces but w_sec is at cell centers so this + // descrepensy needs to be worked out. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { + tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); + }); + }); +} +void compute_subgrid_scale_velocities( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, + MAMAci::view_2d wsig, + MAMAci::const_view_2d tke, + const Real wsubmin, + const int top_lev, + const int nlev) +{ + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = wsubmin; + wsubice(icol,kk) = 0.001; + wsig(icol,kk) = 0.001; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); + wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); + wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); + wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); + }); + }); +} + + KOKKOS_INLINE_FUNCTION Real subgrid_mean_updraft(const Real w0, const Real wsig) { @@ -55,6 +145,43 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) } return ww; } +void compute_subgrid_mean_updraft_velocities( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d w2, + MAMAci::const_view_2d w0, + MAMAci::const_view_2d wsig, + const int nlev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); + }); + }); +} +void compute_aitken_dry_diameter( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, + const int top_lev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); + }); + }); +} + +void compute_nucleate_ice_tendencies( + haero::ThreadTeamPolicy team_policy, + const int nlev) +{ +} + +} + //FIXME: The following variables are namelist variables const Real wsubmin = 1; @@ -99,7 +226,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) Kokkos::resize(rpdel_, ncol_, nlev_); for (int i=0; i<15; ++i) { - Kokkos::resize(scratch_mem_[i],ncol_, nlev_); + Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); } // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -117,31 +244,48 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid mass mixing ratio [kg/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // + // + add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints // FIXME: These should all come from the intput file. Might need to be added to the input file? - //add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - //add_field("q_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] - //add_field("q_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] - //add_field("n_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] - //add_field("n_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] + add_field("qi_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] + add_field("qi_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] + //add_field("qi_accum_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for accum mode [kg/kg] + //add_field("qi_accum_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for accum mode [kg/kg] + //add_field("qi_accum_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for accum mode [kg/kg] + //add_field("qi_accumn_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] + //add_field("qi_accumn_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] + //add_field("qi_accumn_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] + //add_field("qi_pcarbon_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] + //add_field("qi_pcarbon_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] + //add_field("qi_pcarbon_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] + //add_field("ni_accum", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] + //add_field("ni_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] + //add_field("ni_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] //add_field("zm", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) //add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] //add_field("ncldwtr", , n_unit, grid_name); // initial droplet number mixing ratio [#/kg] //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] - //add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + // + add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] + add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] @@ -165,7 +309,6 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry @@ -214,7 +357,7 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) // Layout for 4D (2d horiz X 1d vertical x number of modes) variables num_aero_modes_ = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols /*NOTE on other inputs for the aci process: @@ -238,22 +381,53 @@ void MAMAci::initialize_impl(const RunType run_type) { T_mid_ = get_field_in("T_mid").get_view(); w_sec_ = get_field_in("w_sec").get_view(); qv_dry_ = get_field_in("qv").get_view(); + w_updraft_ = get_field_in("w_updraft").get_view(); // updraft velocity [m/s] + // + stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); + activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); + activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); + // FIXME: These should all come from the intput file. Might need to be added to the input file? - Kokkos::resize(w_updraft_, ncol_, nlev_); // w_updraft_ = get_field_in("w_updraft").get_view(); - Kokkos::resize(q_coarse_dst_, ncol_, nlev_); // q_coarse_dst_ = get_field_in("q_coarse_dst").get_view(); - Kokkos::resize(q_coarse_nacl_, ncol_, nlev_); // q_coarse_nacl_ = get_field_in("q_coarse_nacl").get_view(); - Kokkos::resize(q_coarse_so4_, ncol_, nlev_); // q_coarse_so4_ = get_field_in("q_coarse_so4").get_view(); - Kokkos::resize(q_coarse_mom_, ncol_, nlev_); // q_coarse_mom_ = get_field_in("q_coarse_mom").get_view(); - Kokkos::resize(q_coarse_bc_, ncol_, nlev_); // q_coarse_bc_ = get_field_in("q_coarse_bc").get_view(); - Kokkos::resize(q_coarse_pom_, ncol_, nlev_); // q_coarse_pom_ = get_field_in("q_coarse_pom").get_view(); - Kokkos::resize(q_coarse_soa_, ncol_, nlev_); // q_coarse_soa_ = get_field_in("q_coarse_soa").get_view(); - Kokkos::resize(n_coarse_, ncol_, nlev_); // n_coarse_ = get_field_in("n_coarse").get_view(); - Kokkos::resize(n_aitken_, ncol_, nlev_); // n_aitken_ = get_field_in("n_aitken").get_view(); + Kokkos::resize(qc_accum_dst_, ncol_, nlev_); // qc_accum_dst_ = get_field_in("qc_accum_dst").get_view(); + Kokkos::resize(qc_accum_so4_, ncol_, nlev_); // qc_accum_so4_ = get_field_in("qc_accum_so4").get_view(); + Kokkos::resize(qc_accum_mom_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); + Kokkos::resize(qc_accum_nacl_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); + Kokkos::resize(qc_accum_bc_, ncol_, nlev_); // qc_accum_bc_ = get_field_in("qc_accum_bc").get_view(); + Kokkos::resize(qc_accum_pom_, ncol_, nlev_); // qc_accum_pom_ = get_field_in("qc_accum_pom").get_view(); + Kokkos::resize(qc_accum_soa_, ncol_, nlev_); // qc_accum_soa_ = get_field_in("qc_accum_soa").get_view(); + Kokkos::resize(qi_accum_dst_, ncol_, nlev_); // qi_accum_dst_ = get_field_in("qi_accum_dst").get_view(); + Kokkos::resize(qi_accum_so4_, ncol_, nlev_); // qi_accum_so4_ = get_field_in("qi_accum_so4").get_view(); + Kokkos::resize(qi_accum_mom_, ncol_, nlev_); // qi_accum_mom_ = get_field_in("qi_accum_mom").get_view(); + Kokkos::resize(qi_accum_bc_, ncol_, nlev_); // qi_accum_bc_ = get_field_in("qi_accum_bc").get_view(); + Kokkos::resize(qi_accum_pom_, ncol_, nlev_); // qi_accum_pom_ = get_field_in("qi_accum_pom").get_view(); + Kokkos::resize(qi_accum_soa_, ncol_, nlev_); // qi_accum_soa_ = get_field_in("qi_accum_soa").get_view(); + Kokkos::resize(qc_coarse_dst_, ncol_, nlev_); // qc_coarse_dst_ = get_field_in("qc_coarse_dst").get_view(); + Kokkos::resize(qc_coarse_nacl_, ncol_, nlev_); // qc_coarse_nacl_ = get_field_in("qc_coarse_nacl").get_view(); + Kokkos::resize(qc_coarse_mom_, ncol_, nlev_); // ci_coarse_mom_ = get_field_in("qc_coarse_mom").get_view(); + Kokkos::resize(qc_coarse_bc_, ncol_, nlev_); // qc_coarse_bc_ = get_field_in("qc_coarse_bc").get_view(); + Kokkos::resize(qc_coarse_pom_, ncol_, nlev_); // qc_coarse_pom_ = get_field_in("qc_coarse_pom").get_view(); + Kokkos::resize(qc_coarse_soa_, ncol_, nlev_); // qc_coarse_soa_ = get_field_in("qc_coarse_soa").get_view(); + // + qi_coarse_dst_ = get_field_in("qi_coarse_dst").get_view(); + qi_coarse_nacl_ = get_field_in("qi_coarse_nacl").get_view(); + qi_coarse_so4_ = get_field_in("qi_coarse_so4").get_view(); + qi_coarse_mom_ = get_field_in("qi_coarse_mom").get_view(); + qi_coarse_bc_ = get_field_in("qi_coarse_bc").get_view(); + qi_coarse_pom_ = get_field_in("qi_coarse_pom").get_view(); + qi_coarse_soa_ = get_field_in("qi_coarse_soa").get_view(); + + Kokkos::resize(qi_pcarbon_bc_, ncol_, nlev_); // qi_pcarbon_bc_ = get_field_in("qi_pcarbon_bc").get_view(); + Kokkos::resize(qi_pcarbon_mom_, ncol_, nlev_); // qi_pcarbon_mom_ = get_field_in("qi_pcarbon_mom").get_view(); + Kokkos::resize(qi_pcarbon_pom_, ncol_, nlev_); // qi_pcarbon_pom_ = get_field_in("qi_pcarbon_pom").get_view(); + Kokkos::resize(nc_accum_, ncol_, nlev_); // nc_accum_ = get_field_in("nc_accum").get_view(); + Kokkos::resize(ni_accum_, ncol_, nlev_); // ni_accum_ = get_field_in("ni_accum").get_view(); + Kokkos::resize(ni_coarse_, ncol_, nlev_); // ni_coarse_ = get_field_in("ni_coarse").get_view(); + Kokkos::resize(ni_aitken_, ncol_, nlev_); // ni_aitken_ = get_field_in("ni_aitken").get_view(); Kokkos::resize(zm_, ncol_, nlev_); // zm_ = get_field_in("zm").get_view(); Kokkos::resize(state_q_, ncol_, nlev_, mam4::ndrop::nvars); // state_q_ = get_field_in("state_q").get_view(); Kokkos::resize(ncldwtr_, ncol_, nlev_); // ncldwtr_ = get_field_in("ncldwtr").get_view(); Kokkos::resize(cldo_, ncol_, nlev_); // cldo_ = get_field_in("cldo").get_view(); - Kokkos::resize(qqcw_inp_, ncol_, nlev_, mam4::ndrop::ncnst_tot); // qqcw_inp_ = get_field_in("qqcw").get_view(); + qqcw_input_ = get_field_in("qqcw").get_view(); for (int i=0; i(); - dgnum_ = get_field_out("dgnum").get_view(); + cldfrac_ = get_field_in("cldfrac_tot").get_view(); + dgnum_ = get_field_out("dgnum").get_view(); nihf_ = get_field_out("nihf").get_view(); niim_ = get_field_out("niim").get_view(); nidep_ = get_field_out("nidep").get_view(); @@ -282,7 +462,7 @@ void MAMAci::initialize_impl(const RunType run_type) { naai_hom_ = get_field_out("naai_hom").get_view(); naai_ = get_field_out("naai").get_view(); qcld_ = get_field_out("qcld").get_view(); - ptend_q_inp_ = get_field_out("ptend_q").get_view(); + ptend_q_output_ = get_field_out("ptend_q").get_view(); tendnd_ = get_field_out("tendnd").get_view(); factnum_ = get_field_out("factnum").get_view(); ndropcol_ = get_field_out("ndropcol").get_view(); @@ -298,14 +478,18 @@ void MAMAci::initialize_impl(const RunType run_type) { liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); qc_ = get_field_in("qc").get_view(); qi_ = get_field_in("qi").get_view(); + nc_ = get_field_in("nc").get_view(); kvh_ = get_field_in("kvh").get_view(); - // configure the nucleation parameterization - mam4::NucleateIce::Config config; mam4::AeroConfig aero_config; - nucleate_ice_.init(aero_config, config); + // configure the nucleation parameterization + mam4::NucleateIce::Config nucleate_ice_config; + nucleate_ice_.init(aero_config, nucleate_ice_config); + // configure the heterogeneous freezing parameterization + mam4::Hetfrz::Config hetfrz_config; + hetfrz_.init(aero_config, hetfrz_config); /* NOTE: All derived variables (like rpdel and geopotential height) should be computed in preprocess struct @@ -322,41 +506,57 @@ void MAMAci::run_impl(const double dt) { // Negative omega means rising motion //--------------------------------------------------------- - //Get physical constants - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] - // Alias member variables - auto w0 = w0_; - auto w2 = w2_; auto rho = rho_; auto tke = tke_; auto T_mid = T_mid_; auto p_mid = p_mid_; auto p_int = p_int_; auto pdel = pdel_; - auto w_sec = w_sec_; auto wsub = wsub_; auto wsubice = wsubice_; - auto wsig = wsig_; auto qv_dry = qv_dry_; auto cldfrac = cldfrac_; auto w_updraft = w_updraft_; const auto nlev = nlev_; const auto top_lev = top_lev_; - auto q_coarse_dst = q_coarse_dst_; - auto q_coarse_nacl = q_coarse_nacl_; - auto q_coarse_so4 = q_coarse_so4_; - auto q_coarse_mom = q_coarse_mom_; - auto q_coarse_bc = q_coarse_bc_; - auto q_coarse_pom = q_coarse_pom_; - auto q_coarse_soa = q_coarse_soa_; - auto n_coarse = n_coarse_; - auto n_aitken = n_aitken_; - auto dgnum = dgnum_; + auto qc_accum_dst = qc_accum_dst_; + auto qc_accum_so4 = qc_accum_so4_; + auto qc_accum_mom = qc_accum_mom_; + auto qc_accum_nacl = qc_accum_nacl_; + auto qc_accum_bc = qc_accum_bc_; + auto qc_accum_pom = qc_accum_pom_; + auto qc_accum_soa = qc_accum_soa_; + + auto qi_accum_dst = qi_accum_dst_; + auto qi_accum_so4 = qi_accum_so4_; + auto qi_accum_mom = qi_accum_mom_; + auto qi_accum_bc = qi_accum_bc_; + auto qi_accum_pom = qi_accum_pom_; + auto qi_accum_soa = qi_accum_soa_; + + auto qc_coarse_dst = qc_coarse_dst_; + auto qc_coarse_nacl = qc_coarse_nacl_; + auto qc_coarse_mom = qc_coarse_mom_; + auto qc_coarse_bc = qc_coarse_bc_; + auto qc_coarse_pom = qc_coarse_pom_; + auto qc_coarse_soa = qc_coarse_soa_; + auto qi_coarse_dst = qi_coarse_dst_; + auto qi_coarse_nacl = qi_coarse_nacl_; + auto qi_coarse_so4 = qi_coarse_so4_; + auto qi_coarse_mom = qi_coarse_mom_; + auto qi_coarse_bc = qi_coarse_bc_; + auto qi_coarse_pom = qi_coarse_pom_; + auto qi_coarse_soa = qi_coarse_soa_; + auto qi_pcarbon_bc = qi_pcarbon_bc_; + auto qi_pcarbon_mom = qi_pcarbon_mom_; + auto qi_pcarbon_pom = qi_pcarbon_pom_; + auto nc_accum = nc_accum_; + auto ni_accum = ni_accum_; + auto ni_coarse = ni_coarse_; + auto ni_aitken = ni_aitken_; auto nihf = nihf_; auto niim = niim_; auto nidep = nidep_; @@ -366,6 +566,7 @@ void MAMAci::run_impl(const double dt) { auto liqcldf = liqcldf_; auto qc = qc_; auto qi = qi_; + auto nc = nc_; auto lcldn = lcldn_; auto lcldo = lcldo_; auto aitken_dry_dia = aitken_dry_dia_; @@ -375,7 +576,6 @@ void MAMAci::run_impl(const double dt) { auto ncldwtr = ncldwtr_; auto kvh = kvh_; auto qcld = qcld_; - auto ptend_q_inp = ptend_q_inp_; auto tendnd = tendnd_; auto factnum = factnum_; auto ndropcol = ndropcol_ ; @@ -383,108 +583,53 @@ void MAMAci::run_impl(const double dt) { auto nsource = nsource_ ; auto wtke = wtke_ ; auto cldo = cldo_; - auto qqcw_inp = qqcw_inp_; auto ccn = ccn_; auto coltend_outp = coltend_outp_; auto coltend_cw_outp = coltend_cw_outp_; auto nact = nact_; auto mact = mact_; - auto eddy_diff = scratch_mem_[0]; - auto zn = scratch_mem_[1]; - auto csbot = scratch_mem_[2]; - auto zs = scratch_mem_[3]; - auto overlapp = scratch_mem_[4]; - auto overlapm = scratch_mem_[5]; - auto eddy_diff_kp = scratch_mem_[6]; - auto eddy_diff_km = scratch_mem_[7]; - auto qncld = scratch_mem_[8]; - auto srcn = scratch_mem_[9]; - auto source = scratch_mem_[10]; - auto dz = scratch_mem_[11]; - auto csbot_cscen = scratch_mem_[12]; - auto raertend = scratch_mem_[13]; - auto qqcwtend = scratch_mem_[14]; - - auto team_policy = haero::ThreadTeamPolicy(ncol_, Kokkos::AUTO); - - // NOTE: All the inputs are available to compute w0 - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - for (int i=0; i::template view_1d; view_1d dummy("DummyView", nlev); - - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); - }); - }); - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -519,21 +655,26 @@ void MAMAci::run_impl(const double dt) { mam4::Prognostics progs(nlev); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(q_coarse_dst, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(q_coarse_nacl, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(q_coarse_so4, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(q_coarse_mom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(q_coarse_bc, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(q_coarse_pom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(q_coarse_soa, icol); - progs.n_mode_i[coarse_idx] = ekat::subview(n_coarse, icol); - progs.n_mode_i[aitken_idx] = ekat::subview(n_aitken, icol); + const int dst_idx = static_cast(mam4::AeroId::DST); + + progs.q_aero_i[coarse_idx][dst_idx] = ekat::subview(qi_coarse_dst, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); + progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); + progs.n_mode_i[aitken_idx] = ekat::subview(ni_aitken, icol); // nucleation doesn't use any diagnostics, so it's okay to leave this alone // for now mam4::Diagnostics diags(nlev); diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated + // in nucleate_ice_.compute_tendencies. diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); diags.icenuc_num_immfrz = ekat::subview(niim, icol); diags.icenuc_num_depnuc = ekat::subview(nidep, icol); @@ -564,6 +705,7 @@ void MAMAci::run_impl(const double dt) { // is above qsmall threshold value //------------------------------------------------------------- + const Real dtmicro = dtmicro_; static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) /* @@ -683,7 +825,7 @@ void MAMAci::run_impl(const double dt) { mam4::ndrop::dropmixnuc( - team, dtmicro, + team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -737,7 +879,7 @@ void MAMAci::run_impl(const double dt) { const int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { for (int i=0; i(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + const int pcarbon_idx = static_cast(mam4::ModeIndex::PrimaryCarbon); + + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_accum_bc, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_accum_dst, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_accum_mom, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_accum_pom, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_accum_nacl, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qc_accum_so4, icol); + progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_accum_soa, icol); + + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_accum_bc, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_accum_dst, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_accum_mom, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_accum_pom, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_accum_so4, icol); + progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_accum_soa, icol); + + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_coarse_bc, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_coarse_dst, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_coarse_mom, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_coarse_nacl, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_coarse_pom, icol); + progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_coarse_soa, icol); + + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_coarse_dst, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); + progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); + + progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_pcarbon_bc, icol); + progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_pcarbon_mom, icol); + progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_pcarbon_pom, icol); + + progs.n_mode_c[accum_idx ] = ekat::subview(nc_accum, icol); + + progs.n_mode_i[accum_idx ] = ekat::subview(ni_accum, icol); + progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); + + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = ekat::subview(activation_fraction_coarse_idx, icol); + + diags.hetfrz_immersion_nucleation_tend = ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t=0, dt=0; + hetfrz_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); + + }); + /* call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, & ! in qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index c130ff06d862..c5ed6a5a90c7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -15,10 +15,12 @@ namespace scream class MAMAci final : public scream::AtmosphereProcess { +public: using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; + mam4::Hetfrz hetfrz_; // views for single- and multi-column data using view_1d = typename KT::template view_1d; @@ -33,6 +35,7 @@ class MAMAci final : public scream::AtmosphereProcess { // FIXME the time step for microphysics [s] need to get from the input const Real dtmicro_ = .0001; +private: // rho is air density [kg/m3] view_2d rho_; @@ -43,21 +46,48 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d tke_; const_view_2d qv_dry_; - view_2d cldfrac_; - //const_view_2d w_updraft_; - view_2d w_updraft_; + const_view_2d cldfrac_; + const_view_2d w_updraft_; view_2d aitken_dry_dia_; - view_2d q_coarse_dst_; - view_2d q_coarse_nacl_; - view_2d q_coarse_so4_; - view_2d q_coarse_mom_; - view_2d q_coarse_bc_; - view_2d q_coarse_pom_; - view_2d q_coarse_soa_; - view_2d n_coarse_; - view_2d n_aitken_; - view_3d dgnum_; + view_2d qc_coarse_bc_; + view_2d qc_coarse_dst_; + view_2d qc_coarse_mom_; + view_2d qc_coarse_nacl_; + view_2d qc_coarse_pom_; + view_2d qc_coarse_soa_; + + view_2d qc_accum_bc_; + view_2d qc_accum_dst_; + view_2d qc_accum_mom_; + view_2d qc_accum_nacl_; + view_2d qc_accum_pom_; + view_2d qc_accum_so4_; + view_2d qc_accum_soa_; + + view_2d qi_accum_dst_; + view_2d qi_accum_so4_; + view_2d qi_accum_mom_; + view_2d qi_accum_bc_; + view_2d qi_accum_pom_; + view_2d qi_accum_soa_; + + view_2d qi_coarse_dst_; + view_2d qi_coarse_nacl_; + view_2d qi_coarse_so4_; + view_2d qi_coarse_mom_; + view_2d qi_coarse_bc_; + view_2d qi_coarse_pom_; + view_2d qi_coarse_soa_; + + view_2d qi_pcarbon_bc_; + view_2d qi_pcarbon_mom_; + view_2d qi_pcarbon_pom_; + view_2d nc_accum_; + view_2d ni_accum_; + view_2d ni_coarse_; + view_2d ni_aitken_; + const_view_3d dgnum_; view_2d nihf_; view_2d niim_; view_2d nidep_; @@ -67,6 +97,7 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d qc_; const_view_2d qi_; + const_view_2d nc_; const_view_2d kvh_; view_2d cldo_; @@ -79,9 +110,9 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d qcld_; view_2d tendnd_; view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; - view_3d ptend_q_inp_; + view_3d ptend_q_output_; view_3d factnum_; - view_3d qqcw_inp_; + const_view_3d qqcw_input_; view_2d qqcw_[mam4::ndrop::ncnst_tot]; view_2d ndropcol_; view_2d ndropmix_; @@ -96,8 +127,16 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d raercol_[mam4::ndrop::pver][2]; view_3d nact_; view_3d mact_; - view_2d scratch_mem_[15]; + view_2d dropmixnuc_scratch_mem_[15]; + + view_2d stratiform_cloud_fraction_; + view_2d activation_fraction_accum_idx_; + view_2d activation_fraction_coarse_idx_; + view_2d hetfrz_immersion_nucleation_tend_; + view_2d hetfrz_contact_nucleation_tend_; + view_2d hetfrz_depostion_nucleation_tend_; + view_2d diagnostic_scratch_[42]; // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; From 83ba683df32baa756e6797c420b8e341218f5a9c Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 25 Jan 2024 15:30:34 -0700 Subject: [PATCH 250/476] General code clean up by using structs from mam_coupling.hpp --- .../mam/eamxx_mam_aci_process_interface.cpp | 1329 ++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 107 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + .../eamxx/tests/uncoupled/mam4/input_aci.yaml | 46 +- 4 files changed, 707 insertions(+), 776 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 1fb52659ac5f..91de9d4a03aa 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -28,9 +28,8 @@ void compute_w0_and_rho( haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, MAMAci::view_2d rho, - MAMAci::const_view_2d p_mid, - MAMAci::const_view_2d T_mid, - MAMAci::const_view_2d omega, + mam_coupling::WetAtmosphere &wet_atmosphere, + mam_coupling::DryAtmosphere &dry_atmosphere, const int top_lev, const int nlev) { @@ -38,6 +37,9 @@ void compute_w0_and_rho( using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + MAMAci::const_view_2d omega = wet_atmosphere.omega; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { @@ -175,466 +177,28 @@ void compute_aitken_dry_diameter( } void compute_nucleate_ice_tendencies( + const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, - const int nlev) + MAMAci::view_2d nihf, + MAMAci::view_2d niim, + MAMAci::view_2d nidep, + MAMAci::view_2d nimey, + MAMAci::view_2d naai_hom, + MAMAci::view_2d naai, + mam_coupling::AerosolState &aerosol_state, + mam_coupling::DryAtmosphere &dry_atmosphere, + MAMAci::view_2d aitken_dry_dia, + const int nlev) { -} - -} - - -//FIXME: The following variables are namelist variables -const Real wsubmin = 1; - -MAMAci::MAMAci( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params){ -} - - -//Return type of the process -AtmosphereProcessType MAMAci::type() const{ - return AtmosphereProcessType::Physics; -} - -//return name of the process -std::string MAMAci::name() const{ - return "mam4_aci"; - } - -//set grid for all the inputs and outputs -void MAMAci::set_grids(const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); - - grid_ = grids_manager->get_grid("Physics"); //Use physics grid - const auto& grid_name = grid_->name(); - - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_+1); - Kokkos::resize(wsub_, ncol_, nlev_); - Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(lcldn_, ncol_, nlev_); - Kokkos::resize(lcldo_, ncol_, nlev_); - Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); - Kokkos::resize(rpdel_, ncol_, nlev_); - - for (int i=0; i<15; ++i) { - Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); - } - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - - // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points - FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces - - using namespace ekat::units; - auto q_unit = kg/kg; // units of mass mixing ratios of tracers - q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); - auto nondim = Units::nondimensional(); - // atmospheric quantities - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints - add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // - // - add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - - - // FIXME: These should all come from the intput file. Might need to be added to the input file? - add_field("qi_coarse_dst", scalar3d_layout_mid, q_unit, grid_name); // Dust mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_nacl", scalar3d_layout_mid, q_unit, grid_name); // Salt mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_so4", scalar3d_layout_mid, q_unit, grid_name); // Sulfuric Acid mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for coarse mode [kg/kg] - add_field("qi_coarse_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for coarse mode [kg/kg] - //add_field("qi_accum_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for accum mode [kg/kg] - //add_field("qi_accum_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for accum mode [kg/kg] - //add_field("qi_accum_soa", scalar3d_layout_mid, q_unit, grid_name); // Secondary Organic Aerosol mixing ratio for accum mode [kg/kg] - //add_field("qi_accumn_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] - //add_field("qi_accumn_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] - //add_field("qi_accumn_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] - //add_field("qi_pcarbon_bc", scalar3d_layout_mid, q_unit, grid_name); // Black Carbon mixing ratio for primary carbon mode [kg/kg] - //add_field("qi_pcarbon_mom", scalar3d_layout_mid, q_unit, grid_name); // Marine Organic Matter mixing ratio for primary carbon mode [kg/kg]] - //add_field("qi_pcarbon_pom", scalar3d_layout_mid, q_unit, grid_name); // Primary Organic Matter mixing ratio for primary carbon mode [kg/kg]kg] - //add_field("ni_accum", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] - //add_field("ni_coarse", scalar3d_layout_mid, 1/kg, grid_name); // Coarse mode number mixing ratio [1/kg dry air] - //add_field("ni_aitken", scalar3d_layout_mid, 1/kg, grid_name); // Aitken mode number mixing ratio [1/kg dry air] - //add_field("zm", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) - //add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] - //add_field("ncldwtr", , n_unit, grid_name); // initial droplet number mixing ratio [#/kg] - //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] - // - add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] - - - add_field("nihf",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] - add_field("niim",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] - add_field("nidep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] - add_field("nimey",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] - add_field("naai_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] - add_field("naai",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] - add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] - add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] - add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::AeroConfig::num_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] - add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] - add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] - // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output - add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency - - - //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - //cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } - - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); - } - } - } - - //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - //These outputs should come from the cloud macrophysics process (e.g., SHOC) - auto m2 = m*m; - m2.set_string("m^2"); - auto s2 = s*s; - s2.set_string("s^2"); - - //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time - add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints - - add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints - add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints - add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat - - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - num_aero_modes_ = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes_} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols - - - /*NOTE on other inputs for the aci process: - 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density - 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using - pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). - qv_mid can be obtained from "get_field_in" call*/ - -} - -void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); - /* - NOTE: All other inputs should follow the way "pseudo_density" is initialized - */ - // set atmosphere state data - pdel_ = get_field_in("pseudo_density").get_view(); - omega_ = get_field_in("omega").get_view(); - p_mid_ = get_field_in("p_mid").get_view(); - p_int_ = get_field_in("p_int").get_view(); - T_mid_ = get_field_in("T_mid").get_view(); - w_sec_ = get_field_in("w_sec").get_view(); - qv_dry_ = get_field_in("qv").get_view(); - w_updraft_ = get_field_in("w_updraft").get_view(); // updraft velocity [m/s] - // - stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); - activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); - activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); - - // FIXME: These should all come from the intput file. Might need to be added to the input file? - Kokkos::resize(qc_accum_dst_, ncol_, nlev_); // qc_accum_dst_ = get_field_in("qc_accum_dst").get_view(); - Kokkos::resize(qc_accum_so4_, ncol_, nlev_); // qc_accum_so4_ = get_field_in("qc_accum_so4").get_view(); - Kokkos::resize(qc_accum_mom_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); - Kokkos::resize(qc_accum_nacl_, ncol_, nlev_); // qc_accum_mom_ = get_field_in("qc_accum_mom").get_view(); - Kokkos::resize(qc_accum_bc_, ncol_, nlev_); // qc_accum_bc_ = get_field_in("qc_accum_bc").get_view(); - Kokkos::resize(qc_accum_pom_, ncol_, nlev_); // qc_accum_pom_ = get_field_in("qc_accum_pom").get_view(); - Kokkos::resize(qc_accum_soa_, ncol_, nlev_); // qc_accum_soa_ = get_field_in("qc_accum_soa").get_view(); - Kokkos::resize(qi_accum_dst_, ncol_, nlev_); // qi_accum_dst_ = get_field_in("qi_accum_dst").get_view(); - Kokkos::resize(qi_accum_so4_, ncol_, nlev_); // qi_accum_so4_ = get_field_in("qi_accum_so4").get_view(); - Kokkos::resize(qi_accum_mom_, ncol_, nlev_); // qi_accum_mom_ = get_field_in("qi_accum_mom").get_view(); - Kokkos::resize(qi_accum_bc_, ncol_, nlev_); // qi_accum_bc_ = get_field_in("qi_accum_bc").get_view(); - Kokkos::resize(qi_accum_pom_, ncol_, nlev_); // qi_accum_pom_ = get_field_in("qi_accum_pom").get_view(); - Kokkos::resize(qi_accum_soa_, ncol_, nlev_); // qi_accum_soa_ = get_field_in("qi_accum_soa").get_view(); - Kokkos::resize(qc_coarse_dst_, ncol_, nlev_); // qc_coarse_dst_ = get_field_in("qc_coarse_dst").get_view(); - Kokkos::resize(qc_coarse_nacl_, ncol_, nlev_); // qc_coarse_nacl_ = get_field_in("qc_coarse_nacl").get_view(); - Kokkos::resize(qc_coarse_mom_, ncol_, nlev_); // ci_coarse_mom_ = get_field_in("qc_coarse_mom").get_view(); - Kokkos::resize(qc_coarse_bc_, ncol_, nlev_); // qc_coarse_bc_ = get_field_in("qc_coarse_bc").get_view(); - Kokkos::resize(qc_coarse_pom_, ncol_, nlev_); // qc_coarse_pom_ = get_field_in("qc_coarse_pom").get_view(); - Kokkos::resize(qc_coarse_soa_, ncol_, nlev_); // qc_coarse_soa_ = get_field_in("qc_coarse_soa").get_view(); - // - qi_coarse_dst_ = get_field_in("qi_coarse_dst").get_view(); - qi_coarse_nacl_ = get_field_in("qi_coarse_nacl").get_view(); - qi_coarse_so4_ = get_field_in("qi_coarse_so4").get_view(); - qi_coarse_mom_ = get_field_in("qi_coarse_mom").get_view(); - qi_coarse_bc_ = get_field_in("qi_coarse_bc").get_view(); - qi_coarse_pom_ = get_field_in("qi_coarse_pom").get_view(); - qi_coarse_soa_ = get_field_in("qi_coarse_soa").get_view(); - - Kokkos::resize(qi_pcarbon_bc_, ncol_, nlev_); // qi_pcarbon_bc_ = get_field_in("qi_pcarbon_bc").get_view(); - Kokkos::resize(qi_pcarbon_mom_, ncol_, nlev_); // qi_pcarbon_mom_ = get_field_in("qi_pcarbon_mom").get_view(); - Kokkos::resize(qi_pcarbon_pom_, ncol_, nlev_); // qi_pcarbon_pom_ = get_field_in("qi_pcarbon_pom").get_view(); - Kokkos::resize(nc_accum_, ncol_, nlev_); // nc_accum_ = get_field_in("nc_accum").get_view(); - Kokkos::resize(ni_accum_, ncol_, nlev_); // ni_accum_ = get_field_in("ni_accum").get_view(); - Kokkos::resize(ni_coarse_, ncol_, nlev_); // ni_coarse_ = get_field_in("ni_coarse").get_view(); - Kokkos::resize(ni_aitken_, ncol_, nlev_); // ni_aitken_ = get_field_in("ni_aitken").get_view(); - Kokkos::resize(zm_, ncol_, nlev_); // zm_ = get_field_in("zm").get_view(); - Kokkos::resize(state_q_, ncol_, nlev_, mam4::ndrop::nvars); // state_q_ = get_field_in("state_q").get_view(); - Kokkos::resize(ncldwtr_, ncol_, nlev_); // ncldwtr_ = get_field_in("ncldwtr").get_view(); - Kokkos::resize(cldo_, ncol_, nlev_); // cldo_ = get_field_in("cldo").get_view(); - qqcw_input_ = get_field_in("qqcw").get_view(); - - for (int i=0; i(); - dgnum_ = get_field_out("dgnum").get_view(); - nihf_ = get_field_out("nihf").get_view(); - niim_ = get_field_out("niim").get_view(); - nidep_ = get_field_out("nidep").get_view(); - nimey_ = get_field_out("nimey").get_view(); - naai_hom_ = get_field_out("naai_hom").get_view(); - naai_ = get_field_out("naai").get_view(); - qcld_ = get_field_out("qcld").get_view(); - ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("tendnd").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); - ccn_ = get_field_out("ccn").get_view(); - coltend_outp_ = get_field_out("coltend").get_view(); - coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - - - - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - qc_ = get_field_in("qc").get_view(); - qi_ = get_field_in("qi").get_view(); - nc_ = get_field_in("nc").get_view(); - kvh_ = get_field_in("kvh").get_view(); - - - mam4::AeroConfig aero_config; - // configure the nucleation parameterization - mam4::NucleateIce::Config nucleate_ice_config; - nucleate_ice_.init(aero_config, nucleate_ice_config); - - // configure the heterogeneous freezing parameterization - mam4::Hetfrz::Config hetfrz_config; - hetfrz_.init(aero_config, hetfrz_config); - /* - NOTE: All derived variables (like rpdel and geopotential height) should be computed in - preprocess struct - */ - //preprocess_.set_variables(pdel_); - -} - -void MAMAci::run_impl(const double dt) { - m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI run"); - - //---------------------------------------------------------- - // Convert from omega to w (vertical velocity) - // Negative omega means rising motion - //--------------------------------------------------------- - - - // Alias member variables - auto rho = rho_; - auto tke = tke_; - auto T_mid = T_mid_; - auto p_mid = p_mid_; - auto p_int = p_int_; - auto pdel = pdel_; - auto wsub = wsub_; - auto wsubice = wsubice_; - auto qv_dry = qv_dry_; - auto cldfrac = cldfrac_; - auto w_updraft = w_updraft_; - const auto nlev = nlev_; - const auto top_lev = top_lev_; - - auto qc_accum_dst = qc_accum_dst_; - auto qc_accum_so4 = qc_accum_so4_; - auto qc_accum_mom = qc_accum_mom_; - auto qc_accum_nacl = qc_accum_nacl_; - auto qc_accum_bc = qc_accum_bc_; - auto qc_accum_pom = qc_accum_pom_; - auto qc_accum_soa = qc_accum_soa_; - - auto qi_accum_dst = qi_accum_dst_; - auto qi_accum_so4 = qi_accum_so4_; - auto qi_accum_mom = qi_accum_mom_; - auto qi_accum_bc = qi_accum_bc_; - auto qi_accum_pom = qi_accum_pom_; - auto qi_accum_soa = qi_accum_soa_; - - auto qc_coarse_dst = qc_coarse_dst_; - auto qc_coarse_nacl = qc_coarse_nacl_; - auto qc_coarse_mom = qc_coarse_mom_; - auto qc_coarse_bc = qc_coarse_bc_; - auto qc_coarse_pom = qc_coarse_pom_; - auto qc_coarse_soa = qc_coarse_soa_; - auto qi_coarse_dst = qi_coarse_dst_; - auto qi_coarse_nacl = qi_coarse_nacl_; - auto qi_coarse_so4 = qi_coarse_so4_; - auto qi_coarse_mom = qi_coarse_mom_; - auto qi_coarse_bc = qi_coarse_bc_; - auto qi_coarse_pom = qi_coarse_pom_; - auto qi_coarse_soa = qi_coarse_soa_; - auto qi_pcarbon_bc = qi_pcarbon_bc_; - auto qi_pcarbon_mom = qi_pcarbon_mom_; - auto qi_pcarbon_pom = qi_pcarbon_pom_; - auto nc_accum = nc_accum_; - auto ni_accum = ni_accum_; - auto ni_coarse = ni_coarse_; - auto ni_aitken = ni_aitken_; - auto nihf = nihf_; - auto niim = niim_; - auto nidep = nidep_; - auto nimey = nimey_; - auto naai_hom = naai_hom_; - auto naai = naai_; - auto liqcldf = liqcldf_; - auto qc = qc_; - auto qi = qi_; - auto nc = nc_; - auto lcldn = lcldn_; - auto lcldo = lcldo_; - auto aitken_dry_dia = aitken_dry_dia_; - auto rpdel = rpdel_; - auto zm = zm_; - auto state_q = state_q_; - auto ncldwtr = ncldwtr_; - auto kvh = kvh_; - auto qcld = qcld_; - auto tendnd = tendnd_; - auto factnum = factnum_; - auto ndropcol = ndropcol_ ; - auto ndropmix = ndropmix_ ; - auto nsource = nsource_ ; - auto wtke = wtke_ ; - auto cldo = cldo_; - auto ccn = ccn_; - auto coltend_outp = coltend_outp_; - auto coltend_cw_outp = coltend_cw_outp_; - auto nact = nact_; - auto mact = mact_; - auto eddy_diff = dropmixnuc_scratch_mem_[0]; - auto zn = dropmixnuc_scratch_mem_[1]; - auto csbot = dropmixnuc_scratch_mem_[2]; - auto zs = dropmixnuc_scratch_mem_[3]; - auto overlapp = dropmixnuc_scratch_mem_[4]; - auto overlapm = dropmixnuc_scratch_mem_[5]; - auto eddy_diff_kp = dropmixnuc_scratch_mem_[6]; - auto eddy_diff_km = dropmixnuc_scratch_mem_[7]; - auto qncld = dropmixnuc_scratch_mem_[8]; - auto srcn = dropmixnuc_scratch_mem_[9]; - auto source = dropmixnuc_scratch_mem_[10]; - auto dz = dropmixnuc_scratch_mem_[11]; - auto csbot_cscen = dropmixnuc_scratch_mem_[12]; - auto raertend = dropmixnuc_scratch_mem_[13]; - auto qqcwtend = dropmixnuc_scratch_mem_[14]; - - auto hetfrz_immersion_nucleation_tend = hetfrz_immersion_nucleation_tend_; - auto hetfrz_contact_nucleation_tend = hetfrz_contact_nucleation_tend_; - auto hetfrz_depostion_nucleation_tend = hetfrz_depostion_nucleation_tend_; - auto stratiform_cloud_fraction = stratiform_cloud_fraction_; - auto activation_fraction_accum_idx = activation_fraction_accum_idx_; - auto activation_fraction_coarse_idx = activation_fraction_coarse_idx_; - auto diagnostic_scratch = diagnostic_scratch_; - - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - - copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_, mam4::ndrop::ncnst_tot); - - // All the inputs are available to compute w0 and rho - compute_w0_and_rho(team_policy, w0_, rho_, p_mid_, T_mid_, omega_, top_lev_, nlev_); - - compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); - - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); - - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); - - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); - - compute_nucleate_ice_tendencies(team_policy, nlev_); - //T_mid_, p_mid_, qv_dry_, cldfrac_, w_updraft_ - // qi_coarse_dst_, qi_coarse_nacl_, qi_coarse_so4_, qi_coarse_mom_, qi_coarse_bc_, qi_coarse_pom_, qi_coarse_soa_, - //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation //------------------------------------------------------------- + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; + MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; + MAMAci::const_view_2d w_updraft = dry_atmosphere.w_updraft; using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -652,29 +216,17 @@ void MAMAci::run_impl(const double dt) { ekat::subview(w_updraft, icol), pblh); // set surface state data haero::Surface surf{}; - mam4::Prognostics progs(nlev); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - const int dst_idx = static_cast(mam4::AeroId::DST); - - progs.q_aero_i[coarse_idx][dst_idx] = ekat::subview(qi_coarse_dst, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); - progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); - progs.n_mode_i[aitken_idx] = ekat::subview(ni_aitken, icol); + mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); // nucleation doesn't use any diagnostics, so it's okay to leave this alone // for now mam4::Diagnostics diags(nlev); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); // These are the fields that are updated. Taking subviews means that // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated - // in nucleate_ice_.compute_tendencies. + // in nucleate_ice.compute_tendencies. diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); diags.icenuc_num_immfrz = ekat::subview(niim, icol); diags.icenuc_num_depnuc = ekat::subview(nidep, icol); @@ -688,8 +240,6 @@ void MAMAci::run_impl(const double dt) { const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t=0, dt=0; - nucleate_ice_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - /* NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". Only output we care about is "naai", "naai_hom" is never used anywhere @@ -699,196 +249,162 @@ void MAMAci::run_impl(const double dt) { rho, wsubice, strat_cld_frac, dgnum, & ! input naai, naai_hom) ! output */ + nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); - //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud - // is above qsmall threshold value - //------------------------------------------------------------- - - const Real dtmicro = dtmicro_; - static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) - - /* - MUST FIXME NOTE: We need old and new liquid cloud fractions here. - We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to - store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make - a note of it and use the new cloud fraction for the - old cloud fraction. - */ +} +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d liqcldf, + const int top_lev) +{ + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + + // MUST FIXME NOTE: We need old and new liquid cloud fractions here. + // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to + // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make + // a note of it and use the new cloud fraction for the + // old cloud fraction. + //------------------------------------------------------------- + MAMAci::const_view_2d qc = wet_atmosphere.qc; + MAMAci::const_view_2d qi = wet_atmosphere.qi; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); + static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { const Real qcld = qc(icol,kk) + qi(icol,kk); if (qcld > qsmall) { - lcldn(icol,kk)=liqcldf(icol,kk); - lcldo(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + cloud_frac_new(icol,kk)=liqcldf(icol,kk); + cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + } else { + cloud_frac_new(icol,kk)=0; + cloud_frac_old(icol,kk)=0; } }); }); - /* - Fortran code: - liqcldf(:ncol,:pver) = liq_strat_cld_frac(:ncol,:pver) - lcldn = 0._r8 - lcldo = 0._r8 - do kk = top_lev, pver - do icol = 1, ncol - qcld = qc(icol,kk) + qi(icol,kk) - if (qcld > qsmall) then - lcldn(icol,kk)=liqcldf(icol,kk) - lcldo(icol,kk)=liqcldfo(icol,kk) - end if - end do - end do - */ - - //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. - //------------------------------------------------------------- +} +void compute_recipical_pseudo_density( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int nlev) +{ + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + rpdel(icol,kk) = 1/pdel(icol,kk); + }); + }); +} - /*NOTE: We probably need to store indices for the select few - cloud borne aerosols - - Fortran code: - lchnk_zb = lchnk - begchunk - ! save copy of cloud borne aerosols for use in heterogeneous freezing before - !we change it in dropmixnuc - do ispec = 1, ncnst - call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) - aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) - aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) - enddo - */ +void call_function_dropmixnuc( + haero::ThreadTeamPolicy team_policy, + mam_coupling::DryAtmosphere &dry_atmosphere, + const Real dtmicro, + MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], + MAMAci::view_2d raercol[mam4::ndrop::pver][2], + MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + MAMAci::const_view_2d p_int, + MAMAci::const_view_2d pdel, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d zm, + MAMAci::const_view_3d state_q, + MAMAci::const_view_2d ncldwtr, + MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, + MAMAci::view_2d wsub, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + MAMAci::view_2d tendnd, + MAMAci::view_3d factnum, + MAMAci::view_2d ndropcol, + MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, + MAMAci::view_2d wtke, + MAMAci::view_3d ccn, + MAMAci::view_3d nact, + MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[15], + const int nlev) +{ + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); - //------------------------------------------------------------- - // Compute activated fraction of aerosols - //------------------------------------------------------------- - const int ntot_amode = mam4::AeroConfig::num_modes(); + // Initialize the ndrop class. + const int ntot_amode = mam_coupling::num_aero_modes(); const int maxd_aspectype = mam4::ndrop::maxd_aspectype; const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode]; - int lspectype_amode[maxd_aspectype][ntot_amode]; - int lmassptr_amode[maxd_aspectype][ntot_amode]; - Real specdens_amode[maxd_aspectype]; - Real spechygro[maxd_aspectype]; - int numptr_amode[ntot_amode]; - int mam_idx[ntot_amode][nspec_max]; - int mam_cnst_idx[ntot_amode][nspec_max]; + int nspec_amode[ntot_amode] = {}; + int lspectype_amode[maxd_aspectype][ntot_amode] = {}; + int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; + int numptr_amode[ntot_amode] = {}; + int mam_idx[ntot_amode][nspec_max] = {}; + int mam_cnst_idx[ntot_amode][nspec_max] = {}; + Real exp45logsig[ntot_amode] = {}, + alogsig[ntot_amode] = {}, + num2vol_ratio_min_nmodes[ntot_amode] = {}, + num2vol_ratio_max_nmodes[ntot_amode] = {}; + Real aten = 0; + mam4::ndrop::get_e3sm_parameters( nspec_amode, lspectype_amode, lmassptr_amode, numptr_amode, specdens_amode, spechygro, mam_idx, mam_cnst_idx); - Real exp45logsig[ntot_amode], - alogsig[ntot_amode], - num2vol_ratio_min_nmodes[ntot_amode], - num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, // voltonumbhi_amode num2vol_ratio_max_nmodes); // voltonumblo_amode - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - rpdel(icol,kk) = 1/pdel(icol,kk); - }); - }); - - - - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - mam4::ndrop::View1D raercol_cw[mam4::ndrop::pver][2]; - mam4::ndrop::View1D raercol[mam4::ndrop::pver][2]; - for (int i=0; i::template view_1d; + view_1d dummy("DummyView", nlev); + MAMAci::const_view_2d qc = wet_atmosphere.qc; + MAMAci::const_view_2d nc = wet_atmosphere.nc; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -925,55 +524,18 @@ void MAMAci::run_impl(const double dt) { pblh); // set surface state data haero::Surface surf{}; - mam4::Prognostics progs(nlev); + mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - const int pcarbon_idx = static_cast(mam4::ModeIndex::PrimaryCarbon); - - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_accum_bc, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_accum_dst, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_accum_mom, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_accum_pom, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_accum_nacl, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qc_accum_so4, icol); - progs.q_aero_c[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_accum_soa, icol); - - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_accum_bc, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_accum_dst, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_accum_mom, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_accum_pom, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_accum_so4, icol); - progs.q_aero_i[accum_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_accum_soa, icol); - - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qc_coarse_bc, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qc_coarse_dst, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qc_coarse_mom, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qc_coarse_nacl, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qc_coarse_pom, icol); - progs.q_aero_c[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qc_coarse_soa, icol); - - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_coarse_bc, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::DST)] = ekat::subview(qi_coarse_dst, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_coarse_mom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::NaCl)] = ekat::subview(qi_coarse_nacl, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_coarse_pom, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SO4)] = ekat::subview(qi_coarse_so4, icol); - progs.q_aero_i[coarse_idx][static_cast(mam4::AeroId::SOA)] = ekat::subview(qi_coarse_soa, icol); - - progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::BC)] = ekat::subview(qi_pcarbon_bc, icol); - progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::MOM)] = ekat::subview(qi_pcarbon_mom, icol); - progs.q_aero_i[pcarbon_idx][static_cast(mam4::AeroId::POM)] = ekat::subview(qi_pcarbon_pom, icol); - - progs.n_mode_c[accum_idx ] = ekat::subview(nc_accum, icol); - - progs.n_mode_i[accum_idx ] = ekat::subview(ni_accum, icol); - progs.n_mode_i[coarse_idx] = ekat::subview(ni_coarse, icol); mam4::Diagnostics diags(nlev); diags.stratiform_cloud_fraction = ekat::subview(stratiform_cloud_fraction, icol); diags.activation_fraction[accum_idx] = ekat::subview(activation_fraction_accum_idx, icol); diags.activation_fraction[coarse_idx] = ekat::subview(activation_fraction_coarse_idx, icol); + // These are the output tendencies from heterogeneous freezing that need to be + // added correctly to the cloud-micorphysics scheme. diags.hetfrz_immersion_nucleation_tend = ekat::subview(hetfrz_immersion_nucleation_tend, icol); diags.hetfrz_contact_nucleation_tend = ekat::subview(hetfrz_contact_nucleation_tend, icol); diags.hetfrz_depostion_nucleation_tend = ekat::subview(hetfrz_depostion_nucleation_tend, icol); @@ -1025,20 +587,393 @@ void MAMAci::run_impl(const double dt) { diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // // grab views from the buffer to store tendencies, not used as all values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t=0, dt=0; - hetfrz_.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - + hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); - - /* - call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, & ! in - qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in - frzimm, frzcnt, frzdep) - */ +} +} + +//FIXME: The following variables are namelist variables +const Real wsubmin = 1; + +MAMAci::MAMAci( + const ekat::Comm& comm, + const ekat::ParameterList& params) + : AtmosphereProcess(comm, params){ +} + + +//Return type of the process +AtmosphereProcessType MAMAci::type() const{ + return AtmosphereProcessType::Physics; +} + +//return name of the process +std::string MAMAci::name() const{ + return "mam4_aci"; + } + +//set grid for all the inputs and outputs +void MAMAci::set_grids(const std::shared_ptr grids_manager) { + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); + + grid_ = grids_manager->get_grid("Physics"); //Use physics grid + const auto& grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_+1); + Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(wsubice_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(cloud_frac_new_, ncol_, nlev_); + Kokkos::resize(cloud_frac_old_, ncol_, nlev_); + Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); + + for (int i=0; i<15; ++i) { + Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); + } + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variables + FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points + FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces + + using namespace ekat::units; + auto q_unit = kg/kg; // units of mass mixing ratios of tracers + q_unit.set_string("kg/kg"); + auto n_unit = 1/kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + auto nondim = Units::nondimensional(); + // atmospheric quantities + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints + add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints + add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces + add_field ("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // + // + add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + + add_field ("z_mid", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) + add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] + add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] + //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] + // + add_field ("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] + add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] + + add_field("icenuc_num_hetfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] + add_field("icenuc_num_immfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] + add_field("icenuc_num_depnuc",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + add_field("icenuc_num_meydep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + add_field("num_act_aerosol_ice_nucle_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] + add_field("num_act_aerosol_ice_nucle",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] + add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] + add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] + add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] + add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam_coupling::num_aero_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] + add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] + add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // + add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output + add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency + + + //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + //interstitial aerosol tracers of interest: number (n) mixing ratios + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + //cloudborne aerosol tracers of interest: number (n) mixing ratios + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); + if (strlen(cld_mmr_field_name) > 0) + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + } + } + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } + + //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM + //These outputs should come from the cloud macrophysics process (e.g., SHOC) + auto m2 = m*m; + m2.set_string("m^2"); + auto s2 = s*s; + s2.set_string("s^2"); + + //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time + add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + + add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints + add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints + add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat + add_field("zm", scalar3d_layout_int, m, grid_name); // Eddy diffusivity for heat + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes} }; // mid points + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols + + + auto cm = m/100; + auto frz_unit = 1/(cm*cm*cm*s); // units of number mixing ratios of tracers + n_unit.set_string("1(cm^-3 s^-1)"); + + add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + // + /* + * NOTE on other inputs for the aci process: + * 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density + * 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using + * pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). + * qv_mid can be obtained from "get_field_in" call + */ + +} + +void MAMAci::initialize_impl(const RunType run_type) { + m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); + + // set atmosphere state data + pdel_ = get_field_in("pseudo_density").get_view(); + p_int_ = get_field_in("p_int").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); + state_q_ = get_field_in("state_q").get_view(); + ncldwtr_ = get_field_in("ncldwtr").get_view(); + qqcw_input_ = get_field_in("qqcw").get_view(); + dgnum_ = get_field_in("dgnum").get_view(); + nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); + niim_ = get_field_out("icenuc_num_immfrz").get_view(); + nidep_ = get_field_out("icenuc_num_depnuc").get_view(); + nimey_ = get_field_out("icenuc_num_meydep").get_view(); + naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); + naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); + qcld_ = get_field_out("qcld").get_view(); + ptend_q_output_ = get_field_out("ptend_q").get_view(); + tendnd_ = get_field_out("tendnd").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + ccn_ = get_field_out("ccn").get_view(); + coltend_outp_ = get_field_out("coltend").get_view(); + coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + kvh_ = get_field_in("kvh").get_view(); + zm_ = get_field_in("zm").get_view(); + + wet_atmosphere_.qc = get_field_in("qc").get_view(); + wet_atmosphere_.nc = get_field_in("nc").get_view(); + wet_atmosphere_.qi = get_field_in("qi").get_view(); + wet_atmosphere_.omega = get_field_in("omega").get_view(); + + dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); + dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.qv = get_field_out("qv").get_view(); + dry_atmosphere_.z_mid = get_field_out("z_mid").get_view(); + dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); + + hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); + hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + + stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); + activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); + activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); + + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + auto prog_index = [](const int mode, const int species) { + const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); + const int ind = + (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; + return ind; + }; + //interstitial aerosol tracers of interest: number (n) mixing ratios + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + aerosol_state_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + + //cloudborne aerosol tracers of interest: number (n) mixing ratios + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + aerosol_state_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + const int index = prog_index(m, a); + aerosol_state_.int_aero_mmr[m][index] = get_field_out(int_mmr_field_name).get_view(); + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); + if (strlen(cld_mmr_field_name) > 0) { + const int index = prog_index(m, a); + aerosol_state_.cld_aero_mmr[m][index] = get_field_out(cld_mmr_field_name).get_view(); + } + } + } + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + aerosol_state_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + } + + for (int i=0; ilog(ekat::logger::LogLevel::info,"calling ACI run"); + + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + + copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_, mam4::ndrop::ncnst_tot); + + // All the inputs are available to compute w0 and rho + // Convert from omega to w (vertical velocity) + // Negative omega means rising motion + compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + + compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); + Kokkos::fence(); // wait for for tke_ to be computed. + + compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); + Kokkos::fence(); // wait for wsig_ to be computed. + + compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); + + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); + Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + + compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, + nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, + aerosol_state_, dry_atmosphere_, aitken_dry_dia_, nlev_); + + store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atmosphere_, liqcldf_, top_lev_); + + //------------------------------------------------------------- + // Save cloud borne aerosols to be used in the heterozenous + // freezing before they are changed by the droplet activation + // process. This is only a select subset of cloud borne + // aerosols, not all the cloud borne aerosols. + //------------------------------------------------------------- + /*NOTE: We probably need to store indices for the select few cloud borne aerosols + Fortran code: + lchnk_zb = lchnk - begchunk + ! save copy of cloud borne aerosols for use in heterogeneous freezing before + ! we change it in dropmixnuc + do ispec = 1, ncnst + call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) + aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) + aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) + enddo + */ + + compute_recipical_pseudo_density(team_policy, rpdel_, pdel_, nlev_); + Kokkos::fence(); // wait for rpdel_ to be computed. + + call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, + raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, + p_int_, pdel_, rpdel_, zm_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + Kokkos::fence(); // wait for ptend_q_ to be computed. + + copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, mam4::ndrop::nvar_ptend_q, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, mam4::ndrop::ncnst_tot, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, mam4::ndrop::ncnst_tot, nlev_); + + call_hetfrz_compute_tendencies(team_policy, + hetfrz_, aerosol_state_, wet_atmosphere_, dry_atmosphere_, + stratiform_cloud_fraction_, + activation_fraction_accum_idx_, activation_fraction_coarse_idx_, + hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, + naai_hom_, naai_, + diagnostic_scratch_, + nlev_); + + Kokkos::fence(); // wait before returning to calling function } void MAMAci::finalize_impl(){ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index c5ed6a5a90c7..046d8578112b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -23,11 +23,12 @@ class MAMAci final : public scream::AtmosphereProcess { mam4::Hetfrz hetfrz_; // views for single- and multi-column data - using view_1d = typename KT::template view_1d; - using const_view_2d = typename KT::template view_2d; - using view_2d = typename KT::template view_2d; - using view_3d = typename KT::template view_3d; - using const_view_3d = typename KT::template view_3d; + using view_1d = scream::mam_coupling::view_1d; + using view_2d = scream::mam_coupling::view_2d; + using view_3d = scream::mam_coupling::view_3d; + using const_view_1d = scream::mam_coupling::const_view_1d; + using const_view_2d = scream::mam_coupling::const_view_2d; + using const_view_3d = scream::mam_coupling::const_view_3d; template using view_4d = KT::view; @@ -45,48 +46,15 @@ class MAMAci final : public scream::AtmosphereProcess { // turbulent kinetic energy [m^2/s^2] view_2d tke_; - const_view_2d qv_dry_; - const_view_2d cldfrac_; - const_view_2d w_updraft_; - view_2d aitken_dry_dia_; - view_2d qc_coarse_bc_; - view_2d qc_coarse_dst_; - view_2d qc_coarse_mom_; - view_2d qc_coarse_nacl_; - view_2d qc_coarse_pom_; - view_2d qc_coarse_soa_; - - view_2d qc_accum_bc_; - view_2d qc_accum_dst_; - view_2d qc_accum_mom_; - view_2d qc_accum_nacl_; - view_2d qc_accum_pom_; - view_2d qc_accum_so4_; - view_2d qc_accum_soa_; - - view_2d qi_accum_dst_; - view_2d qi_accum_so4_; - view_2d qi_accum_mom_; - view_2d qi_accum_bc_; - view_2d qi_accum_pom_; - view_2d qi_accum_soa_; - - view_2d qi_coarse_dst_; - view_2d qi_coarse_nacl_; - view_2d qi_coarse_so4_; - view_2d qi_coarse_mom_; - view_2d qi_coarse_bc_; - view_2d qi_coarse_pom_; - view_2d qi_coarse_soa_; - - view_2d qi_pcarbon_bc_; - view_2d qi_pcarbon_mom_; - view_2d qi_pcarbon_pom_; - view_2d nc_accum_; - view_2d ni_accum_; - view_2d ni_coarse_; - view_2d ni_aitken_; + + view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()][mam_coupling::num_aero_species()]; + mam_coupling::AerosolState aerosol_state_; + + mam_coupling::WetAtmosphere wet_atmosphere_; + + mam_coupling::DryAtmosphere dry_atmosphere_; + const_view_3d dgnum_; view_2d nihf_; view_2d niim_; @@ -95,20 +63,17 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_hom_; view_2d naai_; const_view_2d liqcldf_; - const_view_2d qc_; - const_view_2d qi_; - const_view_2d nc_; const_view_2d kvh_; - view_2d cldo_; - view_2d zm_; - view_3d state_q_; - view_2d ncldwtr_; + const_view_2d zm_; + const_view_3d state_q_; + const_view_2d ncldwtr_; - view_2d lcldn_; - view_2d lcldo_; + view_2d cloud_frac_new_; + view_2d cloud_frac_old_; view_2d qcld_; view_2d tendnd_; + // ptend_q_ is just ptend_q_output_ reformatted. view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; view_3d ptend_q_output_; view_3d factnum_; @@ -123,8 +88,11 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d coltend_[mam4::ndrop::ncnst_tot]; view_3d coltend_cw_outp_; view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; + + // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the stack. view_2d raercol_cw_[mam4::ndrop::pver][2]; view_2d raercol_[mam4::ndrop::pver][2]; + view_3d nact_; view_3d mact_; view_2d dropmixnuc_scratch_mem_[15]; @@ -133,6 +101,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d activation_fraction_accum_idx_; view_2d activation_fraction_coarse_idx_; + // These are the output tendencies from heterogeneous freezing that need to be + // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; view_2d hetfrz_depostion_nucleation_tend_; @@ -144,6 +114,12 @@ class MAMAci final : public scream::AtmosphereProcess { // FIXME: This should be read in to make user selectable. const int top_lev_ = 6; + // local atmospheric state column variables + const_view_2d p_int_; // Total pressure [Pa] at interfaces + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ + const_view_2d w_sec_; // Vertical velocity variance + public: // Constructor MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); @@ -165,23 +141,13 @@ class MAMAci final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; - // number of aerosol modes - int num_aero_modes_; - - // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. struct Preprocess { Preprocess() = default; - - - //const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - // assigns local variables - void set_variables(const const_view_2d& pdel) { - //p1del_ = pdel; + void set_variables(const const_view_2d &pdel) { } // set_variables }; // MAMAci::Preprocess @@ -189,15 +155,6 @@ class MAMAci final : public scream::AtmosphereProcess { // pre- and postprocessing scratch pads Preprocess preprocess_; - // local atmospheric state column variables - const_view_2d omega_; // Vertical pressure velocity [Pa/s] at midpoints - const_view_2d p_mid_; // Total pressure [Pa] at midpoints - const_view_2d p_int_; // Total pressure [Pa] at interfaces - const_view_2d T_mid_; // Temperature[K] at midpoints - const_view_2d pdel_; // pressure thickess of layer [Pa] - view_2d rpdel_; // Inverse of pdel_ - const_view_2d w_sec_; // Vertical velocity variance - // physics grid for column information std::shared_ptr grid_; }; // MAMAci diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 278e97c033ec..d04633885681 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -21,6 +21,7 @@ using view_2d = typename KT::template view_2d; using view_3d = typename KT::template view_3d; using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; +using const_view_3d = typename KT::template view_3d; using complex_view_3d = typename KT::template view_3d>; using complex_view_2d = typename KT::template view_2d>; diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index a7103ad01a88..c87894362ae6 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -4,9 +4,9 @@ driver_options: atmosphere_dag_verbosity_level: 5 time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} + time_step: 1 + run_t0: 2021-10-12-45000 # YYYY-MM-DD-XXXXX + number_of_steps: 24 atmosphere_processes: atm_procs_list: [mam4_aci] @@ -23,12 +23,29 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + Filename: /Users/jroverf/Codes/scream/build/atm/scream/init/screami_unit_tests_ne2np4L72_20220822.nc T_mid: 273.0 p_mid: 1.e5 num_a1: 1.e5 num_c1: 1.e5 bc_a1: 1.e-5 + qqcw: 1e5 + + aero_gas_mmr_o3: 0.0 + aero_gas_mmr_h2o2: 0.0 + aero_gas_mmr_h2so4: 0.0 + aero_gas_mmr_so2: 0.0 + aero_gas_mmr_dms: 0.0 + aero_gas_mmr_soag: 0.0 + + hetfrz_immersion_nucleation_tend: 0 + hetfrz_contact_nucleation_tend: 0 + hetfrz_depostion_nucleation_tend: 0 + + zm: 1e1 + + w_updraft: 1e0 + ptend: 1e0 num_a2: 1e5 num_a3: 1e5 num_a4: 1e5 @@ -56,6 +73,27 @@ initial_conditions: bc_a4: 1e-5 mom_a4: 1e-5 #cloud borne aerosols + qi_coarse_dst: 1e-5 + qi_coarse_nacl: 1e-5 + qi_coarse_so4: 1e-5 + qi_coarse_mom: 1e-5 + qi_coarse_bc: 1e-5 + qi_coarse_pom: 1e-5 + qi_coarse_soa: 1e-5 + ni_accum: 1e-5 + ni_coarse: 1e-5 + ni_aitken: 1e-5 + + state_q: 1e-5 + ncldwtr: 1e-12 + + icenuc_num_hetfrz: 1e3 + icenuc_num_immfrz: 1e3 + icenuc_num_depnuc: 1e3 + icenuc_num_meydep: 1e3 + num_act_aerosol_ice_nucle_hom: 1e3 + num_act_aerosol_ice_nucle: 1e3 + soa_c1: 1e-5 so4_c1: 1e-5 pom_c1: 1e-5 From 1b174de79338688b5cc3726abb2541a777334d38 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 25 Jan 2024 15:51:22 -0700 Subject: [PATCH 251/476] Fix accidental push of wrong file. --- components/eamxx/tests/uncoupled/mam4/input_aci.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml index c87894362ae6..40d3ebd9a986 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/input_aci.yaml @@ -4,9 +4,9 @@ driver_options: atmosphere_dag_verbosity_level: 5 time_stepping: - time_step: 1 - run_t0: 2021-10-12-45000 # YYYY-MM-DD-XXXXX - number_of_steps: 24 + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} atmosphere_processes: atm_procs_list: [mam4_aci] @@ -23,7 +23,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /Users/jroverf/Codes/scream/build/atm/scream/init/screami_unit_tests_ne2np4L72_20220822.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} T_mid: 273.0 p_mid: 1.e5 num_a1: 1.e5 @@ -46,6 +46,7 @@ initial_conditions: w_updraft: 1e0 ptend: 1e0 + num_a2: 1e5 num_a3: 1e5 num_a4: 1e5 From 7621745568cc717f860a585224d2ce7c7f51dfe1 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Thu, 25 Jan 2024 15:54:43 -0700 Subject: [PATCH 252/476] Fix accidental push of wrong file. --- components/eamxx/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index f0afd1c30451..8567512abf2b 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -546,7 +546,7 @@ if (SCREAM_DOUBLE_PRECISION) set(SCREAM_Fortran_FLAGS -real-size 64) elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -eZ) + set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() From 4880e6cdbe3e78c5f94d4a5521330f3fdc196496 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 31 Jan 2024 14:35:18 -0700 Subject: [PATCH 253/476] Fixes for Cuda compiler. Requires latest mam4xx to work. --- .../mam/eamxx_mam_aci_process_interface.cpp | 249 +++++++++++++----- 1 file changed, 181 insertions(+), 68 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 91de9d4a03aa..e6be4a9c4661 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -8,6 +8,22 @@ namespace scream namespace { +KOKKOS_INLINE_FUNCTION +void copy_scream_array_to_mam4xx( + const haero::ThreadTeam &team, + MAMAci::view_2d mam4xx_view[], + MAMAci::const_view_3d scream_view, + const int icol, + const int nlev, + const int num_views) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol,kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); + w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); }); } void compute_w0_and_rho( @@ -35,21 +73,28 @@ void compute_w0_and_rho( { //Get physical constants using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] MAMAci::const_view_2d omega = wet_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol,kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); - w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); - }); + compute_w0_and_rho( team, w0, rho, omega, T_mid, p_mid, icol, top_lev, nlev); + }); +} + +KOKKOS_INLINE_FUNCTION +void compute_tke_using_w_sec( + const haero::ThreadTeam &team, + MAMAci::view_2d tke, + MAMAci::const_view_2d w_sec, + const int icol, + const int nlev) +{ + // FIXME Is this the correct boundary condition for tke at the surface? + // TKE seems to be at interfaces but w_sec is at cell centers so this + // descrepensy needs to be worked out. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { + tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); }); } void compute_tke_using_w_sec( @@ -60,39 +105,48 @@ void compute_tke_using_w_sec( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // FIXME Is this the correct boundary condition for tke at the surface? - // TKE seems to be at interfaces but w_sec is at cell centers so this - // descrepensy needs to be worked out. - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { - tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); - }); + compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); }); } +KOKKOS_INLINE_FUNCTION void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, + const haero::ThreadTeam &team, MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, const Real wsubmin, + const int icol, const int top_lev, const int nlev) { // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = wsubmin; + wsubice(icol,kk) = 0.001; + wsig(icol,kk) = 0.001; + }); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); + wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); + wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); + wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); + }); +} +void compute_subgrid_scale_velocities( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, + MAMAci::view_2d wsig, + MAMAci::const_view_2d tke, + const Real wsubmin, + const int top_lev, + const int nlev) +{ Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = wsubmin; - wsubice(icol,kk) = 0.001; - wsig(icol,kk) = 0.001; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); - wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); - wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); - wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); - }); + compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, wsubmin, icol, top_lev, nlev); }); } @@ -147,6 +201,19 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) } return ww; } +KOKKOS_INLINE_FUNCTION +void compute_subgrid_mean_updraft_velocities( + const haero::ThreadTeam &team, + MAMAci::view_2d w2, + MAMAci::const_view_2d w0, + MAMAci::const_view_2d wsig, + const int icol, + const int nlev) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); + }); +} void compute_subgrid_mean_updraft_velocities( haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w2, @@ -156,9 +223,20 @@ void compute_subgrid_mean_updraft_velocities( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); - }); + compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_aitken_dry_diameter( + const haero::ThreadTeam &team, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, + const int icol, + const int top_lev) +{ + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); }); } void compute_aitken_dry_diameter( @@ -169,10 +247,7 @@ void compute_aitken_dry_diameter( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); - }); + compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); }); } @@ -252,12 +327,15 @@ void compute_nucleate_ice_tendencies( nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); } +KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( - haero::ThreadTeamPolicy team_policy, + const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, - mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d qc, + MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, + const int icol, const int top_lev) { //------------------------------------------------------------- @@ -270,21 +348,43 @@ void store_liquid_cloud_fraction( // a note of it and use the new cloud fraction for the // old cloud fraction. //------------------------------------------------------------- + static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const Real qcld = qc(icol,kk) + qi(icol,kk); + if (qcld > qsmall) { + cloud_frac_new(icol,kk)=liqcldf(icol,kk); + cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old + } else { + cloud_frac_new(icol,kk)=0; + cloud_frac_old(icol,kk)=0; + } + }); +} +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d liqcldf, + const int top_lev) +{ MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d qi = wet_atmosphere.qi; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const Real qcld = qc(icol,kk) + qi(icol,kk); - if (qcld > qsmall) { - cloud_frac_new(icol,kk)=liqcldf(icol,kk); - cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old - } else { - cloud_frac_new(icol,kk)=0; - cloud_frac_old(icol,kk)=0; - } - }); + store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, qi, liqcldf, icol, top_lev); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density( + const haero::ThreadTeam &team, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int icol, + const int nlev) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + rpdel(icol,kk) = 1/pdel(icol,kk); }); } void compute_recipical_pseudo_density( @@ -295,9 +395,7 @@ void compute_recipical_pseudo_density( { Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - rpdel(icol,kk) = 1/pdel(icol,kk); - }); + compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); }); } @@ -468,6 +566,21 @@ void call_function_dropmixnuc( ekat::subview(qqcwtend, icol) ); }); } +KOKKOS_INLINE_FUNCTION +void copy_mam4xx_array_to_scream( + const haero::ThreadTeam &team, + MAMAci::view_3d scream, + MAMAci::view_2d mam4xx[], + const int icol, + const int len, + const int nlev) +{ + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + for (int i=0; i grids_manager) FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces - using namespace ekat::units; + ekat::units::Units kg = ekat::units::kg; + ekat::units::Units Pa = ekat::units::Pa; + ekat::units::Units s = ekat::units::s; + ekat::units::Units m = ekat::units::m; + ekat::units::Units K = ekat::units::K; auto q_unit = kg/kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); auto n_unit = 1/kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); - auto nondim = Units::nondimensional(); + auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] @@ -711,25 +824,25 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); //cloudborne aerosol tracers of interest: number (n) mixing ratios //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if (strlen(int_mmr_field_name) > 0) add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); if (strlen(cld_mmr_field_name) > 0) add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } From 161b8ed4ca740669b8564a682e5306865838ffc9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 6 Feb 2024 12:07:21 -0800 Subject: [PATCH 254/476] Fixes broken aci test after rebase --- .../eamxx/src/physics/mam/mam_coupling.hpp | 3 +- .../tests/uncoupled/mam4/aci/CMakeLists.txt | 24 +++++++ .../mam4/{input_aci.yaml => aci/input.yaml} | 2 +- .../{mam4_aci_output.yaml => aci/output.yaml} | 0 .../uncoupled/mam4/mam4_aci_standalone.cpp | 68 ------------------- 5 files changed, 27 insertions(+), 70 deletions(-) create mode 100644 components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt rename components/eamxx/tests/uncoupled/mam4/{input_aci.yaml => aci/input.yaml} (98%) rename components/eamxx/tests/uncoupled/mam4/{mam4_aci_output.yaml => aci/output.yaml} (100%) delete mode 100644 components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index d04633885681..6436c0c80ff2 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -606,7 +606,8 @@ void compute_updraft_velocities(const Team& team, constexpr int nlev = mam4::nlev; int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); + dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); + const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); }); } diff --git a/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt new file mode 100644 index 000000000000..45532850822d --- /dev/null +++ b/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt @@ -0,0 +1,24 @@ +include (ScreamUtils) + + + +# Create the test +CreateADUnitTest(mam4_aci_standalone + LABELS mam4 aci physics driver + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) diff --git a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml similarity index 98% rename from components/eamxx/tests/uncoupled/mam4/input_aci.yaml rename to components/eamxx/tests/uncoupled/mam4/aci/input.yaml index 40d3ebd9a986..dc807241f968 100644 --- a/components/eamxx/tests/uncoupled/mam4/input_aci.yaml +++ b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml @@ -124,5 +124,5 @@ initial_conditions: dgnum: 1e-3 # The parameters for I/O control Scorpio: - output_yaml_files: ["mam4_aci_output.yaml"] + output_yaml_files: ["output.yaml"] ... diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml b/components/eamxx/tests/uncoupled/mam4/aci/output.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/mam4/mam4_aci_output.yaml rename to components/eamxx/tests/uncoupled/mam4/aci/output.yaml diff --git a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp b/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp deleted file mode 100644 index dd3ed62ec15b..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/mam4_aci_standalone.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include - -#include "control/atmosphere_driver.hpp" -#include "diagnostics/register_diagnostics.hpp" - -#include "physics/register_physics.hpp" -#include "physics/mam/eamxx_mam_aci_process_interface.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" - -#include "ekat/ekat_parse_yaml_file.hpp" -#include "ekat/logging/ekat_logger.hpp" - -#include - -namespace scream { - -TEST_CASE("mam4-aci-standalone", "") { - using namespace scream; - using namespace scream::control; - - // Create a comm - ekat::Comm atm_comm (MPI_COMM_WORLD); - ekat::logger::Logger<> logger("mam4-aci", - ekat::logger::LogLevel::debug, atm_comm); - - // Load ad parameter list - std::string fname = "input_aci.yaml"; - ekat::ParameterList ad_params("Atmosphere Driver"); - parse_yaml_file(fname,ad_params); - logger.debug("aci yaml parsed."); - - // Time stepping parameters - const auto& ts = ad_params.sublist("time_stepping"); - const auto dt = ts.get("time_step"); - const auto nsteps = ts.get("number_of_steps"); - const auto t0_str = ts.get("run_t0"); - const auto t0 = util::str_to_time_stamp(t0_str); - - logger.info("running MAMAci standalone test with dt = {} for {} steps.", dt, nsteps); - - // Need to register products in the factory *before* we create any atm process or grids manager. - register_physics(); - register_mesh_free_grids_manager(); - register_diagnostics(); - logger.debug("products registered."); - - // Create the driver - AtmosphereDriver ad; - logger.debug("driver created."); - - // Init and run - ad.initialize(atm_comm,ad_params,t0); - logger.debug("driver initialized."); - - logger.info("Start time stepping loop ... [0%]"); - for (int i=0; i Date: Mon, 5 Feb 2024 14:39:49 -0700 Subject: [PATCH 255/476] Finally fix all the problems with running on the GPU. --- .../mam/eamxx_mam_aci_process_interface.cpp | 150 +++++++++++------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 - 2 files changed, 95 insertions(+), 56 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e6be4a9c4661..b71052fb2d55 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -11,29 +11,33 @@ namespace KOKKOS_INLINE_FUNCTION void copy_scream_array_to_mam4xx( const haero::ThreadTeam &team, - MAMAci::view_2d mam4xx_view[], + const MAMAci::view_2d mam4xx_view, MAMAci::const_view_3d scream_view, const int icol, const int nlev, - const int num_views) + const int view_num) { Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - for (int i=0; i void copy_mam4xx_array_to_scream( haero::ThreadTeamPolicy team_policy, MAMAci::view_3d scream, - MAMAci::view_2d mam4xx[], - const int len, + MAMAci::view_2d mam4xx[len], const int nlev) { + // Localize the input arrays. + MAMAci::view_2d mam4xx_loc[len]; + for (int view_num=0; view_num < len; ++view_num) + mam4xx_loc[view_num] = mam4xx[view_num]; Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - copy_mam4xx_array_to_scream(team, scream, mam4xx, icol, len, nlev); + for (int view_num=0; view_num < len; ++view_num) { + MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; + copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, view_num); + } }); } void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, - mam4::Hetfrz &hetfrz, - mam_coupling::AerosolState &aerosol_state, - mam_coupling::WetAtmosphere &wet_atmosphere, - mam_coupling::DryAtmosphere &dry_atmosphere, + mam4::Hetfrz &hetfrz_, + mam_coupling::AerosolState &aerosol_state_, + mam_coupling::WetAtmosphere &wet_atmosphere_, + mam_coupling::DryAtmosphere &dry_atmosphere_, MAMAci::view_2d stratiform_cloud_fraction, MAMAci::view_2d activation_fraction_accum_idx, MAMAci::view_2d activation_fraction_coarse_idx, @@ -608,15 +640,25 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, - MAMAci::view_2d diagnostic_scratch[], + MAMAci::view_2d diagnostic_scratch_[], const int nlev) { using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); + + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState aerosol_state = aerosol_state_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + MAMAci::view_2d diagnostic_scratch[42]; + for (int i=0; i<42; ++i) + diagnostic_scratch[i] = diagnostic_scratch_[i];; + MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d nc = wet_atmosphere.nc; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -629,7 +671,7 @@ void call_hetfrz_compute_tendencies( dummy, ekat::subview(qc, icol), ekat::subview(nc, icol), - dummy, dummy, dummy, dummy, dummy, dummy, + dummy, dummy, dummy, dummy, dummy, dummy, dummy, pblh); // set surface state data haero::Surface surf{}; @@ -865,7 +907,6 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat - add_field("zm", scalar3d_layout_int, m, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -921,7 +962,6 @@ void MAMAci::initialize_impl(const RunType run_type) { coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); kvh_ = get_field_in("kvh").get_view(); - zm_ = get_field_in("zm").get_view(); wet_atmosphere_.qc = get_field_in("qc").get_view(); wet_atmosphere_.nc = get_field_in("nc").get_view(); @@ -1021,7 +1061,7 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_, mam4::ndrop::ncnst_tot); + copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_); // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) @@ -1068,14 +1108,14 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, - p_int_, pdel_, rpdel_, zm_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + p_int_, pdel_, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. - copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, mam4::ndrop::nvar_ptend_q, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, mam4::ndrop::ncnst_tot, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, mam4::ndrop::ncnst_tot, nlev_); + copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, nlev_); + copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies(team_policy, hetfrz_, aerosol_state_, wet_atmosphere_, dry_atmosphere_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 046d8578112b..2c590791744a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -65,7 +65,6 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - const_view_2d zm_; const_view_3d state_q_; const_view_2d ncldwtr_; From 608369a6f69e0e2b895f83ce7ef3a74170016eb2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 7 Feb 2024 08:41:55 -0800 Subject: [PATCH 256/476] Fixes compilation errors and update submodules of mam4xx and haero --- .../eamxx/src/physics/mam/mam_coupling.hpp | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6436c0c80ff2..89966b591a86 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -23,9 +23,6 @@ using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; using const_view_3d = typename KT::template view_3d; -using complex_view_3d = typename KT::template view_3d>; -using complex_view_2d = typename KT::template view_2d>; - // Kokkos thread team (league member) using Team = Kokkos::TeamPolicy::member_type; @@ -35,8 +32,6 @@ using uview_2d = typename ekat::template Unmanaged; -using view_int_1d = typename KT::template view_1d; - // number of constituents in gas chemistry "work arrays" KOKKOS_INLINE_FUNCTION constexpr int gas_pcnst() { @@ -77,7 +72,7 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. -inline +KOKKOS_INLINE_FUNCTION const char* aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { "1", @@ -88,17 +83,33 @@ const char* aero_mode_name(const int mode) { return mode_names[mode]; } +// Given a MAM aerosol species ID, returns a string denoting the symbolic +// name of the species. +KOKKOS_INLINE_FUNCTION +const char* aero_species_name(const int species_id) { + static const char *species_names[num_aero_species()] = { + "soa", + "so4", + "pom", + "bc", + "nacl", + "dst", + "mom", + }; + return species_names[species_id]; +} + // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. -inline +KOKKOS_INLINE_FUNCTION const char* gas_species_name(const int gas_id) { static const char *species_names[num_aero_gases()] = { - "O3", - "H2O2", - "H2SO4", - "SO2", - "DMS", - "SOAG" + "o3", + "h2o2", + "h2so4", + "so2", + "dms", + "soag" }; return species_names[gas_id]; } @@ -111,14 +122,14 @@ constexpr int max_field_name_len() { return 128; } -inline +KOKKOS_INLINE_FUNCTION size_t gpu_strlen(const char* s) { size_t l = 0; while (s[l]) ++l; return l; } -inline +KOKKOS_INLINE_FUNCTION void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); for (size_t i = 0; i < len1; ++i) @@ -129,7 +140,7 @@ void concat_2_strings(const char *s1, const char *s2, char *concatted) { concatted[len1+len2] = 0; } -inline +KOKKOS_INLINE_FUNCTION void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { size_t len1 = gpu_strlen(s1); for (size_t i = 0; i < len1; ++i) @@ -143,31 +154,31 @@ void concat_3_strings(const char *s1, const char *s2, const char *s3, char *conc concatted[len1+len2+len3] = 0; } -inline +KOKKOS_INLINE_FUNCTION char* int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } -inline +KOKKOS_INLINE_FUNCTION char* cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } -inline +KOKKOS_INLINE_FUNCTION char* int_aero_mmr_names(int mode, int species) { static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } -inline +KOKKOS_INLINE_FUNCTION char* cld_aero_mmr_names(int mode, int species) { static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } -inline +KOKKOS_INLINE_FUNCTION char* gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; @@ -177,7 +188,7 @@ char* gas_mmr_names(int gas_id) { // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") -inline +KOKKOS_INLINE_FUNCTION const char* int_aero_nmr_field_name(const int mode) { if (!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); @@ -187,7 +198,7 @@ const char* int_aero_nmr_field_name(const int mode) { // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") -inline +KOKKOS_INLINE_FUNCTION const char* cld_aero_nmr_field_name(const int mode) { if (!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); @@ -200,13 +211,12 @@ const char* cld_aero_nmr_field_name(const int mode) { // field in EAMxx. The form of the field name is "_a<1-based-mode-index>". // If the desired species is not present within the desire mode, returns a blank // string (""). -inline +KOKKOS_INLINE_FUNCTION const char* int_aero_mmr_field_name(const int mode, const int species) { if (!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); if (aero_id != mam4::AeroId::None) { - auto aerosol_species_name =mam4::aero_id_short_name(aero_id); - concat_3_strings(aerosol_species_name.c_str(), + concat_3_strings(aero_species_name(static_cast(aero_id)), "_a", aero_mode_name(mode), int_aero_mmr_names(mode, species)); } @@ -219,13 +229,12 @@ const char* int_aero_mmr_field_name(const int mode, const int species) { // field in EAMxx. The form of the field name is "_c<1-based-mode-index>". // If the desired species is not present within the desire mode, returns a blank // string (""). -inline +KOKKOS_INLINE_FUNCTION const char* cld_aero_mmr_field_name(const int mode, const int species) { if (!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); if (aero_id != mam4::AeroId::None) { - auto aerosol_species_name =mam4::aero_id_short_name(aero_id); - concat_3_strings(aerosol_species_name.c_str(), + concat_3_strings(aero_species_name(static_cast(aero_id)), "_c", aero_mode_name(mode), cld_aero_mmr_names(mode, species)); } @@ -234,10 +243,13 @@ const char* cld_aero_mmr_field_name(const int mode, const int species) { }; // Given a MAM aerosol-related gas identifier, returns the name of its mass -// mixing ratio field in EAMxx -inline +// mixing ratio field in EAMxx ("aero_gas_mmr_") +KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { - return const_cast(gas_species_name(gas)); + if (!gas_mmr_names(gas)[0]) { + concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); + } + return const_cast(gas_mmr_names(gas)); } // This type stores multi-column views related specifically to the wet @@ -579,19 +591,14 @@ void compute_vertical_layer_heights(const Team& team, "Given column index does not correspond to given team!"); const auto dz = ekat::subview(dry_atm.dz, column_index); - const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); - const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); - const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, dz); - team.team_barrier(); + auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + auto z_mid = ekat::subview(dry_atm.z_mid, column_index); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } + // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given // index. From 26a1c50bbe093323119ba968b30030713279e103 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 18 Feb 2024 14:18:24 -0800 Subject: [PATCH 257/476] Add wet/dry atm and aerosol states with buffer --- .../mam/eamxx_mam_aci_process_interface.cpp | 63 +++++++++++++++---- .../mam/eamxx_mam_aci_process_interface.hpp | 51 ++++++++++++--- 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b71052fb2d55..7c545d1e7521 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -629,7 +629,7 @@ void copy_mam4xx_array_to_scream( void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, - mam_coupling::AerosolState &aerosol_state_, + mam_coupling::AerosolState &dry_aero_, mam_coupling::WetAtmosphere &wet_atmosphere_, mam_coupling::DryAtmosphere &dry_atmosphere_, MAMAci::view_2d stratiform_cloud_fraction, @@ -647,7 +647,7 @@ void call_hetfrz_compute_tendencies( view_1d dummy("DummyView", nlev); mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState aerosol_state = aerosol_state_; + mam_coupling::AerosolState aerosol_state = dry_aero_; mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; @@ -816,9 +816,11 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) n_unit.set_string("#/kg"); auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // specific humidity [kg/kg] add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud ice number mixing ratio [1/kg] add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints @@ -932,6 +934,21 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) } +size_t MAMAci::requested_buffer_size_in_bytes() const +{ + return mam_coupling::buffer_size(ncol_, nlev_); +} + +void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMicrophysics."); +} + + void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); @@ -963,15 +980,20 @@ void MAMAci::initialize_impl(const RunType run_type) { liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); kvh_ = get_field_in("kvh").get_view(); + wet_atmosphere_.qv = get_field_in("qv").get_view(); wet_atmosphere_.qc = get_field_in("qc").get_view(); wet_atmosphere_.nc = get_field_in("nc").get_view(); wet_atmosphere_.qi = get_field_in("qi").get_view(); + wet_atmosphere_.ni = get_field_in("ni").get_view(); wet_atmosphere_.omega = get_field_in("omega").get_view(); dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); - dry_atmosphere_.qv = get_field_out("qv").get_view(); - dry_atmosphere_.z_mid = get_field_out("z_mid").get_view(); + dry_atmosphere_.qv = buffer_.qv_dry; + dry_atmosphere_.qc = buffer_.qc_dry; + dry_atmosphere_.nc = buffer_.nc_dry; + dry_atmosphere_.qi = buffer_.qi_dry; + dry_atmosphere_.ni = buffer_.ni_dry; dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); @@ -994,31 +1016,37 @@ void MAMAci::initialize_impl(const RunType run_type) { }; //interstitial aerosol tracers of interest: number (n) mixing ratios const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - aerosol_state_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; //cloudborne aerosol tracers of interest: number (n) mixing ratios const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - aerosol_state_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { const int index = prog_index(m, a); - aerosol_state_.int_aero_mmr[m][index] = get_field_out(int_mmr_field_name).get_view(); + wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + std::cout<<"BALLI:-----"< 0) { const int index = prog_index(m, a); - aerosol_state_.cld_aero_mmr[m][index] = get_field_out(cld_mmr_field_name).get_view(); + wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - aerosol_state_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } for (int i=0; ilog(ekat::logger::LogLevel::info,"calling ACI run"); + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_); @@ -1081,7 +1119,7 @@ void MAMAci::run_impl(const double dt) { compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, - aerosol_state_, dry_atmosphere_, aitken_dry_dia_, nlev_); + dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atmosphere_, liqcldf_, top_lev_); @@ -1105,7 +1143,7 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, rpdel_, pdel_, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. - +#if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, p_int_, pdel_, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, @@ -1118,7 +1156,7 @@ void MAMAci::run_impl(const double dt) { copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies(team_policy, - hetfrz_, aerosol_state_, wet_atmosphere_, dry_atmosphere_, + hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, stratiform_cloud_fraction_, activation_fraction_accum_idx_, activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, @@ -1127,6 +1165,7 @@ void MAMAci::run_impl(const double dt) { nlev_); Kokkos::fence(); // wait before returning to calling function +#endif } void MAMAci::finalize_impl(){ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 2c590791744a..336770092d85 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -15,8 +15,6 @@ namespace scream class MAMAci final : public scream::AtmosphereProcess { -public: - using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; @@ -33,10 +31,10 @@ class MAMAci final : public scream::AtmosphereProcess { template using view_4d = KT::view; +//FIXME:B: Should the following variables be public? They are like that in micriphysics and optics codes // FIXME the time step for microphysics [s] need to get from the input const Real dtmicro_ = .0001; -private: // rho is air density [kg/m3] view_2d rho_; @@ -49,7 +47,6 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d aitken_dry_dia_; view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()][mam_coupling::num_aero_species()]; - mam_coupling::AerosolState aerosol_state_; mam_coupling::WetAtmosphere wet_atmosphere_; @@ -129,6 +126,10 @@ class MAMAci final : public scream::AtmosphereProcess { // grid void set_grids(const std::shared_ptr grids_manager) override; + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; @@ -145,15 +146,51 @@ class MAMAci final : public scream::AtmosphereProcess { // This functor implements this step, which is called during run_impl. struct Preprocess { Preprocess() = default; - // assigns local variables - void set_variables(const const_view_2d &pdel) { - } // set_variables + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + team.team_barrier(); + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess // pre- and postprocessing scratch pads Preprocess preprocess_; + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // workspace manager for internal local variables + mam_coupling::Buffer buffer_; + // physics grid for column information std::shared_ptr grid_; }; // MAMAci From 9da7f7092b4bc57b710af8ceb137ec8e919f677e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 19 Feb 2024 18:19:57 -0800 Subject: [PATCH 258/476] Replaces pmid and zmid with quanties coming from EAMxx --- .../mam/eamxx_mam_aci_process_interface.cpp | 13 ++++++--- .../mam/eamxx_mam_aci_process_interface.hpp | 3 +++ .../eamxx/src/physics/mam/mam_coupling.hpp | 27 +++++++++++++++++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7c545d1e7521..f3e8346b832a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -833,7 +833,6 @@ void MAMAci::set_grids(const std::shared_ptr grids_manager) add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field ("z_mid", scalar3d_layout_mid, m, grid_name); // geopotential height of level (m) add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] @@ -953,7 +952,7 @@ void MAMAci::initialize_impl(const RunType run_type) { m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); // set atmosphere state data - pdel_ = get_field_in("pseudo_density").get_view(); + p_int_ = get_field_in("p_int").get_view(); w_sec_ = get_field_in("w_sec").get_view(); state_q_ = get_field_in("state_q").get_view(); @@ -989,11 +988,17 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.p_del = get_field_in("pseudo_density").get_view(); dry_atmosphere_.qv = buffer_.qv_dry; dry_atmosphere_.qc = buffer_.qc_dry; dry_atmosphere_.nc = buffer_.nc_dry; dry_atmosphere_.qi = buffer_.qi_dry; dry_atmosphere_.ni = buffer_.ni_dry; + + dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) + dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above surface at interface levels (m) + dry_atmosphere_.z_mid = buffer_.z_mid; // geopotential height above surface at mid levels (m) + dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); @@ -1141,12 +1146,12 @@ void MAMAci::run_impl(const double dt) { enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, pdel_, nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. #if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, - p_int_, pdel_, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + p_int_, dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 336770092d85..224314c50997 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -169,6 +169,9 @@ class MAMAci final : public scream::AtmosphereProcess { compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); + //vertical heights has to be computed after computing dry mixing ratios for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + } // operator() // local variables for preprocess struct diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 89966b591a86..c864a7275502 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -583,7 +583,7 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. -KOKKOS_INLINE_FUNCTION +/*KOKKOS_INLINE_FUNCTION void compute_vertical_layer_heights(const Team& team, const DryAtmosphere& dry_atm, const int column_index) { @@ -596,8 +596,31 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); -} +}*/ +KOKKOS_INLINE_FUNCTION +void compute_vertical_layer_heights(const Team& team, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine + const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); + + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs + dz);//output + team.team_barrier(); + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs + z_iface); //output + team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); +} // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given From 8cf3ad79a5aaa2a666dc78a5269245074975cd42 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 13 Mar 2024 20:56:20 -0700 Subject: [PATCH 259/476] clag format+ some minor movement of variables --- .../mam/eamxx_mam_aci_process_interface.cpp | 1823 +++++++++-------- .../mam/eamxx_mam_aci_process_interface.hpp | 100 +- 2 files changed, 980 insertions(+), 943 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f3e8346b832a..4b322245c3f9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,185 +1,163 @@ #include + #include "ekat/util/ekat_units.hpp" #include "mam4xx/aero_config.hpp" #include "mam4xx/ndrop.hpp" -namespace scream -{ +namespace scream { -namespace -{ +namespace { KOKKOS_INLINE_FUNCTION -void copy_scream_array_to_mam4xx( - const haero::ThreadTeam &team, - const MAMAci::view_2d mam4xx_view, - MAMAci::const_view_3d scream_view, - const int icol, - const int nlev, - const int view_num) -{ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - mam4xx_view(icol,kk) = scream_view(icol, kk, view_num); - }); +void copy_scream_array_to_mam4xx(const haero::ThreadTeam &team, + const MAMAci::view_2d mam4xx_view, + MAMAci::const_view_3d scream_view, + const int icol, const int nlev, + const int view_num) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + mam4xx_view(icol, kk) = scream_view(icol, kk, view_num); + }); } void copy_scream_array_to_mam4xx( haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d mam4xx_views[mam4::ndrop::ncnst_tot], - MAMAci::const_view_3d scream_view, - const int nlev) -{ + MAMAci::view_2d mam4xx_views[mam4::ndrop::ncnst_tot], + MAMAci::const_view_3d scream_view, const int nlev) { // Localize the input arrays. MAMAci::view_2d mam4xx[mam4::ndrop::ncnst_tot]; - for (int view_num=0; view_num; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol,kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol,kk) = p_mid(icol,kk)/(rair*T_mid(icol,kk)); - w0(icol,kk) = -1.0*omega(icol,kk)/(rho(icol,kk)*gravit); - }); +void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, + MAMAci::view_2d rho, MAMAci::const_view_2d omega, + MAMAci::const_view_2d T_mid, + MAMAci::const_view_2d p_mid, const int icol, + const int top_lev, const int nlev) { + // Get physical constants + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr auto rair = + C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol, kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); + w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); + }); } -void compute_w0_and_rho( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d w0, - MAMAci::view_2d rho, - mam_coupling::WetAtmosphere &wet_atmosphere, - mam_coupling::DryAtmosphere &dry_atmosphere, - const int top_lev, - const int nlev) -{ - //Get physical constants - using C = physics::Constants; +void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, + MAMAci::view_2d rho, + mam_coupling::WetAtmosphere &wet_atmosphere, + mam_coupling::DryAtmosphere &dry_atmosphere, + const int top_lev, const int nlev) { + // Get physical constants + using C = physics::Constants; MAMAci::const_view_2d omega = wet_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_w0_and_rho( team, w0, rho, omega, T_mid, p_mid, icol, top_lev, nlev); - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_w0_and_rho(team, w0, rho, omega, T_mid, p_mid, icol, top_lev, + nlev); + }); } KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec( - const haero::ThreadTeam &team, - MAMAci::view_2d tke, - MAMAci::const_view_2d w_sec, - const int icol, - const int nlev) -{ +void compute_tke_using_w_sec(const haero::ThreadTeam &team, MAMAci::view_2d tke, + MAMAci::const_view_2d w_sec, const int icol, + const int nlev) { // FIXME Is this the correct boundary condition for tke at the surface? // TKE seems to be at interfaces but w_sec is at cell centers so this // descrepensy needs to be worked out. - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev+1), KOKKOS_LAMBDA(int kk) { - tke(icol,kk) = (3.0/2.0)*w_sec(icol,kk); - }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev + 1), + KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); } -void compute_tke_using_w_sec( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d tke, - MAMAci::const_view_2d w_sec, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); - }); +void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d tke, MAMAci::const_view_2d w_sec, + const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); + }); } KOKKOS_INLINE_FUNCTION void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, - MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, - MAMAci::view_2d wsig, - MAMAci::const_view_2d tke, - const Real wsubmin, - const int icol, - const int top_lev, - const int nlev) -{ + const haero::ThreadTeam &team, MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, + const Real wsubmin, const int icol, const int top_lev, const int nlev) { // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = wsubmin; - wsubice(icol,kk) = 0.001; - wsig(icol,kk) = 0.001; - }); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol,kk) = haero::sqrt(0.5*(tke(icol,kk) + tke(icol,kk+1))*(2.0/3.0)); - wsig(icol,kk) = mam4::utils::min_max_bound(0.001, 10.0, wsub(icol,kk)); - wsubice(icol,kk) = mam4::utils::min_max_bound(0.2, 10.0, wsub(icol,kk)); - wsub(icol,kk) = haero::max(wsubmin, wsub(icol,kk)); - }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = wsubmin; + wsubice(icol, kk) = 0.001; + wsig(icol, kk) = 0.001; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * + (2.0 / 3.0)); + wsig(icol, kk) = + mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); + wsubice(icol, kk) = + mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); + wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); + }); } void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, - MAMAci::view_2d wsig, - MAMAci::const_view_2d tke, - const Real wsubmin, - const int top_lev, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, wsubmin, icol, top_lev, nlev); - }); + haero::ThreadTeamPolicy team_policy, MAMAci::view_2d wsub, + MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, + const Real wsubmin, const int top_lev, const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, + wsubmin, icol, top_lev, nlev); + }); } - KOKKOS_INLINE_FUNCTION -Real subgrid_mean_updraft(const Real w0, const Real wsig) -{ -/* --------------------------------------------------------------------------------- - Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the - vertical velocity distribution is Gaussian and peaks at the - GCM resolved large-scale vertical velocity. - When icenul_wsub_scheme = 2, the model uses the mean updraft velocity as the - characteristic updraft velocity to calculate the ice nucleation rate. - Author: Kai Zhang (kai.zhang@pnnl.gov) - Last Modified: Oct, 2015 --------------------------------------------------------------------------------- */ - -// interface - -// in :: wsig standard deviation (m/s) -// in :: w0 large scale vertical velocity (m/s) -// out:: mean updraft velocity(m/s) -> characteristic w* +Real subgrid_mean_updraft(const Real w0, const Real wsig) { + /* --------------------------------------------------------------------------------- + Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the + vertical velocity distribution is Gaussian and peaks at the + GCM resolved large-scale vertical velocity. + When icenul_wsub_scheme = 2, the model uses the mean updraft + velocity as the characteristic updraft velocity to calculate the ice + nucleation rate. Author: Kai Zhang (kai.zhang@pnnl.gov) Last Modified: Oct, + 2015 + -------------------------------------------------------------------------------- +*/ + + // interface + + // in :: wsig standard deviation (m/s) + // in :: w0 large scale vertical velocity (m/s) + // out:: mean updraft velocity(m/s) -> characteristic w* // FIXME should nbin be a user parameter? const int nbin = 50; - - using C = physics::Constants; - constexpr Real pi = C::Pi; + using C = physics::Constants; + constexpr Real pi = C::Pi; Real zz[nbin], wa = 0, ww = 0; int kp = 0; @@ -188,16 +166,19 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) const Real xx = 6.0 * sigma / nbin; - for (int ibin = 0; ibin < nbin; ++ibin) { - Real yy = wlarge - 3.0*sigma + 0.5*xx; - yy += (ibin-1)*xx; + for(int ibin = 0; ibin < nbin; ++ibin) { + Real yy = wlarge - 3.0 * sigma + 0.5 * xx; + yy += (ibin - 1) * xx; // wbar = integrator < w * f(w) * dw > - zz[ibin] = yy * haero::exp(-1.0*haero::square(yy-wlarge)/(2*sigma*sigma))/(sigma*haero::sqrt(2*pi))*xx; + zz[ibin] = + yy * + haero::exp(-1.0 * haero::square(yy - wlarge) / (2 * sigma * sigma)) / + (sigma * haero::sqrt(2 * pi)) * xx; } - for (int ibin = 0; ibin < nbin; ++ibin) { - if (zz[ibin] > 0) ++kp, wa += zz[ibin]; + for(int ibin = 0; ibin < nbin; ++ibin) { + if(zz[ibin] > 0) ++kp, wa += zz[ibin]; } - if (kp) { + if(kp) { // wbar = integrator < w * f(w) * dw > ww = wa; } else { @@ -206,254 +187,226 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) return ww; } KOKKOS_INLINE_FUNCTION -void compute_subgrid_mean_updraft_velocities( - const haero::ThreadTeam &team, - MAMAci::view_2d w2, - MAMAci::const_view_2d w0, - MAMAci::const_view_2d wsig, - const int icol, - const int nlev) -{ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - w2(icol,kk) = subgrid_mean_updraft(w0(icol,kk), wsig(icol,kk)); - }); +void compute_subgrid_mean_updraft_velocities(const haero::ThreadTeam &team, + MAMAci::view_2d w2, + MAMAci::const_view_2d w0, + MAMAci::const_view_2d wsig, + const int icol, const int nlev) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + w2(icol, kk) = subgrid_mean_updraft(w0(icol, kk), wsig(icol, kk)); + }); } void compute_subgrid_mean_updraft_velocities( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d w2, - MAMAci::const_view_2d w0, - MAMAci::const_view_2d wsig, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); - }); + haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w2, + MAMAci::const_view_2d w0, MAMAci::const_view_2d wsig, const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); + }); } KOKKOS_INLINE_FUNCTION -void compute_aitken_dry_diameter( - const haero::ThreadTeam &team, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, - const int icol, - const int top_lev) -{ +void compute_aitken_dry_diameter(const haero::ThreadTeam &team, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, const int icol, + const int top_lev) { const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol,kk) = dgnum(icol, kk, aitken_idx); - }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol, kk) = dgnum(icol, kk, aitken_idx); + }); } -void compute_aitken_dry_diameter( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, - const int top_lev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); - }); +void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d aitken_dry_dia, + MAMAci::const_view_3d dgnum, + const int top_lev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); + }); } void compute_nucleate_ice_tendencies( - const mam4::NucleateIce &nucleate_ice, - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d nihf, - MAMAci::view_2d niim, - MAMAci::view_2d nidep, - MAMAci::view_2d nimey, - MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, + const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, mam_coupling::AerosolState &aerosol_state, - mam_coupling::DryAtmosphere &dry_atmosphere, - MAMAci::view_2d aitken_dry_dia, - const int nlev) -{ + mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, + const int nlev) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation //------------------------------------------------------------- - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; - MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; + MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; MAMAci::const_view_2d w_updraft = dry_atmosphere.w_updraft; using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, - ekat::subview(T_mid, icol), - ekat::subview(p_mid, icol), - ekat::subview(qv_dry, icol), - dummy, dummy, dummy, dummy, dummy, dummy, dummy, - ekat::subview(cldfrac, icol), - ekat::subview(w_updraft, icol), pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); - - // nucleation doesn't use any diagnostics, so it's okay to leave this alone - // for now - mam4::Diagnostics diags(nlev); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t=0, dt=0; - /* - NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". - Only output we care about is "naai", "naai_hom" is never used anywhere - - Fortran code: - call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & ! input - rho, wsubice, strat_cld_frac, dgnum, & ! input - naai, naai_hom) ! output - */ - nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies + // class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), + ekat::subview(qv_dry, icol), dummy, dummy, dummy, dummy, dummy, + dummy, dummy, ekat::subview(cldfrac, icol), + ekat::subview(w_updraft, icol), pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(aerosol_state, icol); + + // nucleation doesn't use any diagnostics, so it's okay to leave this + // alone for now + mam4::Diagnostics diags(nlev); + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0, dt = 0; + /* + NOTE:"state_q" is a combination of subset of tracers added by + "int_mmr_field_name" and "int_nmr_field_name". Only output we care + about is "naai", "naai_hom" is never used anywhere + + Fortran code: + call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & + ! input rho, wsubice, strat_cld_frac, dgnum, & ! input naai, + naai_hom) ! output + */ + nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, + progs, diags, tends); + }); } KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - MAMAci::const_view_2d qc, - MAMAci::const_view_2d qi, - MAMAci::const_view_2d liqcldf, - const int icol, - const int top_lev) -{ +void store_liquid_cloud_fraction(const haero::ThreadTeam &team, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + MAMAci::const_view_2d qc, + MAMAci::const_view_2d qi, + MAMAci::const_view_2d liqcldf, const int icol, + const int top_lev) { //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud + // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value - + // MUST FIXME NOTE: We need old and new liquid cloud fractions here. - // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to - // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we will make - // a note of it and use the new cloud fraction for the - // old cloud fraction. + // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to + // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we + // will make a note of it and use the new cloud fraction for the old cloud + // fraction. //------------------------------------------------------------- - static constexpr auto qsmall = 1e-18; //cut-off for cloud amount (ice or liquid) - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const Real qcld = qc(icol,kk) + qi(icol,kk); - if (qcld > qsmall) { - cloud_frac_new(icol,kk)=liqcldf(icol,kk); - cloud_frac_old(icol,kk)=liqcldf(icol,kk); // FIXME should be liqcldf_old - } else { - cloud_frac_new(icol,kk)=0; - cloud_frac_old(icol,kk)=0; - } - }); + static constexpr auto qsmall = + 1e-18; // cut-off for cloud amount (ice or liquid) + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + const Real qcld = qc(icol, kk) + qi(icol, kk); + if(qcld > qsmall) { + cloud_frac_new(icol, kk) = liqcldf(icol, kk); + cloud_frac_old(icol, kk) = + liqcldf(icol, kk); // FIXME should be liqcldf_old + } else { + cloud_frac_new(icol, kk) = 0; + cloud_frac_old(icol, kk) = 0; + } + }); } -void store_liquid_cloud_fraction( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - mam_coupling::WetAtmosphere &wet_atmosphere, - MAMAci::const_view_2d liqcldf, - const int top_lev) -{ +void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, + mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::const_view_2d liqcldf, + const int top_lev) { MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d qi = wet_atmosphere.qi; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, qi, liqcldf, icol, top_lev); - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, + qi, liqcldf, icol, top_lev); + }); } KOKKOS_INLINE_FUNCTION -void compute_recipical_pseudo_density( - const haero::ThreadTeam &team, - MAMAci::view_2d rpdel, - MAMAci::const_view_2d pdel, - const int icol, - const int nlev) -{ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - rpdel(icol,kk) = 1/pdel(icol,kk); - }); +void compute_recipical_pseudo_density(const haero::ThreadTeam &team, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int icol, const int nlev) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), + KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); } -void compute_recipical_pseudo_density( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d rpdel, - MAMAci::const_view_2d pdel, - const int nlev) -{ - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); - }); +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + MAMAci::view_2d rpdel, + MAMAci::const_view_2d pdel, + const int nlev) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); + }); } void call_function_dropmixnuc( - haero::ThreadTeamPolicy team_policy, - mam_coupling::DryAtmosphere &dry_atmosphere, - const Real dtmicro, - MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], - MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], - MAMAci::const_view_2d p_int, - MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, - MAMAci::const_view_3d state_q, - MAMAci::const_view_2d ncldwtr, - MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, - MAMAci::view_2d wsub, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - MAMAci::view_2d tendnd, - MAMAci::view_3d factnum, - MAMAci::view_2d ndropcol, - MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, - MAMAci::view_2d wtke, - MAMAci::view_3d ccn, - MAMAci::view_3d nact, - MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[15], - const int nlev) -{ + haero::ThreadTeamPolicy team_policy, + mam_coupling::DryAtmosphere &dry_atmosphere, const Real dtmicro, + MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], + MAMAci::view_2d raercol[mam4::ndrop::pver][2], + MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, + MAMAci::view_2d rpdel, MAMAci::const_view_3d state_q, + MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, + MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d zm = dry_atmosphere.z_mid; - - MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; - MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; - MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; - MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; - MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; - MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; + MAMAci::const_view_2d zm = dry_atmosphere.z_mid; + + MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; + MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; + MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; + MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; + MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; + MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; - MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; - MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; - MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; - MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; - MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; - MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; - MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; + MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; + MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; + MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; + MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; + MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; + MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; + MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; @@ -462,622 +415,703 @@ void call_function_dropmixnuc( MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; - for (int i=0; i -void copy_mam4xx_array_to_scream( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_3d scream, - MAMAci::view_2d mam4xx[len], - const int nlev) -{ +template +void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, + MAMAci::view_3d scream, + MAMAci::view_2d mam4xx[len], const int nlev) { // Localize the input arrays. MAMAci::view_2d mam4xx_loc[len]; - for (int view_num=0; view_num < len; ++view_num) + for(int view_num = 0; view_num < len; ++view_num) mam4xx_loc[view_num] = mam4xx[view_num]; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - for (int view_num=0; view_num < len; ++view_num) { - MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; - copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, view_num); - } - }); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + for(int view_num = 0; view_num < len; ++view_num) { + MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; + copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, + view_num); + } + }); } void call_hetfrz_compute_tendencies( - haero::ThreadTeamPolicy team_policy, - mam4::Hetfrz &hetfrz_, - mam_coupling::AerosolState &dry_aero_, - mam_coupling::WetAtmosphere &wet_atmosphere_, - mam_coupling::DryAtmosphere &dry_atmosphere_, - MAMAci::view_2d stratiform_cloud_fraction, - MAMAci::view_2d activation_fraction_accum_idx, - MAMAci::view_2d activation_fraction_coarse_idx, - MAMAci::view_2d hetfrz_immersion_nucleation_tend, - MAMAci::view_2d hetfrz_contact_nucleation_tend, - MAMAci::view_2d hetfrz_depostion_nucleation_tend, - MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, - MAMAci::view_2d diagnostic_scratch_[], - const int nlev) -{ + haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, + mam_coupling::AerosolState &dry_aero_, + mam_coupling::WetAtmosphere &wet_atmosphere_, + mam_coupling::DryAtmosphere &dry_atmosphere_, + MAMAci::view_2d stratiform_cloud_fraction, + MAMAci::view_2d activation_fraction_accum_idx, + MAMAci::view_2d activation_fraction_coarse_idx, + MAMAci::view_2d hetfrz_immersion_nucleation_tend, + MAMAci::view_2d hetfrz_contact_nucleation_tend, + MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, + MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], + const int nlev) { using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState aerosol_state = dry_aero_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; - for (int i=0; i<42; ++i) - diagnostic_scratch[i] = diagnostic_scratch_[i];; + for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; + ; - MAMAci::const_view_2d qc = wet_atmosphere.qc; - MAMAci::const_view_2d nc = wet_atmosphere.nc; + MAMAci::const_view_2d qc = wet_atmosphere.qc; + MAMAci::const_view_2d nc = wet_atmosphere.nc; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for(team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, - ekat::subview(T_mid, icol), - ekat::subview(p_mid, icol), - dummy, - ekat::subview(qc, icol), - ekat::subview(nc, icol), - dummy, dummy, dummy, dummy, dummy, dummy, dummy, - pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = mam_coupling::aerosols_for_column(aerosol_state, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = ekat::subview(activation_fraction_coarse_idx, icol); - - // These are the output tendencies from heterogeneous freezing that need to be - // added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t=0, dt=0; - hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); - }); -} + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies + // class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, + ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, + dummy, dummy, dummy, dummy, dummy, pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(aerosol_state, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = + ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = + ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = + ekat::subview(activation_fraction_coarse_idx, icol); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0, dt = 0; + hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, + diags, tends); + }); } +} // namespace -//FIXME: The following variables are namelist variables +// FIXME: The following variables are namelist variables const Real wsubmin = 1; -MAMAci::MAMAci( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params){ -} +MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) {} - -//Return type of the process -AtmosphereProcessType MAMAci::type() const{ +// Return type of the process +AtmosphereProcessType MAMAci::type() const { return AtmosphereProcessType::Physics; } -//return name of the process -std::string MAMAci::name() const{ - return "mam4_aci"; - } +// return name of the process +std::string MAMAci::name() const { return "mam4_aci"; } -//set grid for all the inputs and outputs -void MAMAci::set_grids(const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI set grid"); +// set grid for all the inputs and outputs +void MAMAci::set_grids( + const std::shared_ptr grids_manager) { + m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI set grid"); - grid_ = grids_manager->get_grid("Physics"); //Use physics grid - const auto& grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); // Use physics grid + const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_+1); - Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_ + 1); + Kokkos::resize(wsub_, ncol_, nlev_); Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(cloud_frac_new_, ncol_, nlev_); - Kokkos::resize(cloud_frac_old_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(cloud_frac_new_, ncol_, nlev_); + Kokkos::resize(cloud_frac_old_, ncol_, nlev_); Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); - Kokkos::resize(rpdel_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); - for (int i=0; i<15; ++i) { - Kokkos::resize(dropmixnuc_scratch_mem_[i],ncol_, nlev_); + for(int i = 0; i < 15; ++i) { + Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; // mid points - FieldLayout scalar3d_layout_int { {COL,ILEV}, {ncol_, nlev_+1} }; //interfaces - - ekat::units::Units kg = ekat::units::kg; - ekat::units::Units Pa = ekat::units::Pa; - ekat::units::Units s = ekat::units::s; - ekat::units::Units m = ekat::units::m; - ekat::units::Units K = ekat::units::K; - auto q_unit = kg/kg; // units of mass mixing ratios of tracers + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // mid points + FieldLayout scalar3d_layout_int{{COL, ILEV}, + {ncol_, nlev_ + 1}}; // interfaces + + ekat::units::Units kg = ekat::units::kg; + ekat::units::Units Pa = ekat::units::Pa; + ekat::units::Units s = ekat::units::s; + ekat::units::Units m = ekat::units::m; + ekat::units::Units K = ekat::units::K; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; // units of number mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); auto nondim = ekat::units::Units::nondimensional(); // atmospheric quantities - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // specific humidity [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud liquid number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers");// cloud ice number mixing ratio [1/kg] - add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints - add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints - add_field("p_int", scalar3d_layout_int, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field ("qv", scalar3d_layout_mid, q_unit, grid_name); // Water vapor mixing ratio [kg vapor / kg dry air] - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // - // - add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - - add_field("state_q",FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars} } , q_unit, grid_name); // aerosol mmrs [kg/kg] - add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] - //add_field("cldo", , unitless, grid_name); // cloud fraction on previous time step [fraction] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // cloud liquid mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // cloud ice mass mixing ratio [kg/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud ice number mixing ratio [1/kg] + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // Temperature[K] at midpoints + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // Total pressure [Pa] at midpoints + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // Total pressure [Pa] at interfaces + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + // + // + add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, + grid_name); // Layer thickness(pdel) [Pa] at midpoints + + add_field( + "state_q", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars}}, q_unit, + grid_name); // aerosol mmrs [kg/kg] + add_field("ncldwtr", scalar3d_layout_mid, n_unit, + grid_name); // initial droplet number mixing ratio [#/kg] + // add_field("cldo", , unitless, grid_name); // cloud fraction on + // previous time step [fraction] // - add_field ("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - add_field("qqcw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios [#/kg or kg/kg] - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] - - - add_field("icenuc_num_hetfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to heterogeneous freezing [1/m3] - add_field("icenuc_num_immfrz",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to immersion freezing (hetero nuc) [1/m3] - add_field("icenuc_num_depnuc",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] - add_field("icenuc_num_meydep",scalar3d_layout_mid , 1/m/m/m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] - add_field("num_act_aerosol_ice_nucle_hom",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation (homogeneous freezing only) [#/kg] - add_field("num_act_aerosol_ice_nucle",scalar3d_layout_mid , n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] - add_field("qcld",scalar3d_layout_mid , n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] - add_field("ptend_q", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne aerosols [#/kg] - add_field("tendnd",scalar3d_layout_mid , n_unit/s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("factnum", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam_coupling::num_aero_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] - add_field("ndropcol",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ndropmix",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("nsource",scalar3d_layout_mid , n_unit/s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] - add_field("wtke",scalar3d_layout_mid , n_unit/s, grid_name); // - add_field("ccn", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, grid_name); //number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud [#/m^2/s]] - // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - add_field("coltend", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency for diagnostic output - add_field("coltend_cw", FieldLayout{ {COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot} } , nondim, grid_name); // column tendency - - - //MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to dry - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - //cloudborne aerosol tracers of interest: number (n) mixing ratios - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + add_field("w_updraft", scalar3d_layout_mid, q_unit, + grid_name); // updraft velocity [m/s] + add_field( + "qqcw", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, + q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios + // [#/kg or kg/kg] + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // cloud fraction [nondimentional] + + add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to + // heterogeneous freezing [1/m3] + add_field("icenuc_num_immfrz", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to immersion + // freezing (hetero nuc) [1/m3] + add_field("icenuc_num_depnuc", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to + // deposition nucleation (hetero nuc)[1/m3] + add_field( + "icenuc_num_meydep", scalar3d_layout_mid, 1 / m / m / m, + grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + add_field( + "num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, + grid_name); // number of activated aerosol for ice nucleation + // (homogeneous freezing only) [#/kg] + add_field( + "num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, + grid_name); // number of activated aerosol for ice nucleation[#/kg] + add_field("qcld", scalar3d_layout_mid, n_unit, + grid_name); // cloud droplet number mixing ratio [#/kg] + add_field( + "ptend_q", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, + n_unit, grid_name); // tendencies for interstitial and cloud borne + // aerosols [#/kg] + add_field( + "tendnd", scalar3d_layout_mid, n_unit / s, + grid_name); // tendency in droplet number mixing ratio [#/kg/s] + add_field( + "factnum", + FieldLayout{{COL, LEV, CMP}, + {ncol_, nlev_, mam_coupling::num_aero_modes()}}, + nondim, grid_name); // activation fraction for aerosol number [fraction] + add_field("ndropcol", scalar3d_layout_mid, n_unit / s, + grid_name); // + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, + grid_name); // droplet number mixing ratio tendency due + // to mixing [#/kg/s] + add_field( + "nsource", scalar3d_layout_mid, n_unit / s, + grid_name); // droplet number mixing ratio source tendency [#/kg/s] + add_field("wtke", scalar3d_layout_mid, n_unit / s, grid_name); // + add_field( + "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, + n_unit, + grid_name); // number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud + // [#/m^2/s]] + // / [aero. number conc. in updraft, just below + // cloudbase [#/m^3]] + add_field( + "coltend", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, + nondim, grid_name); // column tendency for diagnostic output + add_field( + "coltend_cw", + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, + nondim, grid_name); // column tendency + + // MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to + // dry + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); - if (strlen(int_mmr_field_name) > 0) - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(mode, a); + if(strlen(int_mmr_field_name) > 0) + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - //NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are NOT advected - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if (strlen(cld_mmr_field_name) > 0) - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); } - //Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - //These outputs should come from the cloud macrophysics process (e.g., SHOC) - auto m2 = m*m; + // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM + // These outputs should come from the cloud macrophysics process (e.g., SHOC) + auto m2 = m * m; m2.set_string("m^2"); - auto s2 = s*s; + auto s2 = s * s; s2.set_string("s^2"); - //MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and liq_strat_cld_frac may also need OLD time - add_field("w_sec", scalar3d_layout_int, m2/s2, grid_name); // Vertical velocity variance (wp2) at midpoints + // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and + // liq_strat_cld_frac may also need OLD time + add_field( + "w_sec", scalar3d_layout_int, m2 / s2, + grid_name); // Vertical velocity variance (wp2) at midpoints + + add_field("strat_cld_frac", scalar3d_layout_mid, nondim, + grid_name); // Stratiform cloud fraction at midpoints + add_field( + "liq_strat_cld_frac", scalar3d_layout_mid, nondim, + grid_name); // Liquid stratiform cloud fraction at midpoints + add_field("kvh", scalar3d_layout_int, m2 / s, + grid_name); // Eddy diffusivity for heat - add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints - add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints - add_field("kvh", scalar3d_layout_int, m2/s, grid_name); // Eddy diffusivity for heat - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes} }; // mid points - add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols - - - auto cm = m/100; - auto frz_unit = 1/(cm*cm*cm*s); // units of number mixing ratios of tracers + FieldLayout scalar4d_layout_mid{ + {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + add_field("dgnum", scalar4d_layout_mid, m, + grid_name); // dry diameter of aerosols + + auto cm = m / 100; + auto frz_unit = + 1 / (cm * cm * cm * s); // units of number mixing ratios of tracers n_unit.set_string("1(cm^-3 s^-1)"); - add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols - add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols - add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, + frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, + frz_unit, grid_name); // dry diameter of aerosols + add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, + frz_unit, grid_name); // dry diameter of aerosols // - /* + /* * NOTE on other inputs for the aci process: * 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density - * 2. geopotential height at midpoints: computed geopotential height at interfaces, which inturn is computed using - * pseudo_density, p_mid, T_mid and qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). + * 2. geopotential height at midpoints: computed geopotential height at + * interfaces, which inturn is computed using pseudo_density, p_mid, T_mid and + * qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). * qv_mid can be obtained from "get_field_in" call */ - } -size_t MAMAci::requested_buffer_size_in_bytes() const -{ +size_t MAMAci::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); - - size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); - EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), - "Error! Used memory != requested memory for MAMMicrophysics."); + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMicrophysics."); } - void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info,"Calling ACI init"); + m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI init"); // set atmosphere state data - - p_int_ = get_field_in("p_int").get_view(); - w_sec_ = get_field_in("w_sec").get_view(); - state_q_ = get_field_in("state_q").get_view(); - ncldwtr_ = get_field_in("ncldwtr").get_view(); - qqcw_input_ = get_field_in("qqcw").get_view(); - dgnum_ = get_field_in("dgnum").get_view(); - nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); - niim_ = get_field_out("icenuc_num_immfrz").get_view(); - nidep_ = get_field_out("icenuc_num_depnuc").get_view(); - nimey_ = get_field_out("icenuc_num_meydep").get_view(); - naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); - naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); - qcld_ = get_field_out("qcld").get_view(); - ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("tendnd").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); - ccn_ = get_field_out("ccn").get_view(); - coltend_outp_ = get_field_out("coltend").get_view(); - coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - kvh_ = get_field_in("kvh").get_view(); - - wet_atmosphere_.qv = get_field_in("qv").get_view(); - wet_atmosphere_.qc = get_field_in("qc").get_view(); - wet_atmosphere_.nc = get_field_in("nc").get_view(); - wet_atmosphere_.qi = get_field_in("qi").get_view(); - wet_atmosphere_.ni = get_field_in("ni").get_view(); - wet_atmosphere_.omega = get_field_in("omega").get_view(); - - dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); - dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); - dry_atmosphere_.p_del = get_field_in("pseudo_density").get_view(); - dry_atmosphere_.qv = buffer_.qv_dry; - dry_atmosphere_.qc = buffer_.qc_dry; - dry_atmosphere_.nc = buffer_.nc_dry; - dry_atmosphere_.qi = buffer_.qi_dry; - dry_atmosphere_.ni = buffer_.ni_dry; - - dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) - dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above surface at interface levels (m) - dry_atmosphere_.z_mid = buffer_.z_mid; // geopotential height above surface at mid levels (m) - - dry_atmosphere_.cldfrac = get_field_in("cldfrac_tot").get_view(); - dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); - - hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - - stratiform_cloud_fraction_ = get_field_out("stratiform_cloud_fraction").get_view(); - activation_fraction_accum_idx_ = get_field_out("activation_fraction_accum").get_view(); - activation_fraction_coarse_idx_ = get_field_out("activation_fraction_coarse").get_view(); - - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - auto prog_index = [](const int mode, const int species) { + + p_int_ = get_field_in("p_int").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); + state_q_ = get_field_in("state_q").get_view(); + ncldwtr_ = get_field_in("ncldwtr").get_view(); + qqcw_input_ = get_field_in("qqcw").get_view(); + dgnum_ = get_field_in("dgnum").get_view(); + nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); + niim_ = get_field_out("icenuc_num_immfrz").get_view(); + nidep_ = get_field_out("icenuc_num_depnuc").get_view(); + nimey_ = get_field_out("icenuc_num_meydep").get_view(); + naai_hom_ = + get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); + naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); + qcld_ = get_field_out("qcld").get_view(); + ptend_q_output_ = get_field_out("ptend_q").get_view(); + tendnd_ = get_field_out("tendnd").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + ccn_ = get_field_out("ccn").get_view(); + coltend_outp_ = get_field_out("coltend").get_view(); + coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + kvh_ = get_field_in("kvh").get_view(); + + wet_atmosphere_.qv = get_field_in("qv").get_view(); + wet_atmosphere_.qc = get_field_in("qc").get_view(); + wet_atmosphere_.nc = get_field_in("nc").get_view(); + wet_atmosphere_.qi = get_field_in("qi").get_view(); + wet_atmosphere_.ni = get_field_in("ni").get_view(); + wet_atmosphere_.omega = get_field_in("omega").get_view(); + + dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); + dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.p_del = + get_field_in("pseudo_density").get_view(); + dry_atmosphere_.qv = buffer_.qv_dry; + dry_atmosphere_.qc = buffer_.qc_dry; + dry_atmosphere_.nc = buffer_.nc_dry; + dry_atmosphere_.qi = buffer_.qi_dry; + dry_atmosphere_.ni = buffer_.ni_dry; + + dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) + dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above + // surface at interface levels (m) + dry_atmosphere_.z_mid = + buffer_.z_mid; // geopotential height above surface at mid levels (m) + + dry_atmosphere_.cldfrac = + get_field_in("cldfrac_tot").get_view(); + dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); + + hetfrz_immersion_nucleation_tend_ = + get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + hetfrz_contact_nucleation_tend_ = + get_field_out("hetfrz_contact_nucleation_tend").get_view(); + hetfrz_depostion_nucleation_tend_ = + get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + + stratiform_cloud_fraction_ = + get_field_out("stratiform_cloud_fraction").get_view(); + activation_fraction_accum_idx_ = + get_field_out("activation_fraction_accum").get_view(); + activation_fraction_coarse_idx_ = + get_field_out("activation_fraction_coarse").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + auto prog_index = [](const int mode, const int species) { const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); - const int ind = - (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; + const int ind = + (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; return ind; }; - //interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - //cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { const int index = prog_index(m, a); - wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - std::cout<<"BALLI:-----"<(); + std::cout << "BALLI:-----" << m << " " << a << " " << index << " " + << wet_aero_.int_aero_mmr[m][a](0, 0) << std::endl; dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } - + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { const int index = prog_index(m, a); - wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } - for (int i=0; ilog(ekat::logger::LogLevel::info,"calling ACI run"); + m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI run"); const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1109,45 +1144,47 @@ void MAMAci::run_impl(const double dt) { // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion - compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, + top_lev_, nlev_); compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); - Kokkos::fence(); // wait for for tke_ to be computed. + Kokkos::fence(); // wait for for tke_ to be computed. + + compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, + wsubmin, top_lev_, nlev_); + Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); - Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); - + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); - Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, - nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, - dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); + compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, + nidep_, nimey_, naai_hom_, naai_, dry_aero_, + dry_atmosphere_, aitken_dry_dia_, nlev_); - store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atmosphere_, liqcldf_, top_lev_); + store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, + wet_atmosphere_, liqcldf_, top_lev_); //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous + // Save cloud borne aerosols to be used in the heterozenous // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne + // process. This is only a select subset of cloud borne // aerosols, not all the cloud borne aerosols. //------------------------------------------------------------- - /*NOTE: We probably need to store indices for the select few cloud borne aerosols - Fortran code: - lchnk_zb = lchnk - begchunk - ! save copy of cloud borne aerosols for use in heterogeneous freezing before - ! we change it in dropmixnuc + /*NOTE: We probably need to store indices for the select few cloud borne + aerosols Fortran code: lchnk_zb = lchnk - begchunk ! save copy of cloud borne + aerosols for use in heterogeneous freezing before ! we change it in dropmixnuc do ispec = 1, ncnst call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) - aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * rho(:ncol,:) - enddo + aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * + rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, nlev_); - Kokkos::fence(); // wait for rpdel_ to be computed. + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, + nlev_); + Kokkos::fence(); // wait for rpdel_ to be computed. #if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, @@ -1173,8 +1210,8 @@ void MAMAci::run_impl(const double dt) { #endif } -void MAMAci::finalize_impl(){ - m_atm_logger->log(ekat::logger::LogLevel::info,"calling ACI final"); +void MAMAci::finalize_impl() { + m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI final"); } -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 224314c50997..0523890cff84 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -1,38 +1,37 @@ #ifndef EAMXX_MAM_ACI_HPP #define EAMXX_MAM_ACI_HPP -//For MAM4 aerosol configuration +// For MAM4 aerosol configuration #include -//For declaring ACI class derived from atm process class +// For declaring ACI class derived from atm process class #include -//For physical constants +// For physical constants #include "physics/share/physics_constants.hpp" -namespace scream -{ +namespace scream { class MAMAci final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; mam4::Hetfrz hetfrz_; // views for single- and multi-column data - using view_1d = scream::mam_coupling::view_1d; - using view_2d = scream::mam_coupling::view_2d; - using view_3d = scream::mam_coupling::view_3d; + using view_1d = scream::mam_coupling::view_1d; + using view_2d = scream::mam_coupling::view_2d; + using view_3d = scream::mam_coupling::view_3d; using const_view_1d = scream::mam_coupling::const_view_1d; using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; template - using view_4d = KT::view; + using view_4d = KT::view; -//FIXME:B: Should the following variables be public? They are like that in micriphysics and optics codes - // FIXME the time step for microphysics [s] need to get from the input + // FIXME:B: Should the following variables be public? They are like that in + // micriphysics and optics codes + // FIXME the time step for microphysics [s] need to get from the input const Real dtmicro_ = .0001; // rho is air density [kg/m3] @@ -46,7 +45,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d aitken_dry_dia_; - view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()][mam_coupling::num_aero_species()]; + view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()] + [mam_coupling::num_aero_species()]; mam_coupling::WetAtmosphere wet_atmosphere_; @@ -60,7 +60,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_hom_; view_2d naai_; const_view_2d liqcldf_; - const_view_2d kvh_; + const_view_2d kvh_; const_view_3d state_q_; const_view_2d ncldwtr_; @@ -85,7 +85,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d coltend_cw_outp_; view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; - // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the stack. + // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the + // stack. view_2d raercol_cw_[mam4::ndrop::pver][2]; view_2d raercol_[mam4::ndrop::pver][2]; @@ -111,20 +112,33 @@ class MAMAci final : public scream::AtmosphereProcess { const int top_lev_ = 6; // local atmospheric state column variables - const_view_2d p_int_; // Total pressure [Pa] at interfaces - const_view_2d pdel_; // pressure thickess of layer [Pa] - view_2d rpdel_; // Inverse of pdel_ - const_view_2d w_sec_; // Vertical velocity variance + const_view_2d p_int_; // Total pressure [Pa] at interfaces + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ + const_view_2d w_sec_; // Vertical velocity variance + + // number of horizontal columns and vertical levels + int ncol_, nlev_; -public: + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // workspace manager for internal local variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: // Constructor - MAMAci(const ekat::Comm& comm, const ekat::ParameterList& params); + MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // process metadata AtmosphereProcessType type() const override; std::string name() const override; // grid - void set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -135,12 +149,6 @@ class MAMAci final : public scream::AtmosphereProcess { void run_impl(const double dt) override; void finalize_impl() override; - - //Local variables - - // number of horizontal columns and vertical levels - int ncol_, nlev_; - // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. @@ -148,11 +156,10 @@ class MAMAci final : public scream::AtmosphereProcess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere& wet_atm, - const mam_coupling::AerosolState& wet_aero, - const mam_coupling::DryAtmosphere& dry_atm, - const mam_coupling::AerosolState& dry_aero) { - + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { ncol_pre_ = ncol; nlev_pre_ = nlev; wet_atm_pre_ = wet_atm; @@ -167,9 +174,11 @@ class MAMAci final : public scream::AtmosphereProcess { const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); team.team_barrier(); - //vertical heights has to be computed after computing dry mixing ratios for atmosphere + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, i); } // operator() @@ -181,24 +190,15 @@ class MAMAci final : public scream::AtmosphereProcess { // local atmospheric and aerosol state data mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess - + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + private: // pre- and postprocessing scratch pads Preprocess preprocess_; - // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; - - // workspace manager for internal local variables - mam_coupling::Buffer buffer_; - - // physics grid for column information - std::shared_ptr grid_; -}; // MAMAci - -} // namespace scream +}; // MAMAci +} // namespace scream -#endif // EAMXX_MAM_ACI_HPP +#endif // EAMXX_MAM_ACI_HPP From 93fb19227868c8cfe2e03ae92f20b5826114bed8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 14 Mar 2024 21:37:21 -0700 Subject: [PATCH 260/476] Adds some temp comments and comment out state_q and qqcw inputs --- .../mam/eamxx_mam_aci_process_interface.cpp | 149 +++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 16 +- 2 files changed, 89 insertions(+), 76 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4b322245c3f9..0b7fb739e7cb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -232,7 +232,7 @@ void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, - mam_coupling::AerosolState &aerosol_state, + mam_coupling::AerosolState &dry_aerosol_state, mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, const int nlev) { //------------------------------------------------------------- @@ -261,7 +261,7 @@ void compute_nucleate_ice_tendencies( // set surface state data haero::Surface surf{}; mam4::Prognostics progs = - mam_coupling::aerosols_for_column(aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); // nucleation doesn't use any diagnostics, so it's okay to leave this // alone for now @@ -355,6 +355,7 @@ void compute_recipical_pseudo_density(const haero::ThreadTeam &team, MAMAci::view_2d rpdel, MAMAci::const_view_2d pdel, const int icol, const int nlev) { + // FIXME: Add an assert to ensure pdel is non-zero Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); @@ -490,8 +491,9 @@ void call_function_dropmixnuc( ptend, nctend_mixnuc, factnum) !out */ - //mam4::utils::extract_stateq_from_prognostics(progs, atm, state_q_at_kk, - // kk); + // mam4::utils::extract_stateq_from_prognostics(progs, atm, + // state_q_at_kk, + // kk); mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -570,10 +572,10 @@ void call_hetfrz_compute_tendencies( using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState dry_aerosol_state = dry_aero_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; ; @@ -597,7 +599,7 @@ void call_hetfrz_compute_tendencies( // set surface state data haero::Surface surf{}; mam4::Prognostics progs = - mam_coupling::aerosols_for_column(aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); @@ -690,14 +692,6 @@ const Real wsubmin = 1; MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) {} -// Return type of the process -AtmosphereProcessType MAMAci::type() const { - return AtmosphereProcessType::Physics; -} - -// return name of the process -std::string MAMAci::name() const { return "mam4_aci"; } - // set grid for all the inputs and outputs void MAMAci::set_grids( const std::shared_ptr grids_manager) { @@ -709,7 +703,9 @@ void MAMAci::set_grids( ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize( + rho_, ncol_, + nlev_); // BQ: Does Kokkos::resize allocates memory as well for rho_? Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -732,7 +728,7 @@ void MAMAci::set_grids( FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // interfaces - ekat::units::Units kg = ekat::units::kg; + ekat::units::Units kg = ekat::units::kg; // BQ: why do we need to do this??? ekat::units::Units Pa = ekat::units::Pa; ekat::units::Units s = ekat::units::s; ekat::units::Units m = ekat::units::m; @@ -773,22 +769,26 @@ void MAMAci::set_grids( add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field( + // MUST FIXME: remove state_q from this list + /*add_field( "state_q", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars}}, q_unit, - grid_name); // aerosol mmrs [kg/kg] + grid_name); // aerosol mmrs [kg/kg]*/ + // MUST FIXME: Is it same as nc or may be not???? add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] - // add_field("cldo", , unitless, grid_name); // cloud fraction on - // previous time step [fraction] - // + + // MUST FIXME: This should be an internal variable. why we need this as an + // input??? add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity [m/s] - add_field( + // MUST FIXME: Remove qqcw from here + /*add_field( "qqcw", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios - // [#/kg or kg/kg] + // [#/kg or kg/kg]*/ + // MUST FIXME: move it above so that all "required" variables are together add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] @@ -947,10 +947,6 @@ void MAMAci::set_grids( */ } -size_t MAMAci::requested_buffer_size_in_bytes() const { - return mam_coupling::buffer_size(ncol_, nlev_); -} - void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -968,16 +964,27 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data - p_int_ = get_field_in("p_int").get_view(); - w_sec_ = get_field_in("w_sec").get_view(); - state_q_ = get_field_in("state_q").get_view(); - ncldwtr_ = get_field_in("ncldwtr").get_view(); - qqcw_input_ = get_field_in("qqcw").get_view(); - dgnum_ = get_field_in("dgnum").get_view(); - nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); - niim_ = get_field_out("icenuc_num_immfrz").get_view(); - nidep_ = get_field_out("icenuc_num_depnuc").get_view(); - nimey_ = get_field_out("icenuc_num_meydep").get_view(); + p_int_ = get_field_in("p_int").get_view(); + w_sec_ = get_field_in("w_sec").get_view(); + /*state_q_ = get_field_in("state_q") + .get_view(); // MUST FIXME: remove this*/ + ncldwtr_ = + get_field_in("ncldwtr") + .get_view(); // MUST FIXME: is is nc, may be not??? + /*qqcw_input_ = get_field_in("qqcw") + .get_view(); // MUST FIXME: remove this*/ + dgnum_ = get_field_in("dgnum") + .get_view(); // MUST FIXME: is it an input, can + // we compute it using calcsize??? + liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + kvh_ = get_field_in("kvh") + .get_view(); // MUST FIXME: See if scream has it, + // it should com from the land model + + nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); + niim_ = get_field_out("icenuc_num_immfrz").get_view(); + nidep_ = get_field_out("icenuc_num_depnuc").get_view(); + nimey_ = get_field_out("icenuc_num_meydep").get_view(); naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); @@ -992,8 +999,6 @@ void MAMAci::initialize_impl(const RunType run_type) { ccn_ = get_field_out("ccn").get_view(); coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - kvh_ = get_field_in("kvh").get_view(); wet_atmosphere_.qv = get_field_in("qv").get_view(); wet_atmosphere_.qc = get_field_in("qc").get_view(); @@ -1039,12 +1044,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - auto prog_index = [](const int mode, const int species) { - const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); - const int ind = - (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; - return ind; - }; // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = @@ -1062,11 +1061,8 @@ void MAMAci::initialize_impl(const RunType run_type) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if(strlen(int_mmr_field_name) > 0) { - const int index = prog_index(m, a); wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - std::cout << "BALLI:-----" << m << " " << a << " " << index << " " - << wet_aero_.int_aero_mmr[m][a](0, 0) << std::endl; dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } @@ -1074,7 +1070,6 @@ void MAMAci::initialize_impl(const RunType run_type) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if(strlen(cld_mmr_field_name) > 0) { - const int index = prog_index(m, a); wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; @@ -1091,21 +1086,21 @@ void MAMAci::initialize_impl(const RunType run_type) { for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. - Kokkos::resize(qqcw_[i], ncol_, nlev_); + /*Kokkos::resize(qqcw_[i], ncol_, nlev_); // MUST FIXME: remove this*/ Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); + Kokkos::resize(ptend_q_[i], ncol_, nlev_); // MUST FIXME:Do we need this? } for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { + for(int j = 0; j < 2; ++j) { // MUST FIXME:store 2 in a const variable Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); } } - for(int i = 0; i < 42; ++i) + for(int i = 0; i < 42; ++i) // MUST FIXME:store 42 in a const var Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); // nact : fractional aero. number activation rate [/s] @@ -1133,44 +1128,54 @@ void MAMAci::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - // preprocess input -- needs a scan for the calculation of atm height + // preprocess input -- needs a scan for the calculation of local derivied + // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - copy_scream_array_to_mam4xx(team_policy, qqcw_, qqcw_input_, nlev_); + /*copy_scream_array_to_mam4xx( + team_policy, qqcw_, qqcw_input_, + nlev_); // MUST FIXME: remove this and populate qqcw here*/ // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion - compute_w0_and_rho(team_policy, w0_, rho_, wet_atmosphere_, dry_atmosphere_, - top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, + wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + + compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); - compute_tke_using_w_sec(team_policy, tke_, w_sec_, nlev_); Kokkos::fence(); // wait for for tke_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, + compute_subgrid_scale_velocities(team_policy, wsub_ /*output*/, + wsubice_ /*output*/, wsig_ /*output*/, tke_, wsubmin, top_lev_, nlev_); Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); + compute_subgrid_mean_updraft_velocities(team_policy, w2_ /*output*/, w0_, + wsig_, nlev_); - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_ /*output*/, dgnum_, + top_lev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + // FIXME: Find out in-outs of the following call! compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); - store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atmosphere_, liqcldf_, top_lev_); + store_liquid_cloud_fraction(team_policy, cloud_frac_new_ /*output*/, + cloud_frac_old_ /*output*/, wet_atmosphere_, + liqcldf_, top_lev_); + // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. + // Save cloud borne aerosols to be used in the heterozenous + // freezing before they are changed by the droplet activation + // process. This is only a select subset of cloud borne + // aerosols, not all the cloud borne aerosols. //------------------------------------------------------------- /*NOTE: We probably need to store indices for the select few cloud borne aerosols Fortran code: lchnk_zb = lchnk - begchunk ! save copy of cloud borne @@ -1182,8 +1187,8 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, - nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_ /*output*/, + dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. #if 0 call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0523890cff84..5ea30e6ba88a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -62,7 +62,7 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - const_view_3d state_q_; + // const_view_3d state_q_; const_view_2d ncldwtr_; view_2d cloud_frac_new_; @@ -133,15 +133,23 @@ class MAMAci final : public scream::AtmosphereProcess { // Constructor MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // process metadata - AtmosphereProcessType type() const override; - std::string name() const override; + + // Return type of the process + AtmosphereProcessType MAMAci::type() const { + return AtmosphereProcessType::Physics; + } + + // return name of the process + std::string MAMAci::name() const { return "mam4_aci"; } // grid void set_grids( const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t requested_buffer_size_in_bytes() const override; + size_t MAMAci::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); + } void init_buffers(const ATMBufferManager &buffer_manager) override; // process behavior From 6f0132dbe634dcfd1463f073d39a9dfc485ba437 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 14:04:35 -0700 Subject: [PATCH 261/476] Adds call to progs and atm for getting state_q for dropmixnuc --- .../mam/eamxx_mam_aci_process_interface.cpp | 59 +++++++++++-------- .../mam/eamxx_mam_aci_process_interface.hpp | 3 +- .../eamxx/tests/uncoupled/mam4/aci/input.yaml | 2 + 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 0b7fb739e7cb..41813a1687b7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -373,15 +373,16 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, - mam_coupling::DryAtmosphere &dry_atmosphere, const Real dtmicro, + mam_coupling::DryAtmosphere &dry_atmosphere, + const mam_coupling::AerosolState &dry_aerosol_state, const Real dtmicro, MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + // MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, MAMAci::const_view_3d state_q, + MAMAci::view_2d rpdel, MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, @@ -420,7 +421,7 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_qqcw[i] = qqcw[i]; + // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_qqcw[i] = qqcw[i]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; @@ -462,10 +463,10 @@ void call_function_dropmixnuc( raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); } } - mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; + /*mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); - } + }*/ mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); @@ -491,9 +492,11 @@ void call_function_dropmixnuc( ptend, nctend_mixnuc, factnum) !out */ - // mam4::utils::extract_stateq_from_prognostics(progs, atm, - // state_q_at_kk, - // kk); + mam4::Prognostics prog_at_col = + aerosols_for_column(dry_aerosol_state, icol); + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + /* mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -525,7 +528,7 @@ void call_function_dropmixnuc( ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), ekat::subview(srcn, icol), ekat::subview(source, icol), ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), - ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); + ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol));*/ }); } KOKKOS_INLINE_FUNCTION @@ -727,6 +730,8 @@ void MAMAci::set_grids( FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // mid points FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; // interfaces + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; ekat::units::Units kg = ekat::units::kg; // BQ: why do we need to do this??? ekat::units::Units Pa = ekat::units::Pa; @@ -760,8 +765,9 @@ void MAMAci::set_grids( grid_name); // Total pressure [Pa] at interfaces add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // - // + + add_field("pbl_height", scalar2d_layout_col, m, + grid_name); // planetary boundary layer height add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, @@ -964,7 +970,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data - p_int_ = get_field_in("p_int").get_view(); w_sec_ = get_field_in("w_sec").get_view(); /*state_q_ = get_field_in("state_q") .get_view(); // MUST FIXME: remove this*/ @@ -1009,13 +1014,15 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); + dry_atmosphere_.p_int = get_field_in("p_int").get_view(); dry_atmosphere_.p_del = get_field_in("pseudo_density").get_view(); - dry_atmosphere_.qv = buffer_.qv_dry; - dry_atmosphere_.qc = buffer_.qc_dry; - dry_atmosphere_.nc = buffer_.nc_dry; - dry_atmosphere_.qi = buffer_.qi_dry; - dry_atmosphere_.ni = buffer_.ni_dry; + dry_atmosphere_.qv = buffer_.qv_dry; + dry_atmosphere_.qc = buffer_.qc_dry; + dry_atmosphere_.nc = buffer_.nc_dry; + dry_atmosphere_.qi = buffer_.qi_dry; + dry_atmosphere_.ni = buffer_.ni_dry; + dry_atmosphere_.pblh = get_field_in("pbl_height").get_view(); dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above @@ -1087,6 +1094,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. /*Kokkos::resize(qqcw_[i], ncol_, nlev_); // MUST FIXME: remove this*/ + Kokkos::resize(state_q_[i], ncol_, nlev_); Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } @@ -1190,14 +1198,15 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, rpdel_ /*output*/, dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. -#if 0 - call_function_dropmixnuc(team_policy, dry_atmosphere_, dtmicro_, - raercol_cw_, raercol_, qqcw_, ptend_q_, coltend_, coltend_cw_, - p_int_, dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, - cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); - Kokkos::fence(); // wait for ptend_q_ to be computed. + call_function_dropmixnuc( + team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, + dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + Kokkos::fence(); // wait for ptend_q_ to be computed. +#if 0 copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, nlev_); copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 5ea30e6ba88a..e22e1f54e6f8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -62,7 +62,7 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - // const_view_3d state_q_; + view_2d state_q_[mam4::ndrop::ncnst_tot]; const_view_2d ncldwtr_; view_2d cloud_frac_new_; @@ -112,7 +112,6 @@ class MAMAci final : public scream::AtmosphereProcess { const int top_lev_ = 6; // local atmospheric state column variables - const_view_2d p_int_; // Total pressure [Pa] at interfaces const_view_2d pdel_; // pressure thickess of layer [Pa] view_2d rpdel_; // Inverse of pdel_ const_view_2d w_sec_; // Vertical velocity variance diff --git a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml index dc807241f968..946d11dcc0e0 100644 --- a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml @@ -26,10 +26,12 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} T_mid: 273.0 p_mid: 1.e5 + p_int: 1.e5 num_a1: 1.e5 num_c1: 1.e5 bc_a1: 1.e-5 qqcw: 1e5 + pbl_height: 0.0 aero_gas_mmr_o3: 0.0 aero_gas_mmr_h2o2: 0.0 From 5f5f2a765499c45612f4cdc953f9bfb3db7985e4 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 17:01:44 -0700 Subject: [PATCH 262/476] Adds a working state_q extraction --- .../mam/eamxx_mam_aci_process_interface.cpp | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 41813a1687b7..e6b2ca1ac7fb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -382,7 +382,7 @@ void call_function_dropmixnuc( MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot], + MAMAci::view_2d rpdel, /*MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot],*/ MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, @@ -428,6 +428,11 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; + MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; + + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + Kokkos::resize(state_q[i], mam4::ndrop::pver); + } Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -492,10 +497,22 @@ void call_function_dropmixnuc( ptend, nctend_mixnuc, factnum) !out */ - mam4::Prognostics prog_at_col = + mam4::Prognostics progs_at_col = aerosols_for_column(dry_aerosol_state, icol); haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + + for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { + Real state_q_at_lev_col[mam4::ndrop::ncnst_tot] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + // mam4::utils::extract_stateq_from_prognostics(progs_at_col, + // haero_atm, state_q, klev); + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + state_q[i](klev) = state_q_at_lev_col[i]; + } + } + /* mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), @@ -1202,7 +1219,7 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, state_q_, ncldwtr_, kvh_, qcld_, wsub_, + dry_atmosphere_.p_del, rpdel_, /*state_q_,*/ ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. From 96f577bf3ea43fbe818cf3f3c1b00b77d29d2b97 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 17:13:21 -0700 Subject: [PATCH 263/476] Adds a working qqcw for dropmixnuc --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e6b2ca1ac7fb..99d1a92e4aeb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -429,9 +429,11 @@ void call_function_dropmixnuc( loc_coltend_cw[i] = coltend_cw[i]; MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; + MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { Kokkos::resize(state_q[i], mam4::ndrop::pver); + Kokkos::resize(qqcw[i], mam4::ndrop::pver); } Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -504,12 +506,15 @@ void call_function_dropmixnuc( for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { Real state_q_at_lev_col[mam4::ndrop::ncnst_tot] = {}; + Real qqcw_at_lev_col[mam4::ndrop::ncnst_tot] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); - // mam4::utils::extract_stateq_from_prognostics(progs_at_col, - // haero_atm, state_q, klev); + + mam4::utils::extract_qqcw_from_prognostics(progs_at_col, + qqcw_at_lev_col, klev); for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { state_q[i](klev) = state_q_at_lev_col[i]; + qqcw[i](klev) = qqcw_at_lev_col[i]; } } From ce90fd2d198264cae3239d33656d23e63b2a020e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 17:35:21 -0700 Subject: [PATCH 264/476] A working state_q1 as a 2d view --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 99d1a92e4aeb..91ada8a87dde 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -429,6 +429,8 @@ void call_function_dropmixnuc( loc_coltend_cw[i] = coltend_cw[i]; MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d state_q1; + Kokkos::resize(state_q1, mam4::ndrop::pver, mam4::ndrop::ncnst_tot); MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { @@ -513,8 +515,9 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - state_q[i](klev) = state_q_at_lev_col[i]; - qqcw[i](klev) = qqcw_at_lev_col[i]; + state_q[i](klev) = state_q_at_lev_col[i]; + state_q1(klev, i) = state_q_at_lev_col[i]; + qqcw[i](klev) = qqcw_at_lev_col[i]; } } From 5c5f90bb10acf1589df2ec52a777b0fc0237062f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 17 Mar 2024 20:08:26 -0700 Subject: [PATCH 265/476] Particle args for dropmixnuc upto state_q works --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 91ada8a87dde..ac3136293182 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -521,7 +521,6 @@ void call_function_dropmixnuc( } } - /* mam4::ndrop::dropmixnuc( team, dtmicro, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), @@ -529,7 +528,7 @@ void call_function_dropmixnuc( ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(state_q, icol), ekat::subview(ncldwtr, icol), + state_q1/*, ekat::subview(ncldwtr, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac_new, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, @@ -553,7 +552,7 @@ void call_function_dropmixnuc( ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), ekat::subview(srcn, icol), ekat::subview(source, icol), ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), - ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol));*/ + ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)*/); }); } KOKKOS_INLINE_FUNCTION From 6fd9beca3d599a69769d1a95f19f5cd081d6b8ec Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 14:08:39 -0700 Subject: [PATCH 266/476] A working qqcw using a scratch memory with partial arg list --- .../mam/eamxx_mam_aci_process_interface.cpp | 62 +++++++++++++------ .../mam/eamxx_mam_aci_process_interface.hpp | 3 + 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ac3136293182..91dce1262867 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -378,6 +378,7 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], // MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], @@ -428,18 +429,29 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; - MAMAci::view_1d state_q[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d state_q1; - Kokkos::resize(state_q1, mam4::ndrop::pver, mam4::ndrop::ncnst_tot); - MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d state_q; + Kokkos::resize( + state_q, mam4::ndrop::pver, + mam4::ndrop::nvars); // FIXME: use pcnst here and add ncol dim as well + + // mam4::ColumnView qqcw[mam4::ndrop::ncnst_tot]; + // qqcw = work_; + // MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; + + // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + // Kokkos::resize( + + /*MAMAci::view_2d &qqcw_ptr[25]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + qqcw_ptr[i] = qqcw_fld_work_[i]*/ + MAMAci::view_2d qqcw_fld_work_loc[25]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - Kokkos::resize(state_q[i], mam4::ndrop::pver); - Kokkos::resize(qqcw[i], mam4::ndrop::pver); - } Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); + // for (int icol =0 ; icol<1 ;icol++){ // Initialize the ndrop class. const int ntot_amode = mam_coupling::num_aero_modes(); @@ -472,10 +484,12 @@ void call_function_dropmixnuc( raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); } } - /*mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; + mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); - }*/ + // qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); + qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); + } + mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); @@ -507,17 +521,19 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::ndrop::ncnst_tot] = {}; - Real qqcw_at_lev_col[mam4::ndrop::ncnst_tot] = {}; + Real state_q_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here + Real qqcw_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - state_q[i](klev) = state_q_at_lev_col[i]; - state_q1(klev, i) = state_q_at_lev_col[i]; - qqcw[i](klev) = qqcw_at_lev_col[i]; + for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { + state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; + // std::cout<<"BALLI:"<(); + // allocate work + + for(int icnst = 0; icnst < 25; ++icnst) { + qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); + } + // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -1225,6 +1248,7 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + qqcw_fld_work_, /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, /*state_q_,*/ ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index e22e1f54e6f8..79f099b7d43d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -128,6 +128,9 @@ class MAMAci final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; + // A view array to carry cloud borne aerosol mmrs/nmrs + view_2d qqcw_fld_work_[25]; + public: // Constructor MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); From 812bc3f28da2800fa5ec6e95eac13a70b880a6a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 14:14:11 -0700 Subject: [PATCH 267/476] Full dropmixnuc call is working now --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 91dce1262867..ae41bba07239 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -529,10 +529,7 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; - // std::cout<<"BALLI:"< Date: Mon, 18 Mar 2024 14:40:53 -0700 Subject: [PATCH 268/476] Creates state_q with columns and scratch memory --- .../mam/eamxx_mam_aci_process_interface.cpp | 47 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 + 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ae41bba07239..8ab58132e575 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -384,12 +384,12 @@ void call_function_dropmixnuc( MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, MAMAci::view_2d rpdel, /*MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot],*/ - MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, - MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_3d state_q_work_, MAMAci::const_view_2d ncldwtr, + MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, + MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, + MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, + MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, + MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -434,20 +434,12 @@ void call_function_dropmixnuc( state_q, mam4::ndrop::pver, mam4::ndrop::nvars); // FIXME: use pcnst here and add ncol dim as well - // mam4::ColumnView qqcw[mam4::ndrop::ncnst_tot]; - // qqcw = work_; - // MAMAci::view_1d qqcw[mam4::ndrop::ncnst_tot]; - - // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - // Kokkos::resize( - - /*MAMAci::view_2d &qqcw_ptr[25]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - qqcw_ptr[i] = qqcw_fld_work_[i]*/ MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; + MAMAci::view_3d state_q_work_loc = state_q_work_; + Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -529,8 +521,9 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; - qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; + state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; } } @@ -541,7 +534,7 @@ void call_function_dropmixnuc( ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - state_q, ekat::subview(ncldwtr, icol), + ekat::subview(state_q_work_loc, icol), ekat::subview(ncldwtr, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac_new, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, @@ -1091,6 +1084,7 @@ void MAMAci::initialize_impl(const RunType run_type) { for(int icnst = 0; icnst < 25; ++icnst) { qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); } + state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, 40); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -1243,13 +1237,14 @@ void MAMAci::run_impl(const double dt) { dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. - call_function_dropmixnuc( - team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, - qqcw_fld_work_, - /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, /*state_q_,*/ ncldwtr_, kvh_, qcld_, wsub_, - cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + call_function_dropmixnuc(team_policy, dry_atmosphere_, dry_aero_, dtmicro_, + raercol_cw_, raercol_, qqcw_fld_work_, + /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, + dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, + /*state_q_,*/ state_q_work_, ncldwtr_, kvh_, qcld_, + wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, + factnum_, ndropcol_, ndropmix_, nsource_, wtke_, + ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. #if 0 copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 79f099b7d43d..b6e52026801d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -130,6 +130,7 @@ class MAMAci final : public scream::AtmosphereProcess { // A view array to carry cloud borne aerosol mmrs/nmrs view_2d qqcw_fld_work_[25]; + view_3d state_q_work_; public: // Constructor From a215ef3f27195b3fbdc49163e71ea01ad884b1bc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 17:57:27 -0700 Subject: [PATCH 269/476] Some cleanup-still need a lot of cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 64 +++++-------------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 - 2 files changed, 15 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 8ab58132e575..dd415e970c69 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -377,19 +377,18 @@ void call_function_dropmixnuc( const mam_coupling::AerosolState &dry_aerosol_state, const Real dtmicro, MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - // MAMAci::view_2d qqcw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, /*MAMAci::view_2d state_q[mam4::ndrop::ncnst_tot],*/ - MAMAci::view_3d state_q_work_, MAMAci::const_view_2d ncldwtr, - MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, MAMAci::view_2d wsub, - MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, - MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, - MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, - MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, + MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, + MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -422,18 +421,12 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - // for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_qqcw[i] = qqcw[i]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; - MAMAci::view_2d state_q; - Kokkos::resize( - state_q, mam4::ndrop::pver, - mam4::ndrop::nvars); // FIXME: use pcnst here and add ncol dim as well - MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; @@ -443,7 +436,6 @@ void call_function_dropmixnuc( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for (int icol =0 ; icol<1 ;icol++){ // Initialize the ndrop class. const int ntot_amode = mam_coupling::num_aero_modes(); @@ -478,10 +470,8 @@ void call_function_dropmixnuc( } mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // qqcw_view[i] = ekat::subview(loc_qqcw[i], icol); qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); } - mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); @@ -521,7 +511,6 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q(klev, icnst - 15) = state_q_at_lev_col[icnst]; state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; } @@ -560,7 +549,6 @@ void call_function_dropmixnuc( ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); }); - // } } KOKKOS_INLINE_FUNCTION void copy_mam4xx_array_to_scream(const haero::ThreadTeam &team, @@ -806,11 +794,6 @@ void MAMAci::set_grids( add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // MUST FIXME: remove state_q from this list - /*add_field( - "state_q", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvars}}, q_unit, - grid_name); // aerosol mmrs [kg/kg]*/ // MUST FIXME: Is it same as nc or may be not???? add_field("ncldwtr", scalar3d_layout_mid, n_unit, grid_name); // initial droplet number mixing ratio [#/kg] @@ -818,13 +801,7 @@ void MAMAci::set_grids( // MUST FIXME: This should be an internal variable. why we need this as an // input??? add_field("w_updraft", scalar3d_layout_mid, q_unit, - grid_name); // updraft velocity [m/s] - // MUST FIXME: Remove qqcw from here - /*add_field( - "qqcw", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - q_unit, grid_name); // cloud-borne aerosol mass, number mixing ratios - // [#/kg or kg/kg]*/ + grid_name); // updraft velocity // MUST FIXME: move it above so that all "required" variables are together add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction [nondimentional] @@ -1002,13 +979,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data w_sec_ = get_field_in("w_sec").get_view(); - /*state_q_ = get_field_in("state_q") - .get_view(); // MUST FIXME: remove this*/ ncldwtr_ = get_field_in("ncldwtr") .get_view(); // MUST FIXME: is is nc, may be not??? - /*qqcw_input_ = get_field_in("qqcw") - .get_view(); // MUST FIXME: remove this*/ dgnum_ = get_field_in("dgnum") .get_view(); // MUST FIXME: is it an input, can // we compute it using calcsize??? @@ -1131,8 +1104,6 @@ void MAMAci::initialize_impl(const RunType run_type) { for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. - /*Kokkos::resize(qqcw_[i], ncol_, nlev_); // MUST FIXME: remove this*/ - Kokkos::resize(state_q_[i], ncol_, nlev_); Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } @@ -1181,10 +1152,6 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - /*copy_scream_array_to_mam4xx( - team_policy, qqcw_, qqcw_input_, - nlev_); // MUST FIXME: remove this and populate qqcw here*/ - // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion @@ -1237,14 +1204,13 @@ void MAMAci::run_impl(const double dt) { dry_atmosphere_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. - call_function_dropmixnuc(team_policy, dry_atmosphere_, dry_aero_, dtmicro_, - raercol_cw_, raercol_, qqcw_fld_work_, - /*qqcw_,*/ ptend_q_, coltend_, coltend_cw_, - dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, - /*state_q_,*/ state_q_work_, ncldwtr_, kvh_, qcld_, - wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, - factnum_, ndropcol_, ndropmix_, nsource_, wtke_, - ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + call_function_dropmixnuc( + team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, + dry_atmosphere_.p_del, rpdel_, state_q_work_, ncldwtr_, kvh_, qcld_, + wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, + ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, + nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. #if 0 copy_mam4xx_array_to_scream(team_policy, ptend_q_output_, ptend_q_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index b6e52026801d..88d91b999cda 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -62,7 +62,6 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - view_2d state_q_[mam4::ndrop::ncnst_tot]; const_view_2d ncldwtr_; view_2d cloud_frac_new_; From fb40cd6dcbae8bc893698e9f58aa14ec6afb67fb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 18 Mar 2024 17:59:05 -0700 Subject: [PATCH 270/476] Adds a comment --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index dd415e970c69..776a0acfdf40 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -511,8 +511,10 @@ void call_function_dropmixnuc( mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; + state_q_work_loc(icol, klev, icnst) = + state_q_at_lev_col[icnst]; // FIXME: ensure that indices are + // right! remove "15" if possible!! + qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; } } From 7a16e65b38a70a3bcd1f8e7eb361b0fb1816a2eb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 19 Mar 2024 21:54:13 -0700 Subject: [PATCH 271/476] Modified species order to match E3SM with the folling change in mam4xx mam4xx/aero_model.hpp /// Identifiers for aerosol species that inhabit MAM4 modes. enum class AeroId { SOA = 2, // secondary organic aerosol SO4 = 0, // sulphate POM = 1, // primary organic matter BC = 3, // black carbon NaCl = 5, // sodium chloride DST = 4, // dust MOM = 6, // marine organic matter, None = 7 // invalid aerosol species }; --- .../mam/eamxx_mam_aci_process_interface.cpp | 245 ++--- .../eamxx/src/physics/mam/mam_coupling.hpp | 861 +++++++++--------- .../eamxx/tests/uncoupled/mam4/aci/input.yaml | 7 +- 3 files changed, 571 insertions(+), 542 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 776a0acfdf40..ee8fb3b130ed 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -602,111 +602,112 @@ void call_hetfrz_compute_tendencies( mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; - ; MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d nc = wet_atmosphere.nc; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies - // class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, - ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, - dummy, dummy, dummy, dummy, dummy, pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = - ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = - ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = - ekat::subview(activation_fraction_coarse_idx, icol); - - // These are the output tendencies from heterogeneous freezing that need - // to be added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = - ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = - ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = - ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0; - hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, - diags, tends); - }); + // Kokkos::parallel_for( + // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + // const int icol = team.league_rank(); + for(int icol = 0; icol < 1; icol++) { + // Set up an atmosphere, surface, diagnostics, pronostics and tendencies + // class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, + ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, dummy, + dummy, dummy, dummy, dummy, pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + // BALLI + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = + ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = + ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = + ekat::subview(activation_fraction_coarse_idx, icol); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0, dt = 0; + hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, progs, + diags, tends); + //}); + } } } // namespace @@ -880,6 +881,7 @@ void MAMAci::set_grids( mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + std::cout << "mode:" << mode << " name:" << int_nmr_field_name << std::endl; // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -888,23 +890,30 @@ void MAMAci::set_grids( mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); + // std::cout<<"mode:"<< <<" name:"<< < 0) + if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - + std::cout << "--> mode:" << mode << " spec:" << a + << " int-spec-name:" << int_mmr_field_name << std::endl; + } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) + if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); + // std::cout<<"--> mode:"<< mode<<" spec:"<< a<< " cld-spec name:"<< + // cld_mmr_field_name<(team_policy, ptend_q_output_, ptend_q_, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_outp_, coltend_, nlev_); - copy_mam4xx_array_to_scream(team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - - call_hetfrz_compute_tendencies(team_policy, - hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, - stratiform_cloud_fraction_, - activation_fraction_accum_idx_, activation_fraction_coarse_idx_, - hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, - naai_hom_, naai_, - diagnostic_scratch_, - nlev_); - - Kokkos::fence(); // wait before returning to calling function -#endif + + copy_mam4xx_array_to_scream( + team_policy, ptend_q_output_, ptend_q_, nlev_); + copy_mam4xx_array_to_scream( + team_policy, coltend_outp_, coltend_, nlev_); + copy_mam4xx_array_to_scream( + team_policy, coltend_cw_outp_, coltend_cw_, nlev_); + + call_hetfrz_compute_tendencies( + team_policy, hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, + stratiform_cloud_fraction_, activation_fraction_accum_idx_, + activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, + hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, + naai_hom_, naai_, diagnostic_scratch_, nlev_); + + Kokkos::fence(); // wait before returning to calling function } void MAMAci::finalize_impl() { diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index c864a7275502..9cc487dbe28c 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -1,9 +1,9 @@ #ifndef MAM_COUPLING_HPP #define MAM_COUPLING_HPP -#include -#include #include +#include +#include #include #include @@ -13,7 +13,7 @@ namespace scream::mam_coupling { -using KT = ekat::KokkosTypes; +using KT = ekat::KokkosTypes; // views for single- and multi-column data using view_1d = typename KT::template view_1d; @@ -27,8 +27,10 @@ using const_view_3d = typename KT::template view_3d; using Team = Kokkos::TeamPolicy::member_type; // unmanaged views (for buffer and workspace manager) -using uview_1d = typename ekat::template Unmanaged>; -using uview_2d = typename ekat::template Unmanaged>; +using uview_1d = + typename ekat::template Unmanaged>; +using uview_2d = + typename ekat::template Unmanaged>; using PF = scream::PhysicsFunctions; @@ -45,21 +47,15 @@ constexpr int nqtendbb() { return 4; } // returns the number of distinct aerosol modes KOKKOS_INLINE_FUNCTION -constexpr int num_aero_modes() { - return mam4::AeroConfig::num_modes(); -} +constexpr int num_aero_modes() { return mam4::AeroConfig::num_modes(); } // returns the number of distinct aerosol species KOKKOS_INLINE_FUNCTION -constexpr int num_aero_species() { - return mam4::AeroConfig::num_aerosol_ids(); -} +constexpr int num_aero_species() { return mam4::AeroConfig::num_aerosol_ids(); } // returns the number of distinct aerosol-related gases KOKKOS_INLINE_FUNCTION -constexpr int num_aero_gases() { - return mam4::AeroConfig::num_gas_ids(); -} +constexpr int num_aero_gases() { return mam4::AeroConfig::num_gas_ids(); } // returns the total number of aerosol tracers (i.e. the total number of // distinct valid mode-species pairs) @@ -73,12 +69,12 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. KOKKOS_INLINE_FUNCTION -const char* aero_mode_name(const int mode) { +const char *aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { - "1", - "2", - "3", - "4", + "1", + "2", + "3", + "4", }; return mode_names[mode]; } @@ -86,31 +82,18 @@ const char* aero_mode_name(const int mode) { // Given a MAM aerosol species ID, returns a string denoting the symbolic // name of the species. KOKKOS_INLINE_FUNCTION -const char* aero_species_name(const int species_id) { +const char *aero_species_name(const int species_id) { static const char *species_names[num_aero_species()] = { - "soa", - "so4", - "pom", - "bc", - "nacl", - "dst", - "mom", - }; + "so4", "pom", "soa", "bc", "dst", "nacl", "mom"}; return species_names[species_id]; } // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. KOKKOS_INLINE_FUNCTION -const char* gas_species_name(const int gas_id) { - static const char *species_names[num_aero_gases()] = { - "o3", - "h2o2", - "h2so4", - "so2", - "dms", - "soag" - }; +const char *gas_species_name(const int gas_id) { + static const char *species_names[num_aero_gases()] = {"o3", "h2o2", "h2so4", + "so2", "dms", "soag"}; return species_names[gas_id]; } @@ -118,154 +101,155 @@ const char* gas_species_name(const int gas_id) { namespace { KOKKOS_INLINE_FUNCTION -constexpr int max_field_name_len() { - return 128; -} +constexpr int max_field_name_len() { return 128; } KOKKOS_INLINE_FUNCTION -size_t gpu_strlen(const char* s) { +size_t gpu_strlen(const char *s) { size_t l = 0; - while (s[l]) ++l; + while(s[l]) ++l; return l; } KOKKOS_INLINE_FUNCTION void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); - for (size_t i = 0; i < len1; ++i) - concatted[i] = s1[i]; + for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for (size_t i = 0; i < len2; ++i) - concatted[i + len1] = s2[i]; - concatted[len1+len2] = 0; + for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; + concatted[len1 + len2] = 0; } KOKKOS_INLINE_FUNCTION -void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { +void concat_3_strings(const char *s1, const char *s2, const char *s3, + char *concatted) { size_t len1 = gpu_strlen(s1); - for (size_t i = 0; i < len1; ++i) - concatted[i] = s1[i]; + for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for (size_t i = 0; i < len2; ++i) - concatted[i + len1] = s2[i]; + for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; size_t len3 = gpu_strlen(s3); - for (size_t i = 0; i < len3; ++i) - concatted[i + len1 + len2] = s3[i]; - concatted[len1+len2+len3] = 0; + for(size_t i = 0; i < len3; ++i) concatted[i + len1 + len2] = s3[i]; + concatted[len1 + len2 + len3] = 0; } KOKKOS_INLINE_FUNCTION -char* int_aero_nmr_names(int mode) { +char *int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char* cld_aero_nmr_names(int mode) { +char *cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char* int_aero_mmr_names(int mode, int species) { - static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; +char *int_aero_mmr_names(int mode, int species) { + static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()] + [max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char* cld_aero_mmr_names(int mode, int species) { - static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; +char *cld_aero_mmr_names(int mode, int species) { + static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()] + [max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char* gas_mmr_names(int gas_id) { +char *gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; } -} // end anonymous namespace +} // end anonymous namespace // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") KOKKOS_INLINE_FUNCTION -const char* int_aero_nmr_field_name(const int mode) { - if (!int_aero_nmr_names(mode)[0]) { +const char *int_aero_nmr_field_name(const int mode) { + if(!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); } - return const_cast(int_aero_nmr_names(mode)); + return const_cast(int_aero_nmr_names(mode)); } // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") KOKKOS_INLINE_FUNCTION -const char* cld_aero_nmr_field_name(const int mode) { - if (!cld_aero_nmr_names(mode)[0]) { +const char *cld_aero_nmr_field_name(const int mode) { + if(!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); } - return const_cast(cld_aero_nmr_names(mode)); + return const_cast(cld_aero_nmr_names(mode)); } // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant interstitial mass mixing ratio -// field in EAMxx. The form of the field name is "_a<1-based-mode-index>". -// If the desired species is not present within the desire mode, returns a blank -// string (""). +// field in EAMxx. The form of the field name is +// "_a<1-based-mode-index>". If the desired species is not present +// within the desire mode, returns a blank string (""). KOKKOS_INLINE_FUNCTION -const char* int_aero_mmr_field_name(const int mode, const int species) { - if (!int_aero_mmr_names(mode, species)[0]) { +const char *int_aero_mmr_field_name(const int mode, const int species) { + if(!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if (aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), - "_a", aero_mode_name(mode), - int_aero_mmr_names(mode, species)); + if(aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), "_a", + aero_mode_name(mode), int_aero_mmr_names(mode, species)); } } - return const_cast(int_aero_mmr_names(mode, species)); + return const_cast(int_aero_mmr_names(mode, species)); }; // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant cloudborne mass mixing ratio -// field in EAMxx. The form of the field name is "_c<1-based-mode-index>". -// If the desired species is not present within the desire mode, returns a blank -// string (""). +// field in EAMxx. The form of the field name is +// "_c<1-based-mode-index>". If the desired species is not present +// within the desire mode, returns a blank string (""). KOKKOS_INLINE_FUNCTION -const char* cld_aero_mmr_field_name(const int mode, const int species) { - if (!cld_aero_mmr_names(mode, species)[0]) { +const char *cld_aero_mmr_field_name(const int mode, const int species) { + if(!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if (aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), - "_c", aero_mode_name(mode), - cld_aero_mmr_names(mode, species)); + if(aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), "_c", + aero_mode_name(mode), cld_aero_mmr_names(mode, species)); } } - return const_cast(cld_aero_mmr_names(mode, species)); + return const_cast(cld_aero_mmr_names(mode, species)); }; // Given a MAM aerosol-related gas identifier, returns the name of its mass // mixing ratio field in EAMxx ("aero_gas_mmr_") KOKKOS_INLINE_FUNCTION -const char* gas_mmr_field_name(const int gas) { - if (!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); +const char *gas_mmr_field_name(const int gas) { + if(!gas_mmr_names(gas)[0]) { + concat_2_strings("aero_gas_mmr_", gas_species_name(gas), + gas_mmr_names(gas)); } - return const_cast(gas_mmr_names(gas)); + return const_cast(gas_mmr_names(gas)); } // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { - const_view_2d qv; // wet water vapor specific humidity [kg vapor / kg moist air] - const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud water/kg moist air] - const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] - const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] - const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] - const_view_2d omega; // vertical pressure velocity [Pa/s] + const_view_2d + qv; // wet water vapor specific humidity [kg vapor / kg moist air] + const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud + // water/kg moist air] + const_view_2d + nc; // wet cloud liquid water number mixing ratio [# / kg moist air] + const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice + // water / kg moist air] + const_view_2d + ni; // wet cloud ice water number mixing ratio [# / kg moist air] + const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores multi-column views related to the dry atmospheric state // used by MAM. struct DryAtmosphere { +<<<<<<< HEAD Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] const_view_2d p_mid; // total pressure at grid midpoints [Pa] @@ -283,20 +267,50 @@ struct DryAtmosphere { view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] +======= + Real z_surf; // height of bottom of atmosphere [m] + const_view_2d T_mid; // temperature at grid midpoints [K] + const_view_2d p_mid; // total pressure at grid midpoints [Pa] + view_2d qv; // dry water vapor mixing ratio [kg vapor / kg dry air] + view_2d qc; // dry cloud liquid water mass mixing ratio [kg cloud water/kg + // dry air] + view_2d nc; // dry cloud liquid water number mixing ratio [# / kg dry air] + view_2d qi; // dry cloud ice water mass mixing ratio [kg cloud ice water / kg + // dry air] + view_2d ni; // dry cloud ice water number mixing ratio [# / kg dry air] + view_2d z_mid; // height at layer midpoints [m] + view_2d z_iface; // height at layer interfaces [m] + view_2d dz; // layer thickness [m] + const_view_2d + p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] + const_view_2d p_int; // total pressure at grid interfaces [Pa] + const_view_2d cldfrac; // cloud fraction [-] + view_2d w_updraft; // updraft velocity [m/s] + const_view_1d pblh; // planetary boundary layer height [m] + const_view_1d phis; // surface geopotential [m2/s2] +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It // can be used to represent wet and dry aerosols. When you declare an // AerosolState, you must decide whether it's a dry or wet aerosol state (with // mixing ratios in terms of dry or wet parcels of air, respectively). -// These mixing ratios are organized by mode (and species, for mass mixing ratio) -// in the same way as they are in mam4xx, and indexed using mam4::AeroConfig. +// These mixing ratios are organized by mode (and species, for mass mixing +// ratio) in the same way as they are in mam4xx, and indexed using +// mam4::AeroConfig. struct AerosolState { - view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number mixing ratios [# / kg air] - view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number mixing ratios [# / kg air] - view_2d int_aero_mmr[num_aero_modes()][num_aero_species()]; // interstitial aerosol mass mixing ratios [kg aerosol / kg air] - view_2d cld_aero_mmr[num_aero_modes()][num_aero_species()]; // cloudborne aerosol mass mixing ratios [kg aerosol / kg air] - view_2d gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] + view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number + // mixing ratios [# / kg air] + view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number + // mixing ratios [# / kg air] + view_2d int_aero_mmr[num_aero_modes()] + [num_aero_species()]; // interstitial aerosol mass mixing + // ratios [kg aerosol / kg air] + view_2d cld_aero_mmr[num_aero_modes()] + [num_aero_species()]; // cloudborne aerosol mass mixing + // ratios [kg aerosol / kg air] + view_2d + gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] }; // storage for variables used within MAM atmosphere processes, initialized with @@ -312,20 +326,20 @@ struct Buffer { static constexpr int num_2d_scratch = 10; // number of local fields stored at column midpoints - static constexpr int num_2d_mid = 8 + // number of dry atm fields - 2 * (num_aero_modes() + num_aero_tracers()) + - num_aero_gases() + - num_2d_scratch; + static constexpr int num_2d_mid = + 8 + // number of dry atm fields + 2 * (num_aero_modes() + num_aero_tracers()) + num_aero_gases() + + num_2d_scratch; // (dry) atmospheric state - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - uview_2d qv_dry; // dry water vapor mixing ratio (dry air) - uview_2d qc_dry; // dry cloud water mass mixing ratio - uview_2d nc_dry; // dry cloud water number mixing ratio - uview_2d qi_dry; // cloud ice mass mixing ratio - uview_2d ni_dry; // dry cloud ice number mixing ratio - uview_2d w_updraft; // vertical wind velocity + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // dry water vapor mixing ratio (dry air) + uview_2d qc_dry; // dry cloud water mass mixing ratio + uview_2d nc_dry; // dry cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d ni_dry; // dry cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity // aerosol dry interstitial/cloudborne number/mass mixing ratios // (because the number of species per mode varies, not all of these will @@ -348,125 +362,86 @@ struct Buffer { // number of local fields stored at column interfaces static constexpr int num_2d_iface = 1; - uview_2d z_iface; // height at interfaces + uview_2d z_iface; // height at interfaces // storage - Real* wsm_data; + Real *wsm_data; }; // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels inline size_t buffer_size(const int ncol, const int nlev) { - return sizeof(Real) * - (Buffer::num_2d_mid * ncol * nlev + - Buffer::num_2d_iface * ncol * (nlev+1)); + return sizeof(Real) * (Buffer::num_2d_mid * ncol * nlev + + Buffer::num_2d_iface * ncol * (nlev + 1)); } // ON HOST, initializeѕ the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. inline size_t init_buffer(const ATMBufferManager &buffer_manager, - const int ncol, const int nlev, - Buffer &buffer) { - Real* mem = reinterpret_cast(buffer_manager.get_memory()); + const int ncol, const int nlev, Buffer &buffer) { + Real *mem = reinterpret_cast(buffer_manager.get_memory()); // set view pointers for midpoint fields - uview_2d* view_2d_mid_ptrs[Buffer::num_2d_mid] = { - &buffer.z_mid, - &buffer.dz, - &buffer.qv_dry, - &buffer.qc_dry, - &buffer.nc_dry, - &buffer.qi_dry, - &buffer.ni_dry, - &buffer.w_updraft, - - // aerosol modes - &buffer.dry_int_aero_nmr[0], - &buffer.dry_int_aero_nmr[1], - &buffer.dry_int_aero_nmr[2], - &buffer.dry_int_aero_nmr[3], - &buffer.dry_cld_aero_nmr[0], - &buffer.dry_cld_aero_nmr[1], - &buffer.dry_cld_aero_nmr[2], - &buffer.dry_cld_aero_nmr[3], - - // the following requires knowledge of mam4's mode-species layout - // (see mode_aero_species() in mam4xx/aero_modes.hpp) - - // accumulation mode - &buffer.dry_int_aero_mmr[0][0], - &buffer.dry_int_aero_mmr[0][1], - &buffer.dry_int_aero_mmr[0][2], - &buffer.dry_int_aero_mmr[0][3], - &buffer.dry_int_aero_mmr[0][4], - &buffer.dry_int_aero_mmr[0][5], - &buffer.dry_int_aero_mmr[0][6], - &buffer.dry_cld_aero_mmr[0][0], - &buffer.dry_cld_aero_mmr[0][1], - &buffer.dry_cld_aero_mmr[0][2], - &buffer.dry_cld_aero_mmr[0][3], - &buffer.dry_cld_aero_mmr[0][4], - &buffer.dry_cld_aero_mmr[0][5], - &buffer.dry_cld_aero_mmr[0][6], - - // aitken mode - &buffer.dry_int_aero_mmr[1][0], - &buffer.dry_int_aero_mmr[1][1], - &buffer.dry_int_aero_mmr[1][2], - &buffer.dry_int_aero_mmr[1][3], - &buffer.dry_cld_aero_mmr[1][0], - &buffer.dry_cld_aero_mmr[1][1], - &buffer.dry_cld_aero_mmr[1][2], - &buffer.dry_cld_aero_mmr[1][3], - - // coarse mode - &buffer.dry_int_aero_mmr[2][0], - &buffer.dry_int_aero_mmr[2][1], - &buffer.dry_int_aero_mmr[2][2], - &buffer.dry_int_aero_mmr[2][3], - &buffer.dry_int_aero_mmr[2][4], - &buffer.dry_int_aero_mmr[2][5], - &buffer.dry_int_aero_mmr[2][6], - &buffer.dry_cld_aero_mmr[2][0], - &buffer.dry_cld_aero_mmr[2][1], - &buffer.dry_cld_aero_mmr[2][2], - &buffer.dry_cld_aero_mmr[2][3], - &buffer.dry_cld_aero_mmr[2][4], - &buffer.dry_cld_aero_mmr[2][5], - &buffer.dry_cld_aero_mmr[2][6], - - // primary carbon mode - &buffer.dry_int_aero_mmr[3][0], - &buffer.dry_int_aero_mmr[3][1], - &buffer.dry_int_aero_mmr[3][2], - &buffer.dry_cld_aero_mmr[3][0], - &buffer.dry_cld_aero_mmr[3][1], - &buffer.dry_cld_aero_mmr[3][2], - - // aerosol gases - &buffer.dry_gas_mmr[0], - &buffer.dry_gas_mmr[1], - &buffer.dry_gas_mmr[2], - &buffer.dry_gas_mmr[3], - &buffer.dry_gas_mmr[4], - &buffer.dry_gas_mmr[5] - }; - for (int i = 0; i < Buffer::num_2d_scratch; ++i) { - view_2d_mid_ptrs[Buffer::num_2d_mid+i-Buffer::num_2d_scratch] = &buffer.scratch[i]; + uview_2d *view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer.z_mid, &buffer.dz, &buffer.qv_dry, &buffer.qc_dry, &buffer.nc_dry, + &buffer.qi_dry, &buffer.ni_dry, &buffer.w_updraft, + + // aerosol modes + &buffer.dry_int_aero_nmr[0], &buffer.dry_int_aero_nmr[1], + &buffer.dry_int_aero_nmr[2], &buffer.dry_int_aero_nmr[3], + &buffer.dry_cld_aero_nmr[0], &buffer.dry_cld_aero_nmr[1], + &buffer.dry_cld_aero_nmr[2], &buffer.dry_cld_aero_nmr[3], + + // the following requires knowledge of mam4's mode-species layout + // (see mode_aero_species() in mam4xx/aero_modes.hpp) + + // accumulation mode + &buffer.dry_int_aero_mmr[0][0], &buffer.dry_int_aero_mmr[0][1], + &buffer.dry_int_aero_mmr[0][2], &buffer.dry_int_aero_mmr[0][3], + &buffer.dry_int_aero_mmr[0][4], &buffer.dry_int_aero_mmr[0][5], + &buffer.dry_int_aero_mmr[0][6], &buffer.dry_cld_aero_mmr[0][0], + &buffer.dry_cld_aero_mmr[0][1], &buffer.dry_cld_aero_mmr[0][2], + &buffer.dry_cld_aero_mmr[0][3], &buffer.dry_cld_aero_mmr[0][4], + &buffer.dry_cld_aero_mmr[0][5], &buffer.dry_cld_aero_mmr[0][6], + + // aitken mode + &buffer.dry_int_aero_mmr[1][0], &buffer.dry_int_aero_mmr[1][1], + &buffer.dry_int_aero_mmr[1][2], &buffer.dry_int_aero_mmr[1][3], + &buffer.dry_cld_aero_mmr[1][0], &buffer.dry_cld_aero_mmr[1][1], + &buffer.dry_cld_aero_mmr[1][2], &buffer.dry_cld_aero_mmr[1][3], + + // coarse mode + &buffer.dry_int_aero_mmr[2][0], &buffer.dry_int_aero_mmr[2][1], + &buffer.dry_int_aero_mmr[2][2], &buffer.dry_int_aero_mmr[2][3], + &buffer.dry_int_aero_mmr[2][4], &buffer.dry_int_aero_mmr[2][5], + &buffer.dry_int_aero_mmr[2][6], &buffer.dry_cld_aero_mmr[2][0], + &buffer.dry_cld_aero_mmr[2][1], &buffer.dry_cld_aero_mmr[2][2], + &buffer.dry_cld_aero_mmr[2][3], &buffer.dry_cld_aero_mmr[2][4], + &buffer.dry_cld_aero_mmr[2][5], &buffer.dry_cld_aero_mmr[2][6], + + // primary carbon mode + &buffer.dry_int_aero_mmr[3][0], &buffer.dry_int_aero_mmr[3][1], + &buffer.dry_int_aero_mmr[3][2], &buffer.dry_cld_aero_mmr[3][0], + &buffer.dry_cld_aero_mmr[3][1], &buffer.dry_cld_aero_mmr[3][2], + + // aerosol gases + &buffer.dry_gas_mmr[0], &buffer.dry_gas_mmr[1], &buffer.dry_gas_mmr[2], + &buffer.dry_gas_mmr[3], &buffer.dry_gas_mmr[4], &buffer.dry_gas_mmr[5]}; + for(int i = 0; i < Buffer::num_2d_scratch; ++i) { + view_2d_mid_ptrs[Buffer::num_2d_mid + i - Buffer::num_2d_scratch] = + &buffer.scratch[i]; } - for (int i = 0; i < Buffer::num_2d_mid; ++i) { + for(int i = 0; i < Buffer::num_2d_mid; ++i) { *view_2d_mid_ptrs[i] = view_2d(mem, ncol, nlev); mem += view_2d_mid_ptrs[i]->size(); } // set view pointers for interface fields - uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { - &buffer.z_iface - }; - for (int i = 0; i < Buffer::num_2d_iface; ++i) { - *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev+1); + uview_2d *view_2d_iface_ptrs[Buffer::num_2d_iface] = {&buffer.z_iface}; + for(int i = 0; i < Buffer::num_2d_iface; ++i) { + *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev + 1); mem += view_2d_iface_ptrs[i]->size(); } @@ -476,47 +451,48 @@ inline size_t init_buffer(const ATMBufferManager &buffer_manager, /* // Compute workspace manager size to check used memory vs. requested memory // (if needed) - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); - mem += wsm_size; + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, + 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); mem += wsm_size; */ // return the number of bytes allocated - return (mem - buffer_manager.get_memory())*sizeof(Real); + return (mem - buffer_manager.get_memory()) * sizeof(Real); } // Given a dry atmosphere state, creates a haero::Atmosphere object for the // column with the given index. This object can be provided to mam4xx for the // column. KOKKOS_INLINE_FUNCTION -haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, +haero::Atmosphere atmosphere_for_column(const DryAtmosphere &dry_atm, const int column_index) { EKAT_KERNEL_ASSERT_MSG(dry_atm.T_mid.data() != nullptr, - "T_mid not defined for dry atmosphere state!"); + "T_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_mid.data() != nullptr, - "p_mid not defined for dry atmosphere state!"); + "p_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qv.data() != nullptr, - "qv not defined for dry atmosphere state!"); + "qv not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qc.data() != nullptr, - "qc not defined for dry atmosphere state!"); + "qc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.nc.data() != nullptr, - "nc not defined for dry atmosphere state!"); + "nc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qi.data() != nullptr, - "qi not defined for dry atmosphere state!"); + "qi not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.ni.data() != nullptr, - "ni not defined for dry atmosphere state!"); + "ni not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.z_mid.data() != nullptr, - "z_mid not defined for dry atmosphere state!"); + "z_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del.data() != nullptr, - "p_del not defined for dry atmosphere state!"); + "p_del not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, - "p_int not defined for dry atmosphere state!"); + "p_int not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.cldfrac.data() != nullptr, - "cldfrac not defined for dry atmosphere state!"); + "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, - "w_updraft not defined for dry atmosphere state!"); + "w_updraft not defined for dry atmosphere state!"); return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -538,23 +514,24 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, // ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to // mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, +mam4::Prognostics interstitial_aerosols_for_column(const AerosolState &dry_aero, const int column_index) { constexpr int nlev = mam4::nlev; mam4::Prognostics progs(nlev); - for (int m = 0; m < num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), - "int_aero_nmr not defined for dry aerosol state!"); + "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); - for (int a = 0; a < num_aero_species(); ++a) { - if (dry_aero.int_aero_mmr[m][a].data()) { - progs.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + for(int a = 0; a < num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + progs.q_aero_i[m][a] = + ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } - for (int g = 0; g < num_aero_gases(); ++g) { + for(int g = 0; g < num_aero_gases(); ++g) { EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), - "gas_mmr not defined for dry aerosol state!"); + "gas_mmr not defined for dry aerosol state!"); progs.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); } return progs; @@ -564,16 +541,17 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, +mam4::Prognostics aerosols_for_column(const AerosolState &dry_aero, const int column_index) { auto progs = interstitial_aerosols_for_column(dry_aero, column_index); - for (int m = 0; m < num_aero_modes(); ++m) { + for(int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), - "dry_cld_aero_nmr not defined for aerosol state!"); + "dry_cld_aero_nmr not defined for aerosol state!"); progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); - for (int a = 0; a < num_aero_species(); ++a) { - if (dry_aero.cld_aero_mmr[m][a].data()) { - progs.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + for(int a = 0; a < num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + progs.q_aero_c[m][a] = + ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } @@ -599,26 +577,27 @@ void compute_vertical_layer_heights(const Team& team, }*/ KOKKOS_INLINE_FUNCTION -void compute_vertical_layer_heights(const Team& team, - const DryAtmosphere& dry_atm, +void compute_vertical_layer_heights(const Team &team, + const DryAtmosphere &dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); - const auto dz = ekat::subview(dry_atm.dz, column_index); - const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); - const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); - PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs - dz);//output + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs + dz); // output team.team_barrier(); - PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs - z_iface); //output - team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, // inputs + z_iface); // output + team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } @@ -626,19 +605,23 @@ void compute_vertical_layer_heights(const Team& team, // team to compute the vertical updraft velocity for the column with the given // index. KOKKOS_INLINE_FUNCTION -void compute_updraft_velocities(const Team& team, - const WetAtmosphere& wet_atm, - const DryAtmosphere& dry_atm, +void compute_updraft_velocities(const Team &team, const WetAtmosphere &wet_atm, + const DryAtmosphere &dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); - const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); - dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + dry_atm.dz(i, k) = + PF::calculate_dz(dry_atm.p_del(i, k), dry_atm.p_mid(i, k), + dry_atm.T_mid(i, k), wet_atm.qv(i, k)); + const auto rho = + PF::calculate_density(dry_atm.p_del(i, k), dry_atm.dz(i, k)); + dry_atm.w_updraft(i, k) = + PF::calculate_vertical_velocity(wet_atm.omega(i, k), rho); }); } @@ -646,93 +629,108 @@ void compute_updraft_velocities(const Team& team, // from the team to compute mixing ratios for a dry atmosphere state in th // column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team& team, - const WetAtmosphere& wet_atm, - const DryAtmosphere& dry_atm, +void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, + const DryAtmosphere &dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + const auto qv_ik = wet_atm.qv(i, k); + dry_atm.qv(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i, k), qv_ik); + dry_atm.qc(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i, k), qv_ik); + dry_atm.nc(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i, k), qv_ik); + dry_atm.qi(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i, k), qv_ik); + dry_atm.ni(i, k) = + PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i, k), qv_ik); }); } -// Given a thread team and wet atmospheric and aerosol states, dispatches threads -// from the team to compute mixing ratios for the given dry interstitial aerosol -// state for the column with the given index. +// Given a thread team and wet atmospheric and aerosol states, dispatches +// threads from the team to compute mixing ratios for the given dry interstitial +// aerosol state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team& team, - const WetAtmosphere& wet_atm, - const AerosolState& wet_aero, - const AerosolState& dry_aero, +void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, + const AerosolState &wet_aero, + const AerosolState &dry_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto qv_ik = wet_atm.qv(i,k); - for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); - if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + const auto qv_ik = wet_atm.qv(i, k); + for(int m = 0; m < num_aero_modes(); ++m) { + dry_aero.int_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.int_aero_nmr[m](i, k), qv_ik); + if(dry_aero.cld_aero_nmr[m].data()) { + dry_aero.cld_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.cld_aero_nmr[m](i, k), qv_ik); } - for (int a = 0; a < num_aero_species(); ++a) { - if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + for(int a = 0; a < num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.int_aero_mmr[m][a](i, k), qv_ik); } - if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + if(dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( + wet_aero.cld_aero_mmr[m][a](i, k), qv_ik); } } } - for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + for(int g = 0; g < num_aero_gases(); ++g) { + dry_aero.gas_mmr[g](i, k) = + PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i, k), qv_ik); } }); } -// Given a thread team and dry atmospheric and aerosol states, dispatches threads -// from the team to compute mixing ratios for the given wet interstitial aerosol -// state for the column with the given index. +// Given a thread team and dry atmospheric and aerosol states, dispatches +// threads from the team to compute mixing ratios for the given wet interstitial +// aerosol state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_wet_mixing_ratios(const Team& team, - const DryAtmosphere& dry_atm, - const AerosolState& dry_aero, - const AerosolState& wet_aero, +void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, + const AerosolState &dry_aero, + const AerosolState &wet_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG( + column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { - const auto qv_ik = dry_atm.qv(i,k); - for (int m = 0; m < num_aero_modes(); ++m) { - wet_aero.int_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_nmr[m](i,k), qv_ik); - if (wet_aero.cld_aero_nmr[m].data()) { - wet_aero.cld_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_nmr[m](i,k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { + const auto qv_ik = dry_atm.qv(i, k); + for(int m = 0; m < num_aero_modes(); ++m) { + wet_aero.int_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.int_aero_nmr[m](i, k), qv_ik); + if(wet_aero.cld_aero_nmr[m].data()) { + wet_aero.cld_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.cld_aero_nmr[m](i, k), qv_ik); } - for (int a = 0; a < num_aero_species(); ++a) { - if (wet_aero.int_aero_mmr[m][a].data()) { - wet_aero.int_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_mmr[m][a](i,k), qv_ik); + for(int a = 0; a < num_aero_species(); ++a) { + if(wet_aero.int_aero_mmr[m][a].data()) { + wet_aero.int_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.int_aero_mmr[m][a](i, k), qv_ik); } - if (wet_aero.cld_aero_mmr[m][a].data()) { - wet_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_mmr[m][a](i,k), qv_ik); + if(wet_aero.cld_aero_mmr[m][a].data()) { + wet_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( + dry_aero.cld_aero_mmr[m][a](i, k), qv_ik); } } } - for (int g = 0; g < num_aero_gases(); ++g) { - wet_aero.gas_mmr[g](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i,k), qv_ik); + for(int g = 0; g < num_aero_gases(); ++g) { + wet_aero.gas_mmr[g](i, k) = + PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i, k), qv_ik); } }); } @@ -742,50 +740,55 @@ void compute_wet_mixing_ratios(const Team& team, // these constants where needed within two such functions so we don't define // them inconsistently. Yes, it's the 21st century and we're still struggling // with these basic things. -#define DECLARE_PROG_TRANSFER_CONSTANTS \ - /* mapping of constituent indices to aerosol modes */ \ - const auto Accum = mam4::ModeIndex::Accumulation; \ - const auto Aitken = mam4::ModeIndex::Aitken; \ - const auto Coarse = mam4::ModeIndex::Coarse; \ - const auto PC = mam4::ModeIndex::PrimaryCarbon; \ - const auto NoMode = mam4::ModeIndex::None; \ - static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ - Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ - Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ - Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ - PC, PC, PC, PC, /* 3 aero species + NMR */ \ - }; \ - /* mapping of constituent indices to aerosol species */ \ - const auto SOA = mam4::AeroId::SOA; \ - const auto SO4 = mam4::AeroId::SO4; \ - const auto POM = mam4::AeroId::POM; \ - const auto BC = mam4::AeroId::BC; \ - const auto NaCl = mam4::AeroId::NaCl; \ - const auto DST = mam4::AeroId::DST; \ - const auto MOM = mam4::AeroId::MOM; \ - const auto NoAero = mam4::AeroId::None; \ - static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ - NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ - SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ - SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ - DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ - POM, BC, MOM, NoAero, /* primary carbon mode */ \ - }; \ - /* mapping of constituent indices to gases */ \ - const auto O3 = mam4::GasId::O3; \ - const auto H2O2 = mam4::GasId::H2O2; \ - const auto H2SO4 = mam4::GasId::H2SO4; \ - const auto SO2 = mam4::GasId::SO2; \ - const auto DMS = mam4::GasId::DMS; \ - const auto SOAG = mam4::GasId::SOAG; \ - const auto NoGas = mam4::GasId::None; \ - static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ - O3, H2O2, H2SO4, SO2, DMS, SOAG, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, \ +#define DECLARE_PROG_TRANSFER_CONSTANTS \ + /* mapping of constituent indices to aerosol modes */ \ + const auto Accum = mam4::ModeIndex::Accumulation; \ + const auto Aitken = mam4::ModeIndex::Aitken; \ + const auto Coarse = mam4::ModeIndex::Coarse; \ + const auto PC = mam4::ModeIndex::PrimaryCarbon; \ + const auto NoMode = mam4::ModeIndex::None; \ + static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) \ + */ \ + Accum, Accum, Accum, Accum, Accum, Accum, Accum, \ + Accum, /* 7 aero species + NMR */ \ + Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ + Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, \ + Coarse, /* 7 aero species + NMR */ \ + PC, PC, PC, PC, /* 3 aero species + NMR */ \ + }; \ + /* mapping of constituent indices to aerosol species */ \ + const auto SOA = mam4::AeroId::SOA; \ + const auto SO4 = mam4::AeroId::SO4; \ + const auto POM = mam4::AeroId::POM; \ + const auto BC = mam4::AeroId::BC; \ + const auto NaCl = mam4::AeroId::NaCl; \ + const auto DST = mam4::AeroId::DST; \ + const auto MOM = mam4::AeroId::MOM; \ + const auto NoAero = mam4::AeroId::None; \ + static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ + NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) \ + */ \ + SO4, POM, SOA, BC, DST, NaCl, MOM, \ + NoAero, /* accumulation mode */ \ + SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ + DST, NaCl, SO4, BC, POM, SOA, MOM, \ + NoAero, /* coarse mode */ \ + POM, BC, MOM, NoAero, /* primary carbon mode */ \ + }; \ + /* mapping of constituent indices to gases */ \ + const auto O3 = mam4::GasId::O3; \ + const auto H2O2 = mam4::GasId::H2O2; \ + const auto H2SO4 = mam4::GasId::H2SO4; \ + const auto SO2 = mam4::GasId::SO2; \ + const auto DMS = mam4::GasId::DMS; \ + const auto SOAG = mam4::GasId::SOAG; \ + const auto NoGas = mam4::GasId::None; \ + static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ + O3, H2O2, H2SO4, SO2, DMS, SOAG, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ }; // Given a Prognostics object, transfers data for interstitial aerosols to the @@ -798,30 +801,29 @@ void compute_wet_mixing_ratios(const Team& team, // NOTE: indices KOKKOS_INLINE_FUNCTION void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, - const int k, - Real q[gas_pcnst()], + const int k, Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - q[i] = progs.q_gas[g](k); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + q[i] = progs.q_gas[g](k); qqcw[i] = progs.q_gas[g](k); } else { int m = static_cast(mode_index); - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - q[i] = progs.q_aero_i[m][a](k); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + q[i] = progs.q_aero_i[m][a](k); qqcw[i] = progs.q_aero_c[m][a](k); - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); - q[i] = progs.n_mode_i[m](k); + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + q[i] = progs.n_mode_i[m](k); qqcw[i] = progs.n_mode_c[m](k); } } @@ -837,23 +839,29 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], Real vmrcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { +<<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); +======= + int m = static_cast(mode_index); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx const Real mw = mam4::aero_species(a).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); - } else { // constituent is a modal number mixing ratio - vmr[i] = q[i]; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { // constituent is a modal number mixing ratio + vmr[i] = q[i]; vmrcw[i] = qqcw[i]; } } @@ -865,27 +873,32 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], KOKKOS_INLINE_FUNCTION void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], const Real vmrcw[gas_pcnst()], - Real q[gas_pcnst()], - Real qqcw[gas_pcnst()]) { + Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { +<<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); +======= + int m = static_cast(mode_index); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx const Real mw = mam4::aero_species(a).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); - } else { // constituent is a modal number mixing ratio - q[i] = vmr[i]; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { // constituent is a modal number mixing ratio + q[i] = vmr[i]; qqcw[i] = vmrcw[i]; } } @@ -898,25 +911,31 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], KOKKOS_INLINE_FUNCTION void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], const Real qqcw[gas_pcnst()], - mam4::Prognostics &progs, const int k) { + mam4::Prognostics &progs, + const int k) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for (int i = 0; i < gas_pcnst(); ++i) { + for(int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if (gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if(gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); progs.q_gas[g](k) = q[i]; } else { int m = static_cast(mode_index); - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); + if(aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; +<<<<<<< HEAD } else { // constituent is a modal number mixing ratio +======= + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); +>>>>>>> Modified species order to match E3SM with the folling change in mam4xx progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } @@ -926,6 +945,6 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], #undef DECLARE_PROG_TRANSFER_CONSTANTS -} // namespace scream::mam_coupling +} // namespace scream::mam_coupling #endif diff --git a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml index 946d11dcc0e0..298da6d50adb 100644 --- a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml +++ b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml @@ -6,7 +6,8 @@ driver_options: time_stepping: time_step: ${ATM_TIME_STEP} run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} + #number_of_steps: ${NUM_STEPS} + number_of_steps: 1 atmosphere_processes: atm_procs_list: [mam4_aci] @@ -22,8 +23,8 @@ grids_manager: number_of_vertical_levels: 72 initial_conditions: - # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + # The name of the file containing the initial conditions for this test. FIXME: use a variable for the filename here + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc T_mid: 273.0 p_mid: 1.e5 p_int: 1.e5 From b841ffc427b7adbaa1bb49782f666d69a857e9a0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 23 Mar 2024 12:49:58 -0700 Subject: [PATCH 272/476] Reverted back mam_coupling changes for indexing but added a func to get prog index Changes for formatting and species reordering are reverted back in mam_coupling. Adds a new function in mam_couplingto map indexing to the prognostic array index. I changed the following mam4xx/aero_modes.cpp function (this is the only change in this file) to change species ordering: KOKKOS_INLINE_FUNCTION AeroId mode_aero_species(const int modeNo, const int speciesNo) { // A list of species within each mode for MAM4. static constexpr AeroId mode_aero_species[4][7] = { {// accumulation mode AeroId::SO4, AeroId::POM, AeroId::SOA, AeroId::BC, AeroId::DST, AeroId::NaCl, AeroId::MOM}, { // aitken mode AeroId::SO4, AeroId::SOA, AeroId::NaCl, AeroId::MOM, AeroId::None, AeroId::None, AeroId::None, }, {// coarse mode AeroId::DST, AeroId::NaCl, AeroId::SO4, AeroId::BC, AeroId::POM, AeroId::SOA, AeroId::MOM}, {// primary carbon mode AeroId::POM, AeroId::BC, AeroId::MOM, AeroId::None, AeroId::None, AeroId::None, AeroId::None}}; return mode_aero_species[modeNo][speciesNo]; } --- .../mam/eamxx_mam_aci_process_interface.cpp | 15 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 857 ++++++++++-------- 2 files changed, 466 insertions(+), 406 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ee8fb3b130ed..2e32e0670771 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -686,11 +686,11 @@ void call_hetfrz_compute_tendencies( diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + // diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); // naai and naai_hom are the outputs needed for nucleate_ice and these // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + /*diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); //------------------------------------------------------------- @@ -703,10 +703,11 @@ void call_hetfrz_compute_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0; - hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, progs, - diags, tends); - //}); + const Real t = 0, dt = 0;*/ + // hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, + // progs, + // diags, tends); + // }); } } } // namespace @@ -920,6 +921,8 @@ void MAMAci::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + // std::cout << "--> Gasses:" << g <<" spec:"< -#include #include +#include +#include #include #include @@ -13,7 +13,7 @@ namespace scream::mam_coupling { -using KT = ekat::KokkosTypes; +using KT = ekat::KokkosTypes; // views for single- and multi-column data using view_1d = typename KT::template view_1d; @@ -27,10 +27,8 @@ using const_view_3d = typename KT::template view_3d; using Team = Kokkos::TeamPolicy::member_type; // unmanaged views (for buffer and workspace manager) -using uview_1d = - typename ekat::template Unmanaged>; -using uview_2d = - typename ekat::template Unmanaged>; +using uview_1d = typename ekat::template Unmanaged>; +using uview_2d = typename ekat::template Unmanaged>; using PF = scream::PhysicsFunctions; @@ -47,15 +45,21 @@ constexpr int nqtendbb() { return 4; } // returns the number of distinct aerosol modes KOKKOS_INLINE_FUNCTION -constexpr int num_aero_modes() { return mam4::AeroConfig::num_modes(); } +constexpr int num_aero_modes() { + return mam4::AeroConfig::num_modes(); +} // returns the number of distinct aerosol species KOKKOS_INLINE_FUNCTION -constexpr int num_aero_species() { return mam4::AeroConfig::num_aerosol_ids(); } +constexpr int num_aero_species() { + return mam4::AeroConfig::num_aerosol_ids(); +} // returns the number of distinct aerosol-related gases KOKKOS_INLINE_FUNCTION -constexpr int num_aero_gases() { return mam4::AeroConfig::num_gas_ids(); } +constexpr int num_aero_gases() { + return mam4::AeroConfig::num_gas_ids(); +} // returns the total number of aerosol tracers (i.e. the total number of // distinct valid mode-species pairs) @@ -69,12 +73,12 @@ constexpr int num_aero_tracers() { // Given a MAM aerosol mode index, returns a string denoting the symbolic // name of the mode. KOKKOS_INLINE_FUNCTION -const char *aero_mode_name(const int mode) { +const char* aero_mode_name(const int mode) { static const char *mode_names[num_aero_modes()] = { - "1", - "2", - "3", - "4", + "1", + "2", + "3", + "4", }; return mode_names[mode]; } @@ -82,18 +86,31 @@ const char *aero_mode_name(const int mode) { // Given a MAM aerosol species ID, returns a string denoting the symbolic // name of the species. KOKKOS_INLINE_FUNCTION -const char *aero_species_name(const int species_id) { +const char* aero_species_name(const int species_id) { static const char *species_names[num_aero_species()] = { - "so4", "pom", "soa", "bc", "dst", "nacl", "mom"}; + "soa", + "so4", + "pom", + "bc", + "nacl", + "dst", + "mom", + }; return species_names[species_id]; } // Given a MAM aerosol-related gas ID, returns a string denoting the symbolic // name of the gas species. KOKKOS_INLINE_FUNCTION -const char *gas_species_name(const int gas_id) { - static const char *species_names[num_aero_gases()] = {"o3", "h2o2", "h2so4", - "so2", "dms", "soag"}; +const char* gas_species_name(const int gas_id) { + static const char *species_names[num_aero_gases()] = { + "o3", + "h2o2", + "h2so4", + "so2", + "dms", + "soag" + }; return species_names[gas_id]; } @@ -101,155 +118,158 @@ const char *gas_species_name(const int gas_id) { namespace { KOKKOS_INLINE_FUNCTION -constexpr int max_field_name_len() { return 128; } +constexpr int max_field_name_len() { + return 128; +} KOKKOS_INLINE_FUNCTION -size_t gpu_strlen(const char *s) { +size_t gpu_strlen(const char* s) { size_t l = 0; - while(s[l]) ++l; + while (s[l]) ++l; return l; } KOKKOS_INLINE_FUNCTION void concat_2_strings(const char *s1, const char *s2, char *concatted) { size_t len1 = gpu_strlen(s1); - for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; - concatted[len1 + len2] = 0; + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; + concatted[len1+len2] = 0; } KOKKOS_INLINE_FUNCTION -void concat_3_strings(const char *s1, const char *s2, const char *s3, - char *concatted) { +void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { size_t len1 = gpu_strlen(s1); - for(size_t i = 0; i < len1; ++i) concatted[i] = s1[i]; + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; size_t len2 = gpu_strlen(s2); - for(size_t i = 0; i < len2; ++i) concatted[i + len1] = s2[i]; + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; size_t len3 = gpu_strlen(s3); - for(size_t i = 0; i < len3; ++i) concatted[i + len1 + len2] = s3[i]; - concatted[len1 + len2 + len3] = 0; + for (size_t i = 0; i < len3; ++i) + concatted[i + len1 + len2] = s3[i]; + concatted[len1+len2+len3] = 0; } KOKKOS_INLINE_FUNCTION -char *int_aero_nmr_names(int mode) { +char* int_aero_nmr_names(int mode) { static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return int_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char *cld_aero_nmr_names(int mode) { +char* cld_aero_nmr_names(int mode) { static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; return cld_aero_nmr_names_[mode]; } KOKKOS_INLINE_FUNCTION -char *int_aero_mmr_names(int mode, int species) { - static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()] - [max_field_name_len()] = {}; +char* int_aero_mmr_names(int mode, int species) { + static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return int_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char *cld_aero_mmr_names(int mode, int species) { - static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()] - [max_field_name_len()] = {}; +char* cld_aero_mmr_names(int mode, int species) { + static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; return cld_aero_mmr_names_[mode][species]; } KOKKOS_INLINE_FUNCTION -char *gas_mmr_names(int gas_id) { +char* gas_mmr_names(int gas_id) { static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; return gas_mmr_names_[gas_id]; } -} // end anonymous namespace +} // end anonymous namespace // Given a MAM aerosol mode index, returns the name of the related interstitial // modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") KOKKOS_INLINE_FUNCTION -const char *int_aero_nmr_field_name(const int mode) { - if(!int_aero_nmr_names(mode)[0]) { +const char* int_aero_nmr_field_name(const int mode) { + if (!int_aero_nmr_names(mode)[0]) { concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); } - return const_cast(int_aero_nmr_names(mode)); + return const_cast(int_aero_nmr_names(mode)); } // Given a MAM aerosol mode index, returns the name of the related cloudborne // modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") KOKKOS_INLINE_FUNCTION -const char *cld_aero_nmr_field_name(const int mode) { - if(!cld_aero_nmr_names(mode)[0]) { +const char* cld_aero_nmr_field_name(const int mode) { + if (!cld_aero_nmr_names(mode)[0]) { concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); } - return const_cast(cld_aero_nmr_names(mode)); + return const_cast(cld_aero_nmr_names(mode)); } // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant interstitial mass mixing ratio -// field in EAMxx. The form of the field name is -// "_a<1-based-mode-index>". If the desired species is not present -// within the desire mode, returns a blank string (""). +// field in EAMxx. The form of the field name is "_a<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). KOKKOS_INLINE_FUNCTION -const char *int_aero_mmr_field_name(const int mode, const int species) { - if(!int_aero_mmr_names(mode, species)[0]) { +const char* int_aero_mmr_field_name(const int mode, const int species) { + if (!int_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if(aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), "_a", - aero_mode_name(mode), int_aero_mmr_names(mode, species)); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_a", aero_mode_name(mode), + int_aero_mmr_names(mode, species)); } } - return const_cast(int_aero_mmr_names(mode, species)); + return const_cast(int_aero_mmr_names(mode, species)); }; // Given a MAM aerosol mode index and the index of the MAM aerosol species // within it, returns the name of the relevant cloudborne mass mixing ratio -// field in EAMxx. The form of the field name is -// "_c<1-based-mode-index>". If the desired species is not present -// within the desire mode, returns a blank string (""). +// field in EAMxx. The form of the field name is "_c<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). KOKKOS_INLINE_FUNCTION -const char *cld_aero_mmr_field_name(const int mode, const int species) { - if(!cld_aero_mmr_names(mode, species)[0]) { +const char* cld_aero_mmr_field_name(const int mode, const int species) { + if (!cld_aero_mmr_names(mode, species)[0]) { const auto aero_id = mam4::mode_aero_species(mode, species); - if(aero_id != mam4::AeroId::None) { - concat_3_strings(aero_species_name(static_cast(aero_id)), "_c", - aero_mode_name(mode), cld_aero_mmr_names(mode, species)); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_c", aero_mode_name(mode), + cld_aero_mmr_names(mode, species)); } } - return const_cast(cld_aero_mmr_names(mode, species)); + return const_cast(cld_aero_mmr_names(mode, species)); }; // Given a MAM aerosol-related gas identifier, returns the name of its mass // mixing ratio field in EAMxx ("aero_gas_mmr_") KOKKOS_INLINE_FUNCTION -const char *gas_mmr_field_name(const int gas) { - if(!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), - gas_mmr_names(gas)); +const char* gas_mmr_field_name(const int gas) { + if (!gas_mmr_names(gas)[0]) { + concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); } - return const_cast(gas_mmr_names(gas)); + return const_cast(gas_mmr_names(gas)); } // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { - const_view_2d - qv; // wet water vapor specific humidity [kg vapor / kg moist air] - const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud - // water/kg moist air] - const_view_2d - nc; // wet cloud liquid water number mixing ratio [# / kg moist air] - const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice - // water / kg moist air] - const_view_2d - ni; // wet cloud ice water number mixing ratio [# / kg moist air] - const_view_2d omega; // vertical pressure velocity [Pa/s] + const_view_2d qv; // wet water vapor specific humidity [kg vapor / kg moist air] + const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud water/kg moist air] + const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] + const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] + const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] + const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores multi-column views related to the dry atmospheric state // used by MAM. struct DryAtmosphere { <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] const_view_2d p_mid; // total pressure at grid midpoints [Pa] @@ -262,11 +282,16 @@ struct DryAtmosphere { view_2d z_iface; // height at layer interfaces [m] view_2d dz; // layer thickness [m] const_view_2d p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] +<<<<<<< HEAD const_view_2d p_int; // total pressure at grid interfaces [Pa] +======= + const_view_2d p_int; // total pressure at grid interfaces [Pa] +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const_view_2d cldfrac; // cloud fraction [-] view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] +<<<<<<< HEAD ======= Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] @@ -289,28 +314,22 @@ struct DryAtmosphere { const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It // can be used to represent wet and dry aerosols. When you declare an // AerosolState, you must decide whether it's a dry or wet aerosol state (with // mixing ratios in terms of dry or wet parcels of air, respectively). -// These mixing ratios are organized by mode (and species, for mass mixing -// ratio) in the same way as they are in mam4xx, and indexed using -// mam4::AeroConfig. +// These mixing ratios are organized by mode (and species, for mass mixing ratio) +// in the same way as they are in mam4xx, and indexed using mam4::AeroConfig. struct AerosolState { - view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number - // mixing ratios [# / kg air] - view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number - // mixing ratios [# / kg air] - view_2d int_aero_mmr[num_aero_modes()] - [num_aero_species()]; // interstitial aerosol mass mixing - // ratios [kg aerosol / kg air] - view_2d cld_aero_mmr[num_aero_modes()] - [num_aero_species()]; // cloudborne aerosol mass mixing - // ratios [kg aerosol / kg air] - view_2d - gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] + view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number mixing ratios [# / kg air] + view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number mixing ratios [# / kg air] + view_2d int_aero_mmr[num_aero_modes()][num_aero_species()]; // interstitial aerosol mass mixing ratios [kg aerosol / kg air] + view_2d cld_aero_mmr[num_aero_modes()][num_aero_species()]; // cloudborne aerosol mass mixing ratios [kg aerosol / kg air] + view_2d gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] }; // storage for variables used within MAM atmosphere processes, initialized with @@ -326,20 +345,20 @@ struct Buffer { static constexpr int num_2d_scratch = 10; // number of local fields stored at column midpoints - static constexpr int num_2d_mid = - 8 + // number of dry atm fields - 2 * (num_aero_modes() + num_aero_tracers()) + num_aero_gases() + - num_2d_scratch; + static constexpr int num_2d_mid = 8 + // number of dry atm fields + 2 * (num_aero_modes() + num_aero_tracers()) + + num_aero_gases() + + num_2d_scratch; // (dry) atmospheric state - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - uview_2d qv_dry; // dry water vapor mixing ratio (dry air) - uview_2d qc_dry; // dry cloud water mass mixing ratio - uview_2d nc_dry; // dry cloud water number mixing ratio - uview_2d qi_dry; // cloud ice mass mixing ratio - uview_2d ni_dry; // dry cloud ice number mixing ratio - uview_2d w_updraft; // vertical wind velocity + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // dry water vapor mixing ratio (dry air) + uview_2d qc_dry; // dry cloud water mass mixing ratio + uview_2d nc_dry; // dry cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d ni_dry; // dry cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity // aerosol dry interstitial/cloudborne number/mass mixing ratios // (because the number of species per mode varies, not all of these will @@ -362,86 +381,125 @@ struct Buffer { // number of local fields stored at column interfaces static constexpr int num_2d_iface = 1; - uview_2d z_iface; // height at interfaces + uview_2d z_iface; // height at interfaces // storage - Real *wsm_data; + Real* wsm_data; }; // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels inline size_t buffer_size(const int ncol, const int nlev) { - return sizeof(Real) * (Buffer::num_2d_mid * ncol * nlev + - Buffer::num_2d_iface * ncol * (nlev + 1)); + return sizeof(Real) * + (Buffer::num_2d_mid * ncol * nlev + + Buffer::num_2d_iface * ncol * (nlev+1)); } // ON HOST, initializeѕ the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. inline size_t init_buffer(const ATMBufferManager &buffer_manager, - const int ncol, const int nlev, Buffer &buffer) { - Real *mem = reinterpret_cast(buffer_manager.get_memory()); + const int ncol, const int nlev, + Buffer &buffer) { + Real* mem = reinterpret_cast(buffer_manager.get_memory()); // set view pointers for midpoint fields - uview_2d *view_2d_mid_ptrs[Buffer::num_2d_mid] = { - &buffer.z_mid, &buffer.dz, &buffer.qv_dry, &buffer.qc_dry, &buffer.nc_dry, - &buffer.qi_dry, &buffer.ni_dry, &buffer.w_updraft, - - // aerosol modes - &buffer.dry_int_aero_nmr[0], &buffer.dry_int_aero_nmr[1], - &buffer.dry_int_aero_nmr[2], &buffer.dry_int_aero_nmr[3], - &buffer.dry_cld_aero_nmr[0], &buffer.dry_cld_aero_nmr[1], - &buffer.dry_cld_aero_nmr[2], &buffer.dry_cld_aero_nmr[3], - - // the following requires knowledge of mam4's mode-species layout - // (see mode_aero_species() in mam4xx/aero_modes.hpp) - - // accumulation mode - &buffer.dry_int_aero_mmr[0][0], &buffer.dry_int_aero_mmr[0][1], - &buffer.dry_int_aero_mmr[0][2], &buffer.dry_int_aero_mmr[0][3], - &buffer.dry_int_aero_mmr[0][4], &buffer.dry_int_aero_mmr[0][5], - &buffer.dry_int_aero_mmr[0][6], &buffer.dry_cld_aero_mmr[0][0], - &buffer.dry_cld_aero_mmr[0][1], &buffer.dry_cld_aero_mmr[0][2], - &buffer.dry_cld_aero_mmr[0][3], &buffer.dry_cld_aero_mmr[0][4], - &buffer.dry_cld_aero_mmr[0][5], &buffer.dry_cld_aero_mmr[0][6], - - // aitken mode - &buffer.dry_int_aero_mmr[1][0], &buffer.dry_int_aero_mmr[1][1], - &buffer.dry_int_aero_mmr[1][2], &buffer.dry_int_aero_mmr[1][3], - &buffer.dry_cld_aero_mmr[1][0], &buffer.dry_cld_aero_mmr[1][1], - &buffer.dry_cld_aero_mmr[1][2], &buffer.dry_cld_aero_mmr[1][3], - - // coarse mode - &buffer.dry_int_aero_mmr[2][0], &buffer.dry_int_aero_mmr[2][1], - &buffer.dry_int_aero_mmr[2][2], &buffer.dry_int_aero_mmr[2][3], - &buffer.dry_int_aero_mmr[2][4], &buffer.dry_int_aero_mmr[2][5], - &buffer.dry_int_aero_mmr[2][6], &buffer.dry_cld_aero_mmr[2][0], - &buffer.dry_cld_aero_mmr[2][1], &buffer.dry_cld_aero_mmr[2][2], - &buffer.dry_cld_aero_mmr[2][3], &buffer.dry_cld_aero_mmr[2][4], - &buffer.dry_cld_aero_mmr[2][5], &buffer.dry_cld_aero_mmr[2][6], - - // primary carbon mode - &buffer.dry_int_aero_mmr[3][0], &buffer.dry_int_aero_mmr[3][1], - &buffer.dry_int_aero_mmr[3][2], &buffer.dry_cld_aero_mmr[3][0], - &buffer.dry_cld_aero_mmr[3][1], &buffer.dry_cld_aero_mmr[3][2], - - // aerosol gases - &buffer.dry_gas_mmr[0], &buffer.dry_gas_mmr[1], &buffer.dry_gas_mmr[2], - &buffer.dry_gas_mmr[3], &buffer.dry_gas_mmr[4], &buffer.dry_gas_mmr[5]}; - for(int i = 0; i < Buffer::num_2d_scratch; ++i) { - view_2d_mid_ptrs[Buffer::num_2d_mid + i - Buffer::num_2d_scratch] = - &buffer.scratch[i]; + uview_2d* view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer.z_mid, + &buffer.dz, + &buffer.qv_dry, + &buffer.qc_dry, + &buffer.nc_dry, + &buffer.qi_dry, + &buffer.ni_dry, + &buffer.w_updraft, + + // aerosol modes + &buffer.dry_int_aero_nmr[0], + &buffer.dry_int_aero_nmr[1], + &buffer.dry_int_aero_nmr[2], + &buffer.dry_int_aero_nmr[3], + &buffer.dry_cld_aero_nmr[0], + &buffer.dry_cld_aero_nmr[1], + &buffer.dry_cld_aero_nmr[2], + &buffer.dry_cld_aero_nmr[3], + + // the following requires knowledge of mam4's mode-species layout + // (see mode_aero_species() in mam4xx/aero_modes.hpp) + + // accumulation mode + &buffer.dry_int_aero_mmr[0][0], + &buffer.dry_int_aero_mmr[0][1], + &buffer.dry_int_aero_mmr[0][2], + &buffer.dry_int_aero_mmr[0][3], + &buffer.dry_int_aero_mmr[0][4], + &buffer.dry_int_aero_mmr[0][5], + &buffer.dry_int_aero_mmr[0][6], + &buffer.dry_cld_aero_mmr[0][0], + &buffer.dry_cld_aero_mmr[0][1], + &buffer.dry_cld_aero_mmr[0][2], + &buffer.dry_cld_aero_mmr[0][3], + &buffer.dry_cld_aero_mmr[0][4], + &buffer.dry_cld_aero_mmr[0][5], + &buffer.dry_cld_aero_mmr[0][6], + + // aitken mode + &buffer.dry_int_aero_mmr[1][0], + &buffer.dry_int_aero_mmr[1][1], + &buffer.dry_int_aero_mmr[1][2], + &buffer.dry_int_aero_mmr[1][3], + &buffer.dry_cld_aero_mmr[1][0], + &buffer.dry_cld_aero_mmr[1][1], + &buffer.dry_cld_aero_mmr[1][2], + &buffer.dry_cld_aero_mmr[1][3], + + // coarse mode + &buffer.dry_int_aero_mmr[2][0], + &buffer.dry_int_aero_mmr[2][1], + &buffer.dry_int_aero_mmr[2][2], + &buffer.dry_int_aero_mmr[2][3], + &buffer.dry_int_aero_mmr[2][4], + &buffer.dry_int_aero_mmr[2][5], + &buffer.dry_int_aero_mmr[2][6], + &buffer.dry_cld_aero_mmr[2][0], + &buffer.dry_cld_aero_mmr[2][1], + &buffer.dry_cld_aero_mmr[2][2], + &buffer.dry_cld_aero_mmr[2][3], + &buffer.dry_cld_aero_mmr[2][4], + &buffer.dry_cld_aero_mmr[2][5], + &buffer.dry_cld_aero_mmr[2][6], + + // primary carbon mode + &buffer.dry_int_aero_mmr[3][0], + &buffer.dry_int_aero_mmr[3][1], + &buffer.dry_int_aero_mmr[3][2], + &buffer.dry_cld_aero_mmr[3][0], + &buffer.dry_cld_aero_mmr[3][1], + &buffer.dry_cld_aero_mmr[3][2], + + // aerosol gases + &buffer.dry_gas_mmr[0], + &buffer.dry_gas_mmr[1], + &buffer.dry_gas_mmr[2], + &buffer.dry_gas_mmr[3], + &buffer.dry_gas_mmr[4], + &buffer.dry_gas_mmr[5] + }; + for (int i = 0; i < Buffer::num_2d_scratch; ++i) { + view_2d_mid_ptrs[Buffer::num_2d_mid+i-Buffer::num_2d_scratch] = &buffer.scratch[i]; } - for(int i = 0; i < Buffer::num_2d_mid; ++i) { + for (int i = 0; i < Buffer::num_2d_mid; ++i) { *view_2d_mid_ptrs[i] = view_2d(mem, ncol, nlev); mem += view_2d_mid_ptrs[i]->size(); } // set view pointers for interface fields - uview_2d *view_2d_iface_ptrs[Buffer::num_2d_iface] = {&buffer.z_iface}; - for(int i = 0; i < Buffer::num_2d_iface; ++i) { - *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev + 1); + uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { + &buffer.z_iface + }; + for (int i = 0; i < Buffer::num_2d_iface; ++i) { + *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev+1); mem += view_2d_iface_ptrs[i]->size(); } @@ -451,48 +509,47 @@ inline size_t init_buffer(const ATMBufferManager &buffer_manager, /* // Compute workspace manager size to check used memory vs. requested memory // (if needed) - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); const int n_wind_slots = ekat::npack(2)*Spack::n; const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, - 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); mem += wsm_size; + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); + mem += wsm_size; */ // return the number of bytes allocated - return (mem - buffer_manager.get_memory()) * sizeof(Real); + return (mem - buffer_manager.get_memory())*sizeof(Real); } // Given a dry atmosphere state, creates a haero::Atmosphere object for the // column with the given index. This object can be provided to mam4xx for the // column. KOKKOS_INLINE_FUNCTION -haero::Atmosphere atmosphere_for_column(const DryAtmosphere &dry_atm, +haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, const int column_index) { EKAT_KERNEL_ASSERT_MSG(dry_atm.T_mid.data() != nullptr, - "T_mid not defined for dry atmosphere state!"); + "T_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_mid.data() != nullptr, - "p_mid not defined for dry atmosphere state!"); + "p_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qv.data() != nullptr, - "qv not defined for dry atmosphere state!"); + "qv not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qc.data() != nullptr, - "qc not defined for dry atmosphere state!"); + "qc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.nc.data() != nullptr, - "nc not defined for dry atmosphere state!"); + "nc not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.qi.data() != nullptr, - "qi not defined for dry atmosphere state!"); + "qi not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.ni.data() != nullptr, - "ni not defined for dry atmosphere state!"); + "ni not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.z_mid.data() != nullptr, - "z_mid not defined for dry atmosphere state!"); + "z_mid not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del.data() != nullptr, - "p_del not defined for dry atmosphere state!"); + "p_del not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, - "p_int not defined for dry atmosphere state!"); + "p_int not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.cldfrac.data() != nullptr, - "cldfrac not defined for dry atmosphere state!"); + "cldfrac not defined for dry atmosphere state!"); EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, - "w_updraft not defined for dry atmosphere state!"); + "w_updraft not defined for dry atmosphere state!"); return haero::Atmosphere(mam4::nlev, ekat::subview(dry_atm.T_mid, column_index), ekat::subview(dry_atm.p_mid, column_index), @@ -509,29 +566,38 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere &dry_atm, dry_atm.pblh(column_index)); } +KOKKOS_INLINE_FUNCTION +int get_haero_prognostics_index(const int mode, const int species) +{ + const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); + const int ind = + (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; + return ind; +}; + // Given an AerosolState with views for dry aerosol quantities, creates a // mam4::Prognostics object for the column with the given index with // ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to // mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics interstitial_aerosols_for_column(const AerosolState &dry_aero, +mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, const int column_index) { constexpr int nlev = mam4::nlev; mam4::Prognostics progs(nlev); - for(int m = 0; m < num_aero_modes(); ++m) { + for (int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), - "int_aero_nmr not defined for dry aerosol state!"); + "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); - for(int a = 0; a < num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - progs.q_aero_i[m][a] = - ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + int hindex = get_haero_prognostics_index(m,a); + progs.q_aero_i[m][hindex] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } - for(int g = 0; g < num_aero_gases(); ++g) { + for (int g = 0; g < num_aero_gases(); ++g) { EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), - "gas_mmr not defined for dry aerosol state!"); + "gas_mmr not defined for dry aerosol state!"); progs.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); } return progs; @@ -541,17 +607,17 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState &dry_aero, // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. KOKKOS_INLINE_FUNCTION -mam4::Prognostics aerosols_for_column(const AerosolState &dry_aero, +mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, const int column_index) { auto progs = interstitial_aerosols_for_column(dry_aero, column_index); - for(int m = 0; m < num_aero_modes(); ++m) { + for (int m = 0; m < num_aero_modes(); ++m) { EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), - "dry_cld_aero_nmr not defined for aerosol state!"); + "dry_cld_aero_nmr not defined for aerosol state!"); progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); - for(int a = 0; a < num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - progs.q_aero_c[m][a] = - ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + int hindex = get_haero_prognostics_index(m,a); + progs.q_aero_c[m][hindex] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } @@ -577,27 +643,26 @@ void compute_vertical_layer_heights(const Team& team, }*/ KOKKOS_INLINE_FUNCTION -void compute_vertical_layer_heights(const Team &team, - const DryAtmosphere &dry_atm, +void compute_vertical_layer_heights(const Team& team, + const DryAtmosphere& dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); - const auto dz = ekat::subview(dry_atm.dz, column_index); - const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); - const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); - const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); - PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs - dz); // output + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs + dz);//output team.team_barrier(); - PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, // inputs - z_iface); // output - team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs + z_iface); //output + team.team_barrier(); // likely necessary to have z_iface up to date PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } @@ -605,23 +670,19 @@ void compute_vertical_layer_heights(const Team &team, // team to compute the vertical updraft velocity for the column with the given // index. KOKKOS_INLINE_FUNCTION -void compute_updraft_velocities(const Team &team, const WetAtmosphere &wet_atm, - const DryAtmosphere &dry_atm, +void compute_updraft_velocities(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - dry_atm.dz(i, k) = - PF::calculate_dz(dry_atm.p_del(i, k), dry_atm.p_mid(i, k), - dry_atm.T_mid(i, k), wet_atm.qv(i, k)); - const auto rho = - PF::calculate_density(dry_atm.p_del(i, k), dry_atm.dz(i, k)); - dry_atm.w_updraft(i, k) = - PF::calculate_vertical_velocity(wet_atm.omega(i, k), rho); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); + const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); + dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); }); } @@ -629,108 +690,93 @@ void compute_updraft_velocities(const Team &team, const WetAtmosphere &wet_atm, // from the team to compute mixing ratios for a dry atmosphere state in th // column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, - const DryAtmosphere &dry_atm, +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - const auto qv_ik = wet_atm.qv(i, k); - dry_atm.qv(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i, k), qv_ik); - dry_atm.qc(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i, k), qv_ik); - dry_atm.nc(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i, k), qv_ik); - dry_atm.qi(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i, k), qv_ik); - dry_atm.ni(i, k) = - PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i, k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } -// Given a thread team and wet atmospheric and aerosol states, dispatches -// threads from the team to compute mixing ratios for the given dry interstitial -// aerosol state for the column with the given index. +// Given a thread team and wet atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given dry interstitial aerosol +// state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_dry_mixing_ratios(const Team &team, const WetAtmosphere &wet_atm, - const AerosolState &wet_aero, - const AerosolState &dry_aero, +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const AerosolState& wet_aero, + const AerosolState& dry_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - const auto qv_ik = wet_atm.qv(i, k); - for(int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.int_aero_nmr[m](i, k), qv_ik); - if(dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.cld_aero_nmr[m](i, k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + if (dry_aero.cld_aero_nmr[m].data()) { + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } - for(int a = 0; a < num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.int_aero_mmr[m][a](i, k), qv_ik); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } - if(dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_drymmr_from_wetmmr( - wet_aero.cld_aero_mmr[m][a](i, k), qv_ik); + if (dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } - for(int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i, k) = - PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i, k), qv_ik); + for (int g = 0; g < num_aero_gases(); ++g) { + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } -// Given a thread team and dry atmospheric and aerosol states, dispatches -// threads from the team to compute mixing ratios for the given wet interstitial -// aerosol state for the column with the given index. +// Given a thread team and dry atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given wet interstitial aerosol +// state for the column with the given index. KOKKOS_INLINE_FUNCTION -void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, - const AerosolState &dry_aero, - const AerosolState &wet_aero, +void compute_wet_mixing_ratios(const Team& team, + const DryAtmosphere& dry_atm, + const AerosolState& dry_aero, + const AerosolState& wet_aero, const int column_index) { - EKAT_KERNEL_ASSERT_MSG( - column_index == team.league_rank(), - "Given column index does not correspond to given team!"); + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); constexpr int nlev = mam4::nlev; - int i = column_index; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](const int k) { - const auto qv_ik = dry_atm.qv(i, k); - for(int m = 0; m < num_aero_modes(); ++m) { - wet_aero.int_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.int_aero_nmr[m](i, k), qv_ik); - if(wet_aero.cld_aero_nmr[m].data()) { - wet_aero.cld_aero_nmr[m](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.cld_aero_nmr[m](i, k), qv_ik); + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = dry_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + wet_aero.int_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_nmr[m](i,k), qv_ik); + if (wet_aero.cld_aero_nmr[m].data()) { + wet_aero.cld_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_nmr[m](i,k), qv_ik); } - for(int a = 0; a < num_aero_species(); ++a) { - if(wet_aero.int_aero_mmr[m][a].data()) { - wet_aero.int_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.int_aero_mmr[m][a](i, k), qv_ik); + for (int a = 0; a < num_aero_species(); ++a) { + if (wet_aero.int_aero_mmr[m][a].data()) { + wet_aero.int_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_mmr[m][a](i,k), qv_ik); } - if(wet_aero.cld_aero_mmr[m][a].data()) { - wet_aero.cld_aero_mmr[m][a](i, k) = PF::calculate_wetmmr_from_drymmr( - dry_aero.cld_aero_mmr[m][a](i, k), qv_ik); + if (wet_aero.cld_aero_mmr[m][a].data()) { + wet_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } - for(int g = 0; g < num_aero_gases(); ++g) { - wet_aero.gas_mmr[g](i, k) = - PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i, k), qv_ik); + for (int g = 0; g < num_aero_gases(); ++g) { + wet_aero.gas_mmr[g](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i,k), qv_ik); } }); } @@ -740,55 +786,50 @@ void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, // these constants where needed within two such functions so we don't define // them inconsistently. Yes, it's the 21st century and we're still struggling // with these basic things. -#define DECLARE_PROG_TRANSFER_CONSTANTS \ - /* mapping of constituent indices to aerosol modes */ \ - const auto Accum = mam4::ModeIndex::Accumulation; \ - const auto Aitken = mam4::ModeIndex::Aitken; \ - const auto Coarse = mam4::ModeIndex::Coarse; \ - const auto PC = mam4::ModeIndex::PrimaryCarbon; \ - const auto NoMode = mam4::ModeIndex::None; \ - static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ - NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) \ - */ \ - Accum, Accum, Accum, Accum, Accum, Accum, Accum, \ - Accum, /* 7 aero species + NMR */ \ - Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ - Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, \ - Coarse, /* 7 aero species + NMR */ \ - PC, PC, PC, PC, /* 3 aero species + NMR */ \ - }; \ - /* mapping of constituent indices to aerosol species */ \ - const auto SOA = mam4::AeroId::SOA; \ - const auto SO4 = mam4::AeroId::SO4; \ - const auto POM = mam4::AeroId::POM; \ - const auto BC = mam4::AeroId::BC; \ - const auto NaCl = mam4::AeroId::NaCl; \ - const auto DST = mam4::AeroId::DST; \ - const auto MOM = mam4::AeroId::MOM; \ - const auto NoAero = mam4::AeroId::None; \ - static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ - NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) \ - */ \ - SO4, POM, SOA, BC, DST, NaCl, MOM, \ - NoAero, /* accumulation mode */ \ - SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ - DST, NaCl, SO4, BC, POM, SOA, MOM, \ - NoAero, /* coarse mode */ \ - POM, BC, MOM, NoAero, /* primary carbon mode */ \ - }; \ - /* mapping of constituent indices to gases */ \ - const auto O3 = mam4::GasId::O3; \ - const auto H2O2 = mam4::GasId::H2O2; \ - const auto H2SO4 = mam4::GasId::H2SO4; \ - const auto SO2 = mam4::GasId::SO2; \ - const auto DMS = mam4::GasId::DMS; \ - const auto SOAG = mam4::GasId::SOAG; \ - const auto NoGas = mam4::GasId::None; \ - static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ - O3, H2O2, H2SO4, SO2, DMS, SOAG, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ - NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ +#define DECLARE_PROG_TRANSFER_CONSTANTS \ + /* mapping of constituent indices to aerosol modes */ \ + const auto Accum = mam4::ModeIndex::Accumulation; \ + const auto Aitken = mam4::ModeIndex::Aitken; \ + const auto Coarse = mam4::ModeIndex::Coarse; \ + const auto PC = mam4::ModeIndex::PrimaryCarbon; \ + const auto NoMode = mam4::ModeIndex::None; \ + static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ + Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ + Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ + PC, PC, PC, PC, /* 3 aero species + NMR */ \ + }; \ + /* mapping of constituent indices to aerosol species */ \ + const auto SOA = mam4::AeroId::SOA; \ + const auto SO4 = mam4::AeroId::SO4; \ + const auto POM = mam4::AeroId::POM; \ + const auto BC = mam4::AeroId::BC; \ + const auto NaCl = mam4::AeroId::NaCl; \ + const auto DST = mam4::AeroId::DST; \ + const auto MOM = mam4::AeroId::MOM; \ + const auto NoAero = mam4::AeroId::None; \ + static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ + NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ + SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ + SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ + DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ + POM, BC, MOM, NoAero, /* primary carbon mode */ \ + }; \ + /* mapping of constituent indices to gases */ \ + const auto O3 = mam4::GasId::O3; \ + const auto H2O2 = mam4::GasId::H2O2; \ + const auto H2SO4 = mam4::GasId::H2SO4; \ + const auto SO2 = mam4::GasId::SO2; \ + const auto DMS = mam4::GasId::DMS; \ + const auto SOAG = mam4::GasId::SOAG; \ + const auto NoGas = mam4::GasId::None; \ + static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ + O3, H2O2, H2SO4, SO2, DMS, SOAG, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, \ }; // Given a Prognostics object, transfers data for interstitial aerosols to the @@ -801,29 +842,30 @@ void compute_wet_mixing_ratios(const Team &team, const DryAtmosphere &dry_atm, // NOTE: indices KOKKOS_INLINE_FUNCTION void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, - const int k, Real q[gas_pcnst()], + const int k, + Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); - q[i] = progs.q_gas[g](k); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + q[i] = progs.q_gas[g](k); qqcw[i] = progs.q_gas[g](k); } else { int m = static_cast(mode_index); - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); - q[i] = progs.q_aero_i[m][a](k); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + q[i] = progs.q_aero_i[m][a](k); qqcw[i] = progs.q_aero_c[m][a](k); - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); - q[i] = progs.n_mode_i[m](k); + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + q[i] = progs.n_mode_i[m](k); qqcw[i] = progs.n_mode_c[m](k); } } @@ -839,29 +881,34 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], Real vmrcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { <<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ======= int m = static_cast(mode_index); +<<<<<<< HEAD if(aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; - vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); - vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); - } else { // constituent is a modal number mixing ratio - vmr[i] = q[i]; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { // constituent is a modal number mixing ratio + vmr[i] = q[i]; vmrcw[i] = qqcw[i]; } } @@ -873,32 +920,38 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], KOKKOS_INLINE_FUNCTION void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], const Real vmrcw[gas_pcnst()], - Real q[gas_pcnst()], Real qqcw[gas_pcnst()]) { + Real q[gas_pcnst()], + Real qqcw[gas_pcnst()]) { DECLARE_PROG_TRANSFER_CONSTANTS - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); const Real mw = mam4::gas_species(g).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { <<<<<<< HEAD if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ======= int m = static_cast(mode_index); +<<<<<<< HEAD if(aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; - q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); - qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); - } else { // constituent is a modal number mixing ratio - q[i] = vmr[i]; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { // constituent is a modal number mixing ratio + q[i] = vmr[i]; qqcw[i] = vmrcw[i]; } } @@ -911,31 +964,35 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], KOKKOS_INLINE_FUNCTION void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], const Real qqcw[gas_pcnst()], - mam4::Prognostics &progs, - const int k) { + mam4::Prognostics &progs, const int k) { DECLARE_PROG_TRANSFER_CONSTANTS // copy number/mass mixing ratios from progs to q and qqcw at level k, // converting them to VMR - for(int i = 0; i < gas_pcnst(); ++i) { + for (int i = 0; i < gas_pcnst(); ++i) { auto mode_index = mode_for_cnst[i]; - auto aero_id = aero_for_cnst[i]; - auto gas_id = gas_for_cnst[i]; - if(gas_id != NoGas) { // constituent is a gas - int g = static_cast(gas_id); + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); progs.q_gas[g](k) = q[i]; } else { int m = static_cast(mode_index); - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; +<<<<<<< HEAD <<<<<<< HEAD } else { // constituent is a modal number mixing ratio ======= } else { // constituent is a modal number mixing ratio int m = static_cast(mode_index); >>>>>>> Modified species order to match E3SM with the folling change in mam4xx +======= + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); +>>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } @@ -945,6 +1002,6 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], #undef DECLARE_PROG_TRANSFER_CONSTANTS -} // namespace scream::mam_coupling +} // namespace scream::mam_coupling #endif From a1d289bd7bfccfd994d07f75b1e8f2970fe4c562 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 25 Mar 2024 13:40:12 -0700 Subject: [PATCH 273/476] Passes dt as an arg to make hetro freeze work --- .../mam/eamxx_mam_aci_process_interface.cpp | 225 +++++++++--------- .../eamxx/src/physics/mam/mam_coupling.hpp | 1 + 2 files changed, 111 insertions(+), 115 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2e32e0670771..285beec7d423 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -591,8 +591,8 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], - const int nlev) { + MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], const int nlev, + const double dt) { using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); @@ -608,107 +608,106 @@ void call_hetfrz_compute_tendencies( MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - // Kokkos::parallel_for( - // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - // const int icol = team.league_rank(); - for(int icol = 0; icol < 1; icol++) { - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies - // class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, - ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, dummy, - dummy, dummy, dummy, dummy, pblh); - // set surface state data - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - // BALLI - mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = - ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = - ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = - ekat::subview(activation_fraction_coarse_idx, icol); - - // These are the output tendencies from heterogeneous freezing that need - // to be added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = - ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = - ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = - ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - // diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - /*diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0;*/ - // hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, - // progs, - // diags, tends); - // }); - } + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // for(int icol = 0; icol < 1; icol++) { + // Set up an atmosphere, surface, diagnostics, pronostics and + // tendencies class. + Real pblh = 0; + haero::Atmosphere atmos( + nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, + ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, + dummy, dummy, dummy, dummy, dummy, pblh); + // set surface state data + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + // BALLI + mam4::Diagnostics diags(nlev); + diags.stratiform_cloud_fraction = + ekat::subview(stratiform_cloud_fraction, icol); + diags.activation_fraction[accum_idx] = + ekat::subview(activation_fraction_accum_idx, icol); + diags.activation_fraction[coarse_idx] = + ekat::subview(activation_fraction_coarse_idx, icol); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0; //, dt = 0; + hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, + progs, diags, tends); + }); + //} } } // namespace @@ -1176,16 +1175,13 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for for tke_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_ /*output*/, - wsubice_ /*output*/, wsig_ /*output*/, tke_, + compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, wsubmin, top_lev_, nlev_); Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_ /*output*/, w0_, - wsig_, nlev_); + compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_ /*output*/, dgnum_, - top_lev_); + compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. // FIXME: Find out in-outs of the following call! @@ -1193,9 +1189,8 @@ void MAMAci::run_impl(const double dt) { nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atmosphere_, aitken_dry_dia_, nlev_); - store_liquid_cloud_fraction(team_policy, cloud_frac_new_ /*output*/, - cloud_frac_old_ /*output*/, wet_atmosphere_, - liqcldf_, top_lev_); + store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, + wet_atmosphere_, liqcldf_, top_lev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1214,8 +1209,8 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_ /*output*/, - dry_atmosphere_.p_del, nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, + nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. call_function_dropmixnuc( @@ -1239,7 +1234,7 @@ void MAMAci::run_impl(const double dt) { stratiform_cloud_fraction_, activation_fraction_accum_idx_, activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, - naai_hom_, naai_, diagnostic_scratch_, nlev_); + naai_hom_, naai_, diagnostic_scratch_, nlev_, dt); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a69bd9f20423..21408fa7b5b5 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -566,6 +566,7 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, dry_atm.pblh(column_index)); } +//FIXME: move it to mam4xx and add comments and change variable names KOKKOS_INLINE_FUNCTION int get_haero_prognostics_index(const int mode, const int species) { From f958170ea69aeeedc385a5e0adafc6c033e9ce5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 25 Mar 2024 14:38:29 -0700 Subject: [PATCH 274/476] Codes works after changes in mam4xx hetfrz.hpp were reverted to original --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 285beec7d423..fa6700293de0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -704,8 +704,8 @@ void call_hetfrz_compute_tendencies( const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t = 0; //, dt = 0; - hetfrz.compute_tendencies(aero_config, /*team,*/ t, dt, atmos, surf, - progs, diags, tends); + hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, + diags, tends); }); //} } From 2ab183f50cd44f0f655c4662cdbcca9eb81ef062 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 25 Mar 2024 14:53:58 -0700 Subject: [PATCH 275/476] Moves function get_haero_prognostics_index to mam4xx --- .../eamxx/src/physics/mam/mam_coupling.hpp | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 21408fa7b5b5..a80404757bdf 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -566,16 +566,6 @@ haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, dry_atm.pblh(column_index)); } -//FIXME: move it to mam4xx and add comments and change variable names -KOKKOS_INLINE_FUNCTION -int get_haero_prognostics_index(const int mode, const int species) -{ - const mam4::AeroId aero_id = mam4::mode_aero_species(mode, species); - const int ind = - (mam4::AeroId::None != aero_id) ? static_cast(aero_id) : -1; - return ind; -}; - // Given an AerosolState with views for dry aerosol quantities, creates a // mam4::Prognostics object for the column with the given index with // ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to @@ -591,8 +581,8 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - int hindex = get_haero_prognostics_index(m,a); - progs.q_aero_i[m][hindex] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + int h_ind = mam4::utils::get_haero_prognostics_index(m,a);//Index of HAERO's prognostics + progs.q_aero_i[m][h_ind] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } @@ -617,8 +607,8 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.cld_aero_mmr[m][a].data()) { - int hindex = get_haero_prognostics_index(m,a); - progs.q_aero_c[m][hindex] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + int h_ind = mam4::utils::get_haero_prognostics_index(m,a); //Index of HAERO's prognostics array + progs.q_aero_c[m][h_ind] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } From cbe943daf5f13523073e4748f83a4f8042f0a78c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 11:36:48 -0700 Subject: [PATCH 276/476] Some minor FIXME comments and cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 3 ++- .../eamxx/src/physics/mam/mam_coupling.hpp | 16 +--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index fa6700293de0..ba77611fcac7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1066,10 +1066,11 @@ void MAMAci::initialize_impl(const RunType run_type) { get_field_out("activation_fraction_coarse").get_view(); // allocate work - + // FIXME: store 25 into a const var or get from MAM4xx for(int icnst = 0; icnst < 25; ++icnst) { qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); } + // FIXME :store 40 in a const int or get from MAM4xx state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, 40); // interstitial and cloudborne aerosol tracers of interest: mass (q) and diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a80404757bdf..6463ff73e8b4 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -618,21 +618,6 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. -/*KOKKOS_INLINE_FUNCTION -void compute_vertical_layer_heights(const Team& team, - const DryAtmosphere& dry_atm, - const int column_index) { - EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), - "Given column index does not correspond to given team!"); - - const auto dz = ekat::subview(dry_atm.dz, column_index); - auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - auto z_mid = ekat::subview(dry_atm.z_mid, column_index); - PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); - team.team_barrier(); // likely necessary to have z_iface up to date - PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); -}*/ - KOKKOS_INLINE_FUNCTION void compute_vertical_layer_heights(const Team& team, const DryAtmosphere& dry_atm, @@ -657,6 +642,7 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } + // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given // index. From 3ffe00602cb35e42f84a8368f2957f2ba39e8a83 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 16:23:50 -0700 Subject: [PATCH 277/476] Another round of minor cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 4 --- .../mam/eamxx_mam_aci_process_interface.hpp | 34 ++++++++++++------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ba77611fcac7..4552a7d8bc86 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,9 +1,5 @@ #include -#include "ekat/util/ekat_units.hpp" -#include "mam4xx/aero_config.hpp" -#include "mam4xx/ndrop.hpp" - namespace scream { namespace { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 88d91b999cda..a70ac5a6a6af 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -7,8 +7,14 @@ // For declaring ACI class derived from atm process class #include -// For physical constants -#include "physics/share/physics_constants.hpp" +// For EKAT units package +#include "ekat/util/ekat_units.hpp" + +// For aerosol configuration +#include "mam4xx/aero_config.hpp" + +// For calling ndrop functions +#include "mam4xx/ndrop.hpp" namespace scream { @@ -19,16 +25,11 @@ class MAMAci final : public scream::AtmosphereProcess { mam4::Hetfrz hetfrz_; // views for single- and multi-column data - using view_1d = scream::mam_coupling::view_1d; using view_2d = scream::mam_coupling::view_2d; using view_3d = scream::mam_coupling::view_3d; - using const_view_1d = scream::mam_coupling::const_view_1d; using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; - template - using view_4d = KT::view; - // FIXME:B: Should the following variables be public? They are like that in // micriphysics and optics codes // FIXME the time step for microphysics [s] need to get from the input @@ -43,16 +44,20 @@ class MAMAci final : public scream::AtmosphereProcess { // turbulent kinetic energy [m^2/s^2] view_2d tke_; + // Dry diameter of the aitken mode (for ice nucleation) view_2d aitken_dry_dia_; - view_2d cld_aero_mmr_[mam_coupling::num_aero_modes()] - [mam_coupling::num_aero_species()]; - + // wet mixing ratios (water species) mam_coupling::WetAtmosphere wet_atmosphere_; + // dry mixing ratios (water species) mam_coupling::DryAtmosphere dry_atmosphere_; + // aerosol dry diameter const_view_3d dgnum_; + + // FIXME: Make sure these output are used somewhere and add comments about + // these view_2d nihf_; view_2d niim_; view_2d nidep_; @@ -128,20 +133,22 @@ class MAMAci final : public scream::AtmosphereProcess { std::shared_ptr grid_; // A view array to carry cloud borne aerosol mmrs/nmrs + // FIXME: 25 should be a const int view_2d qqcw_fld_work_[25]; + + // A view to carry interstitial aerosol mmrs/nmrs view_3d state_q_work_; public: // Constructor MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); - // process metadata - // Return type of the process + // Process metadata: Return type of the process AtmosphereProcessType MAMAci::type() const { return AtmosphereProcessType::Physics; } - // return name of the process + // Return name of the process std::string MAMAci::name() const { return "mam4_aci"; } // grid @@ -152,6 +159,7 @@ class MAMAci final : public scream::AtmosphereProcess { size_t MAMAci::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } + void init_buffers(const ATMBufferManager &buffer_manager) override; // process behavior From 2d4ecf9c352a2bfee843db6f646d876200f187d6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 18:14:12 -0700 Subject: [PATCH 278/476] Replaces dtmicro with dt and some cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 32 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 6 ++-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4552a7d8bc86..93ee66825a85 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -370,7 +370,7 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aerosol_state, const Real dtmicro, + const mam_coupling::AerosolState &dry_aerosol_state, const Real dt, MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], @@ -386,6 +386,8 @@ void call_function_dropmixnuc( MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { + // FIXME: why can't we use MAMAci::drop_scratch_ above + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; MAMAci::const_view_2d zm = dry_atmosphere.z_mid; @@ -515,9 +517,9 @@ void call_function_dropmixnuc( } mam4::ndrop::dropmixnuc( - team, dtmicro, ekat::subview(T_mid, icol), - ekat::subview(p_mid, icol), ekat::subview(p_int, icol), - ekat::subview(pdel, icol), ekat::subview(rpdel, icol), + team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), + ekat::subview(p_int, icol), ekat::subview(pdel, icol), + ekat::subview(rpdel, icol), ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] @@ -716,17 +718,14 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) // set grid for all the inputs and outputs void MAMAci::set_grids( const std::shared_ptr grids_manager) { - m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI set grid"); - grid_ = grids_manager->get_grid("Physics"); // Use physics grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - Kokkos::resize( - rho_, ncol_, - nlev_); // BQ: Does Kokkos::resize allocates memory as well for rho_? + // Kokkos::resize only works on host to allocates memory + Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -738,7 +737,7 @@ void MAMAci::set_grids( Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); - for(int i = 0; i < 15; ++i) { + for(int i = 0; i < drop_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } // Define the different field layouts that will be used for this process @@ -751,12 +750,8 @@ void MAMAci::set_grids( // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; - ekat::units::Units kg = ekat::units::kg; // BQ: why do we need to do this??? - ekat::units::Units Pa = ekat::units::Pa; - ekat::units::Units s = ekat::units::s; - ekat::units::Units m = ekat::units::m; - ekat::units::Units K = ekat::units::K; - auto q_unit = kg / kg; // units of mass mixing ratios of tracers + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); @@ -774,6 +769,7 @@ void MAMAci::set_grids( "tracers"); // cloud ice number mixing ratio [1/kg] add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints + add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints @@ -786,6 +782,8 @@ void MAMAci::set_grids( add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height + + // Output from this process add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, @@ -1211,7 +1209,7 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. call_function_dropmixnuc( - team_policy, dry_atmosphere_, dry_aero_, dtmicro_, raercol_cw_, raercol_, + team_policy, dry_atmosphere_, dry_aero_, dt, raercol_cw_, raercol_, qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, dry_atmosphere_.p_del, rpdel_, state_q_work_, ncldwtr_, kvh_, qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index a70ac5a6a6af..403de8880246 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -32,8 +32,6 @@ class MAMAci final : public scream::AtmosphereProcess { // FIXME:B: Should the following variables be public? They are like that in // micriphysics and optics codes - // FIXME the time step for microphysics [s] need to get from the input - const Real dtmicro_ = .0001; // rho is air density [kg/m3] view_2d rho_; @@ -96,7 +94,9 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d nact_; view_3d mact_; - view_2d dropmixnuc_scratch_mem_[15]; + + static constexpr int drop_scratch_ = 15; + view_2d dropmixnuc_scratch_mem_[drop_scratch_]; view_2d stratiform_cloud_fraction_; view_2d activation_fraction_accum_idx_; From f7b9c112896e6d03444c33175fd3d26f97ecd6dc Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 18:30:23 -0700 Subject: [PATCH 279/476] Replaces ncldwtr with dry_atmospher.nc --- .../mam/eamxx_mam_aci_process_interface.cpp | 22 +++++++------------ .../mam/eamxx_mam_aci_process_interface.hpp | 2 -- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 93ee66825a85..ee4a8ee9661f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -379,8 +379,8 @@ void call_function_dropmixnuc( MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, - MAMAci::const_view_2d ncldwtr, MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, + MAMAci::const_view_2d nc, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, + MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, @@ -523,7 +523,7 @@ void call_function_dropmixnuc( ekat::subview( zm, icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(state_q_work_loc, icol), ekat::subview(ncldwtr, icol), + ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac_new, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, @@ -791,10 +791,6 @@ void MAMAci::set_grids( add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints - // MUST FIXME: Is it same as nc or may be not???? - add_field("ncldwtr", scalar3d_layout_mid, n_unit, - grid_name); // initial droplet number mixing ratio [#/kg] - // MUST FIXME: This should be an internal variable. why we need this as an // input??? add_field("w_updraft", scalar3d_layout_mid, q_unit, @@ -986,9 +982,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // set atmosphere state data w_sec_ = get_field_in("w_sec").get_view(); - ncldwtr_ = - get_field_in("ncldwtr") - .get_view(); // MUST FIXME: is is nc, may be not??? + dgnum_ = get_field_in("dgnum") .get_view(); // MUST FIXME: is it an input, can // we compute it using calcsize??? @@ -1211,10 +1205,10 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dry_atmosphere_, dry_aero_, dt, raercol_cw_, raercol_, qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, state_q_work_, ncldwtr_, kvh_, qcld_, - wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, - ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, - nlev_); + dry_atmosphere_.p_del, rpdel_, state_q_work_, dry_atmosphere_.nc, kvh_, + qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, + ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, + dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 403de8880246..3d30d797a828 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -65,8 +65,6 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_; const_view_2d kvh_; - const_view_2d ncldwtr_; - view_2d cloud_frac_new_; view_2d cloud_frac_old_; view_2d qcld_; From eb06b3bdc42a393416b365c2ec574233b93a0f08 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 27 Mar 2024 21:34:41 -0700 Subject: [PATCH 280/476] changes some variable names and add dummy comments to fix later --- .../mam/eamxx_mam_aci_process_interface.cpp | 147 ++++++++++-------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- 2 files changed, 83 insertions(+), 68 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ee4a8ee9661f..da7bd10e5c70 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -581,8 +581,8 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::AerosolState &dry_aero_, - mam_coupling::WetAtmosphere &wet_atmosphere_, - mam_coupling::DryAtmosphere &dry_atmosphere_, + mam_coupling::WetAtmosphere &wet_atm_, + mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_2d stratiform_cloud_fraction, MAMAci::view_2d activation_fraction_accum_idx, MAMAci::view_2d activation_fraction_coarse_idx, @@ -596,8 +596,8 @@ void call_hetfrz_compute_tendencies( mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atmosphere_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atmosphere_; + mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; @@ -783,65 +783,87 @@ void MAMAci::set_grids( add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height + // BALLI:??? + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // cloud fraction [nondimentional] + + // ------------------------------------------------------------------------ // Output from this process + // ------------------------------------------------------------------------ + // BALLI:??? add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // BALLI:??? add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints + // BALLI:??? add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, grid_name); // Layer thickness(pdel) [Pa] at midpoints // MUST FIXME: This should be an internal variable. why we need this as an // input??? + // BALLI:??? add_field("w_updraft", scalar3d_layout_mid, q_unit, grid_name); // updraft velocity - // MUST FIXME: move it above so that all "required" variables are together - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); // cloud fraction [nondimentional] + // BALLI:??? add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to - // heterogeneous freezing [1/m3] + // BALLI:??? // heterogeneous freezing [1/m3] add_field("icenuc_num_immfrz", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to immersion - // freezing (hetero nuc) [1/m3] + // BALLI:??? // freezing (hetero nuc) [1/m3] add_field("icenuc_num_depnuc", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to // deposition nucleation (hetero nuc)[1/m3] + // BALLI:??? add_field( "icenuc_num_meydep", scalar3d_layout_mid, 1 / m / m / m, grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + + // BALLI:??? add_field( "num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, grid_name); // number of activated aerosol for ice nucleation // (homogeneous freezing only) [#/kg] + // BALLI:??? add_field( "num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] + // BALLI:??? add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] + // BALLI:??? add_field( "ptend_q", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, n_unit, grid_name); // tendencies for interstitial and cloud borne // aerosols [#/kg] + // BALLI:??? add_field( "tendnd", scalar3d_layout_mid, n_unit / s, grid_name); // tendency in droplet number mixing ratio [#/kg/s] + // BALLI:??? add_field( "factnum", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam_coupling::num_aero_modes()}}, nondim, grid_name); // activation fraction for aerosol number [fraction] + // BALLI:??? add_field("ndropcol", scalar3d_layout_mid, n_unit / s, grid_name); // + // BALLI:??? add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); // droplet number mixing ratio tendency due // to mixing [#/kg/s] + // BALLI:??? add_field( "nsource", scalar3d_layout_mid, n_unit / s, grid_name); // droplet number mixing ratio source tendency [#/kg/s] + // BALLI:??? add_field("wtke", scalar3d_layout_mid, n_unit / s, grid_name); // + + // BALLI:??? add_field( "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, n_unit, @@ -851,10 +873,12 @@ void MAMAci::set_grids( // [#/m^2/s]] // / [aero. number conc. in updraft, just below // cloudbase [#/m^3]] + // BALLI:??? add_field( "coltend", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, nondim, grid_name); // column tendency for diagnostic output + // BALLI:??? add_field( "coltend_cw", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, @@ -871,7 +895,6 @@ void MAMAci::set_grids( mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - std::cout << "mode:" << mode << " name:" << int_nmr_field_name << std::endl; // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -880,8 +903,6 @@ void MAMAci::set_grids( mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - // std::cout<<"mode:"<< <<" name:"<< < 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - std::cout << "--> mode:" << mode << " spec:" << a - << " int-spec-name:" << int_mmr_field_name << std::endl; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are @@ -901,8 +920,6 @@ void MAMAci::set_grids( if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); - // std::cout<<"--> mode:"<< mode<<" spec:"<< a<< " cld-spec name:"<< - // cld_mmr_field_name<(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - // std::cout << "--> Gasses:" << g <<" spec:"<( "w_sec", scalar3d_layout_int, m2 / s2, grid_name); // Vertical velocity variance (wp2) at midpoints - + // BALLI:??? add_field("strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Stratiform cloud fraction at midpoints + // BALLI:??? add_field( "liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // Liquid stratiform cloud fraction at midpoints + // BALLI:??? add_field("kvh", scalar3d_layout_int, m2 / s, grid_name); // Eddy diffusivity for heat @@ -939,6 +957,7 @@ void MAMAci::set_grids( const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + // BALLI:??? add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols @@ -946,11 +965,13 @@ void MAMAci::set_grids( auto frz_unit = 1 / (cm * cm * cm * s); // units of number mixing ratios of tracers n_unit.set_string("1(cm^-3 s^-1)"); - + // BALLI:??? add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + // BALLI:??? add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols + // BALLI:??? add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); // dry diameter of aerosols // @@ -962,7 +983,7 @@ void MAMAci::set_grids( * qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). * qv_mid can be obtained from "get_field_in" call */ -} +} // set_grids ends void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( @@ -977,10 +998,6 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { } void MAMAci::initialize_impl(const RunType run_type) { - m_atm_logger->log(ekat::logger::LogLevel::info, "Calling ACI init"); - - // set atmosphere state data - w_sec_ = get_field_in("w_sec").get_view(); dgnum_ = get_field_in("dgnum") @@ -1010,34 +1027,34 @@ void MAMAci::initialize_impl(const RunType run_type) { coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - wet_atmosphere_.qv = get_field_in("qv").get_view(); - wet_atmosphere_.qc = get_field_in("qc").get_view(); - wet_atmosphere_.nc = get_field_in("nc").get_view(); - wet_atmosphere_.qi = get_field_in("qi").get_view(); - wet_atmosphere_.ni = get_field_in("ni").get_view(); - wet_atmosphere_.omega = get_field_in("omega").get_view(); - - dry_atmosphere_.T_mid = get_field_in("T_mid").get_view(); - dry_atmosphere_.p_mid = get_field_in("p_mid").get_view(); - dry_atmosphere_.p_int = get_field_in("p_int").get_view(); - dry_atmosphere_.p_del = - get_field_in("pseudo_density").get_view(); - dry_atmosphere_.qv = buffer_.qv_dry; - dry_atmosphere_.qc = buffer_.qc_dry; - dry_atmosphere_.nc = buffer_.nc_dry; - dry_atmosphere_.qi = buffer_.qi_dry; - dry_atmosphere_.ni = buffer_.ni_dry; - dry_atmosphere_.pblh = get_field_in("pbl_height").get_view(); - - dry_atmosphere_.dz = buffer_.dz; // geometric thickness of layers (m) - dry_atmosphere_.z_iface = buffer_.z_iface; // geopotential height above - // surface at interface levels (m) - dry_atmosphere_.z_mid = + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // pbl_height from SHOC + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + dry_atm_.dz = buffer_.dz; // geometric thickness of layers (m) + dry_atm_.z_iface = buffer_.z_iface; // geopotential height above + // surface at interface levels (m) + dry_atm_.z_mid = buffer_.z_mid; // geopotential height above surface at mid levels (m) - dry_atmosphere_.cldfrac = - get_field_in("cldfrac_tot").get_view(); - dry_atmosphere_.w_updraft = get_field_out("w_updraft").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.w_updraft = get_field_out("w_updraft").get_view(); hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); @@ -1137,8 +1154,8 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_.init(aero_config, hetfrz_config); // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atmosphere_, wet_aero_, - dry_atmosphere_, dry_aero_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } void MAMAci::run_impl(const double dt) { @@ -1157,8 +1174,8 @@ void MAMAci::run_impl(const double dt) { // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion - compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, - wet_atmosphere_, dry_atmosphere_, top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, + dry_atm_, top_lev_, nlev_); compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); @@ -1176,10 +1193,10 @@ void MAMAci::run_impl(const double dt) { // FIXME: Find out in-outs of the following call! compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, - dry_atmosphere_, aitken_dry_dia_, nlev_); + dry_atm_, aitken_dry_dia_, nlev_); store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atmosphere_, liqcldf_, top_lev_); + wet_atm_, liqcldf_, top_lev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1198,17 +1215,15 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atmosphere_.p_del, - nlev_); + compute_recipical_pseudo_density(team_policy, rpdel_, dry_atm_.p_del, nlev_); Kokkos::fence(); // wait for rpdel_ to be computed. call_function_dropmixnuc( - team_policy, dry_atmosphere_, dry_aero_, dt, raercol_cw_, raercol_, - qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atmosphere_.p_int, - dry_atmosphere_.p_del, rpdel_, state_q_work_, dry_atmosphere_.nc, kvh_, - qcld_, wsub_, cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, - ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, - dropmixnuc_scratch_mem_, nlev_); + team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, + qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atm_.p_int, + dry_atm_.p_del, rpdel_, state_q_work_, dry_atm_.nc, kvh_, qcld_, wsub_, + cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( @@ -1219,7 +1234,7 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies( - team_policy, hetfrz_, dry_aero_, wet_atmosphere_, dry_atmosphere_, + team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, stratiform_cloud_fraction_, activation_fraction_accum_idx_, activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 3d30d797a828..931068cda829 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -46,10 +46,10 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d aitken_dry_dia_; // wet mixing ratios (water species) - mam_coupling::WetAtmosphere wet_atmosphere_; + mam_coupling::WetAtmosphere wet_atm_; // dry mixing ratios (water species) - mam_coupling::DryAtmosphere dry_atmosphere_; + mam_coupling::DryAtmosphere dry_atm_; // aerosol dry diameter const_view_3d dgnum_; From 4952a927e7bf8d8ece3858c64ee9a5221ec0f30d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 28 Mar 2024 16:58:37 -0700 Subject: [PATCH 281/476] Models compiles fine after the rebase off of optics branch --- .../mam/eamxx_mam_aci_process_interface.cpp | 24 ++-- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 44 +----- .../tests/uncoupled/mam4/aci/CMakeLists.txt | 24 ---- .../eamxx/tests/uncoupled/mam4/aci/input.yaml | 131 ------------------ .../tests/uncoupled/mam4/aci/output.yaml | 12 -- 6 files changed, 20 insertions(+), 217 deletions(-) delete mode 100644 components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt delete mode 100644 components/eamxx/tests/uncoupled/mam4/aci/input.yaml delete mode 100644 components/eamxx/tests/uncoupled/mam4/aci/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index da7bd10e5c70..7b3def4ff249 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -374,7 +374,7 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::ndrop::nvar_ptend_q], + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, @@ -411,7 +411,7 @@ void call_function_dropmixnuc( MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d loc_ptend_q[mam4::ndrop::nvar_ptend_q]; + MAMAci::view_2d loc_ptend_q[mam4::aero_model::pcnst]; MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; @@ -419,7 +419,7 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) @@ -470,8 +470,8 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); } - mam4::ColumnView ptend_q_view[mam4::ndrop::nvar_ptend_q]; - for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { + mam4::ColumnView ptend_q_view[mam4::aero_model::pcnst]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); } mam4::ColumnView coltend_view[mam4::ndrop::ncnst_tot], @@ -501,14 +501,14 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here - Real qqcw_at_lev_col[mam4::ndrop::nvars] = {}; // use pcnst here + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); - for(int icnst = 15; icnst < mam4::ndrop::nvars; ++icnst) { + for(int icnst = 15; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; // FIXME: ensure that indices are // right! remove "15" if possible!! @@ -836,7 +836,7 @@ void MAMAci::set_grids( // BALLI:??? add_field( "ptend_q", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::nvar_ptend_q}}, + FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::aero_model::pcnst}}, n_unit, grid_name); // tendencies for interstitial and cloud borne // aerosols [#/kg] // BALLI:??? @@ -956,7 +956,7 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid{ - {COL, LEV, NUM_MODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points // BALLI:??? add_field("dgnum", scalar4d_layout_mid, m, grid_name); // dry diameter of aerosols @@ -1126,7 +1126,7 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(coltend_[i], ncol_, nlev_); Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } - for(int i = 0; i < mam4::ndrop::nvar_ptend_q; ++i) { + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { Kokkos::resize(ptend_q_[i], ncol_, nlev_); // MUST FIXME:Do we need this? } for(int i = 0; i < mam4::ndrop::pver; ++i) { @@ -1226,7 +1226,7 @@ void MAMAci::run_impl(const double dt) { nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. - copy_mam4xx_array_to_scream( + copy_mam4xx_array_to_scream( team_policy, ptend_q_output_, ptend_q_, nlev_); copy_mam4xx_array_to_scream( team_policy, coltend_outp_, coltend_, nlev_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 931068cda829..0a35267ae3f3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -70,7 +70,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d qcld_; view_2d tendnd_; // ptend_q_ is just ptend_q_output_ reformatted. - view_2d ptend_q_[mam4::ndrop::nvar_ptend_q]; + view_2d ptend_q_[mam4::aero_model::pcnst]; view_3d ptend_q_output_; view_3d factnum_; const_view_3d qqcw_input_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6463ff73e8b4..4b71eeaf0f2c 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -23,6 +23,9 @@ using const_view_1d = typename KT::template view_1d; using const_view_2d = typename KT::template view_2d; using const_view_3d = typename KT::template view_3d; +using complex_view_3d = typename KT::template view_3d>; +using complex_view_2d = typename KT::template view_2d>; + // Kokkos thread team (league member) using Team = Kokkos::TeamPolicy::member_type; @@ -266,10 +269,6 @@ struct WetAtmosphere { // This type stores multi-column views related to the dry atmospheric state // used by MAM. struct DryAtmosphere { -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index Real z_surf; // height of bottom of atmosphere [m] const_view_2d T_mid; // temperature at grid midpoints [K] const_view_2d p_mid; // total pressure at grid midpoints [Pa] @@ -282,40 +281,11 @@ struct DryAtmosphere { view_2d z_iface; // height at layer interfaces [m] view_2d dz; // layer thickness [m] const_view_2d p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] -<<<<<<< HEAD - const_view_2d p_int; // total pressure at grid interfaces [Pa] -======= const_view_2d p_int; // total pressure at grid interfaces [Pa] ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const_view_2d cldfrac; // cloud fraction [-] view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] -<<<<<<< HEAD -======= - Real z_surf; // height of bottom of atmosphere [m] - const_view_2d T_mid; // temperature at grid midpoints [K] - const_view_2d p_mid; // total pressure at grid midpoints [Pa] - view_2d qv; // dry water vapor mixing ratio [kg vapor / kg dry air] - view_2d qc; // dry cloud liquid water mass mixing ratio [kg cloud water/kg - // dry air] - view_2d nc; // dry cloud liquid water number mixing ratio [# / kg dry air] - view_2d qi; // dry cloud ice water mass mixing ratio [kg cloud ice water / kg - // dry air] - view_2d ni; // dry cloud ice water number mixing ratio [# / kg dry air] - view_2d z_mid; // height at layer midpoints [m] - view_2d z_iface; // height at layer interfaces [m] - view_2d dz; // layer thickness [m] - const_view_2d - p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] - const_view_2d p_int; // total pressure at grid interfaces [Pa] - const_view_2d cldfrac; // cloud fraction [-] - view_2d w_updraft; // updraft velocity [m/s] - const_view_1d pblh; // planetary boundary layer height [m] - const_view_1d phis; // surface geopotential [m2/s2] ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It @@ -581,8 +551,8 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - int h_ind = mam4::utils::get_haero_prognostics_index(m,a);//Index of HAERO's prognostics - progs.q_aero_i[m][h_ind] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + int prg_idx = mam4::utils::get_prognostics_index(m,a);//Index of HAERO's prognostics + progs.q_aero_i[m][prg_idx] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); } } } @@ -607,8 +577,8 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.cld_aero_mmr[m][a].data()) { - int h_ind = mam4::utils::get_haero_prognostics_index(m,a); //Index of HAERO's prognostics array - progs.q_aero_c[m][h_ind] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + int prg_idx = mam4::utils::get_prognostics_index(m,a); //Index of HAERO's prognostics array + progs.q_aero_c[m][prg_idx] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); } } } diff --git a/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt deleted file mode 100644 index 45532850822d..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/aci/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -include (ScreamUtils) - - - -# Create the test -CreateADUnitTest(mam4_aci_standalone - LABELS mam4 aci physics driver - LIBS mam - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) - -# Set AD configurable options -set (ATM_TIME_STEP 1) -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (RUN_T0 2021-10-12-45000) - -## Copy (and configure) yaml files needed by tests -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - -# Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) diff --git a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml b/components/eamxx/tests/uncoupled/mam4/aci/input.yaml deleted file mode 100644 index 298da6d50adb..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/aci/input.yaml +++ /dev/null @@ -1,131 +0,0 @@ -%YAML 1.1 ---- -driver_options: - atmosphere_dag_verbosity_level: 5 - -time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - #number_of_steps: ${NUM_STEPS} - number_of_steps: 1 - -atmosphere_processes: - atm_procs_list: [mam4_aci] - mam4_micro: - compute_tendencies: [] - -grids_manager: - Type: Mesh Free - grids_names: [Physics] - Physics: - type: point_grid - number_of_global_columns: 218 - number_of_vertical_levels: 72 - -initial_conditions: - # The name of the file containing the initial conditions for this test. FIXME: use a variable for the filename here - Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc - T_mid: 273.0 - p_mid: 1.e5 - p_int: 1.e5 - num_a1: 1.e5 - num_c1: 1.e5 - bc_a1: 1.e-5 - qqcw: 1e5 - pbl_height: 0.0 - - aero_gas_mmr_o3: 0.0 - aero_gas_mmr_h2o2: 0.0 - aero_gas_mmr_h2so4: 0.0 - aero_gas_mmr_so2: 0.0 - aero_gas_mmr_dms: 0.0 - aero_gas_mmr_soag: 0.0 - - hetfrz_immersion_nucleation_tend: 0 - hetfrz_contact_nucleation_tend: 0 - hetfrz_depostion_nucleation_tend: 0 - - zm: 1e1 - - w_updraft: 1e0 - ptend: 1e0 - - num_a2: 1e5 - num_a3: 1e5 - num_a4: 1e5 - num_c2: 1e5 - num_c3: 1e5 - num_c4: 1e5 - soa_a1: 1e-5 - so4_a1: 1e-5 - pom_a1: 1e-5 - nacl_a1: 1e-5 - dst_a1: 1e-5 - mom_a1: 1e-5 - soa_a2: 1e-5 - so4_a2: 1e-5 - nacl_a2: 1e-5 - mom_a2: 1e-5 - soa_a3: 1e-5 - so4_a3: 1e-5 - pom_a3: 1e-5 - bc_a3: 1e-5 - nacl_a3: 1e-5 - dst_a3: 1e-5 - mom_a3: 1e-5 - pom_a4: 1e-5 - bc_a4: 1e-5 - mom_a4: 1e-5 - #cloud borne aerosols - qi_coarse_dst: 1e-5 - qi_coarse_nacl: 1e-5 - qi_coarse_so4: 1e-5 - qi_coarse_mom: 1e-5 - qi_coarse_bc: 1e-5 - qi_coarse_pom: 1e-5 - qi_coarse_soa: 1e-5 - ni_accum: 1e-5 - ni_coarse: 1e-5 - ni_aitken: 1e-5 - - state_q: 1e-5 - ncldwtr: 1e-12 - - icenuc_num_hetfrz: 1e3 - icenuc_num_immfrz: 1e3 - icenuc_num_depnuc: 1e3 - icenuc_num_meydep: 1e3 - num_act_aerosol_ice_nucle_hom: 1e3 - num_act_aerosol_ice_nucle: 1e3 - - soa_c1: 1e-5 - so4_c1: 1e-5 - pom_c1: 1e-5 - bc_c1: 1e-5 - nacl_c1: 1e-5 - dst_c1: 1e-5 - mom_c1: 1e-5 - soa_c2: 1e-5 - so4_c2: 1e-5 - nacl_c2: 1e-5 - mom_c2: 1e-5 - soa_c3: 1e-5 - so4_c3: 1e-5 - pom_c3: 1e-5 - bc_c3: 1e-5 - nacl_c3: 1e-5 - dst_c3: 1e-5 - mom_c3: 1e-5 - pom_c4: 1e-5 - bc_c4: 1e-5 - mom_c4: 1e-5 - qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae - w_sec: 1e-2 - strat_cld_frac: 0.1 - liq_strat_cld_frac: 0.1 - kvh: 0.1 - dgnum: 1e-3 -# The parameters for I/O control -Scorpio: - output_yaml_files: ["output.yaml"] -... diff --git a/components/eamxx/tests/uncoupled/mam4/aci/output.yaml b/components/eamxx/tests/uncoupled/mam4/aci/output.yaml deleted file mode 100644 index 0087223d1d6a..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/aci/output.yaml +++ /dev/null @@ -1,12 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: mam4_aci_standalone_output -Averaging Type: Instant -Field Names: - - T_mid - - p_mid -output_control: - Frequency: 1 - frequency_units: nsteps - MPI Ranks in Filename: true -... From 59bf802d7b11a13c77224181e5a8f2e7227964fe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 30 Mar 2024 12:04:42 -0700 Subject: [PATCH 282/476] Single process test is working with a new init file now --- .../mam/eamxx_mam_aci_process_interface.cpp | 5 ++- .../eamxx/src/physics/mam/mam_coupling.hpp | 28 +++++------- components/eamxx/tests/CMakeLists.txt | 2 +- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../single-process/mam/aci/CMakeLists.txt | 45 +++++++++++++++++++ .../tests/single-process/mam/aci/input.yaml | 43 ++++++++++++++++++ .../tests/single-process/mam/aci/output.yaml | 14 ++++++ 7 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 components/eamxx/tests/single-process/mam/aci/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/aci/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/aci/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7b3def4ff249..cddc51e6de88 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -609,7 +609,6 @@ void call_hetfrz_compute_tendencies( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for(int icol = 0; icol < 1; icol++) { // Set up an atmosphere, surface, diagnostics, pronostics and // tendencies class. Real pblh = 0; @@ -705,7 +704,7 @@ void call_hetfrz_compute_tendencies( hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, diags, tends); }); - //} + //} } } // namespace @@ -1097,10 +1096,12 @@ void MAMAci::initialize_impl(const RunType run_type) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + std::cout<") +// mixing ratio field in EAMxx KOKKOS_INLINE_FUNCTION const char* gas_mmr_field_name(const int gas) { - if (!gas_mmr_names(gas)[0]) { - concat_2_strings("aero_gas_mmr_", gas_species_name(gas), gas_mmr_names(gas)); - } - return const_cast(gas_mmr_names(gas)); + return const_cast(gas_species_name(gas)); } // This type stores multi-column views related specifically to the wet @@ -597,12 +594,12 @@ void compute_vertical_layer_heights(const Team& team, const auto dz = ekat::subview(dry_atm.dz, column_index); const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); - const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); // worked fine - const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); - const auto qv = ekat::subview(dry_atm.qv, column_index); - + const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + // NOTE: we are using dry qv. Does calculate_dz require dry or wet? PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); @@ -612,7 +609,6 @@ void compute_vertical_layer_heights(const Team& team, PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); } - // Given a thread team and wet and dry atmospheres, dispatches threads from the // team to compute the vertical updraft velocity for the column with the given // index. diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 7a7a9e821493..287fdf25af1a 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -63,7 +63,7 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") - set(EAMxx_tests_IC_FILE_MAM4xx_72lev "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") + set(EAMxx_tests_IC_FILE_MAM4xx_72lev "screami_unit_tests_mam4xx_ne2np4L72_20240329.nc") # Testing individual atm processes add_subdirectory(single-process) diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index d417a9920f46..c6b4e0748af3 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -20,6 +20,7 @@ if (SCREAM_ENABLE_MAM) # corresponding test here needs to be reworked with valid aerosol # initial conditions. add_subdirectory(mam/optics) + add_subdirectory(mam/aci) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt new file mode 100644 index 000000000000..2b33c95a5174 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -0,0 +1,45 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_aci_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_aci physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_aci physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() \ No newline at end of file diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml new file mode 100644 index 000000000000..1e4c7327d71e --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -0,0 +1,43 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_aci] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + dgnum: 1e-3 + kvh: 1e-3 + liq_strat_cld_frac: 1e-3 + strat_cld_frac: 1e-3 + w_sec: 1e-3 + w_updraft: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml new file mode 100644 index 000000000000..cb54b742c5a6 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -0,0 +1,14 @@ +%YAML 1.1 +--- +filename_prefix: mam4_aci_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - T_mid + +output_control: + Frequency: 2 + frequency_units: nsteps + MPI Ranks in Filename: true +... From 1afbf982e9e52d0e7c487faaff3497d91f968727 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 17:29:20 -0700 Subject: [PATCH 283/476] Removes some variables from the input file and cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 111 ++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- .../tests/single-process/mam/aci/input.yaml | 1 - 3 files changed, 50 insertions(+), 66 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index cddc51e6de88..31aaf9025a9b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -419,8 +419,7 @@ void call_function_dropmixnuc( for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; for(int i = 0; i < mam4::ndrop::pver; ++i) for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) - loc_ptend_q[i] = ptend_q[i]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend_cw[i] = coltend_cw[i]; @@ -501,8 +500,9 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here + Real state_q_at_lev_col[mam4::aero_model::pcnst] = + {}; // use pcnst here + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); @@ -583,9 +583,7 @@ void call_hetfrz_compute_tendencies( mam_coupling::AerosolState &dry_aero_, mam_coupling::WetAtmosphere &wet_atm_, mam_coupling::DryAtmosphere &dry_atm_, - MAMAci::view_2d stratiform_cloud_fraction, - MAMAci::view_2d activation_fraction_accum_idx, - MAMAci::view_2d activation_fraction_coarse_idx, + MAMAci::view_3d factnum_, MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, @@ -594,6 +592,7 @@ void call_hetfrz_compute_tendencies( using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); + mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; @@ -601,21 +600,21 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; - MAMAci::const_view_2d qc = wet_atmosphere.qc; - MAMAci::const_view_2d nc = wet_atmosphere.nc; - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); + //for (int icol=0; icol<1; icol++){ // Set up an atmosphere, surface, diagnostics, pronostics and // tendencies class. - Real pblh = 0; + + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + /*Real pblh = 0; haero::Atmosphere atmos( nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, - dummy, dummy, dummy, dummy, dummy, pblh); + dummy, dummy, dummy, dummy, dummy, pblh);*/ // set surface state data haero::Surface surf{}; mam4::Prognostics progs = @@ -626,12 +625,10 @@ void call_hetfrz_compute_tendencies( // BALLI mam4::Diagnostics diags(nlev); - diags.stratiform_cloud_fraction = - ekat::subview(stratiform_cloud_fraction, icol); - diags.activation_fraction[accum_idx] = - ekat::subview(activation_fraction_accum_idx, icol); - diags.activation_fraction[coarse_idx] = - ekat::subview(activation_fraction_coarse_idx, icol); + + diags.activation_fraction[accum_idx] = ekat::subview(factnum_,icol, accum_idx); + + diags.activation_fraction[coarse_idx] = ekat::subview(factnum_, icol, coarse_idx); // These are the output tendencies from heterogeneous freezing that need // to be added correctly to the cloud-micorphysics scheme. @@ -701,10 +698,10 @@ void call_hetfrz_compute_tendencies( const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t = 0; //, dt = 0; - hetfrz.compute_tendencies(aero_config, team, t, dt, atmos, surf, progs, + hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); - //} + //} } } // namespace @@ -723,8 +720,9 @@ void MAMAci::set_grids( ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + // Allocate memory for the class members // Kokkos::resize only works on host to allocates memory - Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_);// FIXME: rho_ is only used internally in compute_w0_and_rho Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -766,6 +764,7 @@ void MAMAci::set_grids( "tracers"); // cloud liquid number mixing ratio [1/kg] add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud ice number mixing ratio [1/kg] + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints @@ -782,28 +781,13 @@ void MAMAci::set_grids( add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height - // BALLI:??? + // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); // cloud fraction [nondimentional] + grid_name); // ------------------------------------------------------------------------ // Output from this process // ------------------------------------------------------------------------ - // BALLI:??? - add_field("stratiform_cloud_fraction", scalar3d_layout_mid, nondim, - grid_name); // Layer thickness(pdel) [Pa] at midpoints - // BALLI:??? - add_field("activation_fraction_accum", scalar3d_layout_mid, nondim, - grid_name); // Layer thickness(pdel) [Pa] at midpoints - // BALLI:??? - add_field("activation_fraction_coarse", scalar3d_layout_mid, nondim, - grid_name); // Layer thickness(pdel) [Pa] at midpoints - - // MUST FIXME: This should be an internal variable. why we need this as an - // input??? - // BALLI:??? - add_field("w_updraft", scalar3d_layout_mid, q_unit, - grid_name); // updraft velocity // BALLI:??? add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, @@ -845,8 +829,8 @@ void MAMAci::set_grids( // BALLI:??? add_field( "factnum", - FieldLayout{{COL, LEV, CMP}, - {ncol_, nlev_, mam_coupling::num_aero_modes()}}, + FieldLayout{{COL, NMODES, LEV}, + {ncol_, mam_coupling::num_aero_modes(),nlev_}}, nondim, grid_name); // activation fraction for aerosol number [fraction] // BALLI:??? add_field("ndropcol", scalar3d_layout_mid, n_unit / s, @@ -941,9 +925,7 @@ void MAMAci::set_grids( add_field( "w_sec", scalar3d_layout_int, m2 / s2, grid_name); // Vertical velocity variance (wp2) at midpoints - // BALLI:??? - add_field("strat_cld_frac", scalar3d_layout_mid, nondim, - grid_name); // Stratiform cloud fraction at midpoints + // BALLI:??? add_field( "liq_strat_cld_frac", scalar3d_layout_mid, nondim, @@ -961,18 +943,22 @@ void MAMAci::set_grids( grid_name); // dry diameter of aerosols auto cm = m / 100; - auto frz_unit = - 1 / (cm * cm * cm * s); // units of number mixing ratios of tracers + + // units of number mixing ratios of tracers + auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - // BALLI:??? + + // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); // dry diameter of aerosols - // BALLI:??? + frz_unit, grid_name); + + // heterogeous freezing by contact nucleation [cm^-3 s^-1] add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); // dry diameter of aerosols - // BALLI:??? + frz_unit, grid_name); + + // heterogeous freezing by deposition nucleation [cm^-3 s^-1] add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, - frz_unit, grid_name); // dry diameter of aerosols + frz_unit, grid_name); // /* * NOTE on other inputs for the aci process: @@ -1053,7 +1039,7 @@ void MAMAci::initialize_impl(const RunType run_type) { buffer_.z_mid; // geopotential height above surface at mid levels (m) dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); - dry_atm_.w_updraft = get_field_out("w_updraft").get_view(); + dry_atm_.w_updraft = buffer_.w_updraft; hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); @@ -1062,13 +1048,6 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - stratiform_cloud_fraction_ = - get_field_out("stratiform_cloud_fraction").get_view(); - activation_fraction_accum_idx_ = - get_field_out("activation_fraction_accum").get_view(); - activation_fraction_coarse_idx_ = - get_field_out("activation_fraction_coarse").get_view(); - // allocate work // FIXME: store 25 into a const var or get from MAM4xx for(int icnst = 0; icnst < 25; ++icnst) { @@ -1096,12 +1075,14 @@ void MAMAci::initialize_impl(const RunType run_type) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - std::cout<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1236,8 +1220,7 @@ void MAMAci::run_impl(const double dt) { call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, - stratiform_cloud_fraction_, activation_fraction_accum_idx_, - activation_fraction_coarse_idx_, hetfrz_immersion_nucleation_tend_, + factnum_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, naai_hom_, naai_, diagnostic_scratch_, nlev_, dt); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0a35267ae3f3..be7d116fe5e6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -25,8 +25,10 @@ class MAMAci final : public scream::AtmosphereProcess { mam4::Hetfrz hetfrz_; // views for single- and multi-column data + using view_1d = scream::mam_coupling::view_1d; using view_2d = scream::mam_coupling::view_2d; using view_3d = scream::mam_coupling::view_3d; + using const_view_1d = scream::mam_coupling::const_view_1d; using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; @@ -196,7 +198,7 @@ class MAMAci final : public scream::AtmosphereProcess { // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, dry_atm_pre_, i); - + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); } // operator() // local variables for preprocess struct diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 1e4c7327d71e..b63060e04a58 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -33,7 +33,6 @@ initial_conditions: dgnum: 1e-3 kvh: 1e-3 liq_strat_cld_frac: 1e-3 - strat_cld_frac: 1e-3 w_sec: 1e-3 w_updraft: 1e-3 From 94cd1fa7b85687ea1b350c6ec657703f7cd1c621 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 20:19:52 -0700 Subject: [PATCH 284/476] Cleanup of some I/O variables for ice nuc and dropmixnuc processes --- .../mam/eamxx_mam_aci_process_interface.cpp | 142 +++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- 2 files changed, 76 insertions(+), 70 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 31aaf9025a9b..25afa6012e49 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -386,7 +386,7 @@ void call_function_dropmixnuc( MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { - // FIXME: why can't we use MAMAci::drop_scratch_ above + // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -582,8 +582,7 @@ void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::AerosolState &dry_aero_, mam_coupling::WetAtmosphere &wet_atm_, - mam_coupling::DryAtmosphere &dry_atm_, - MAMAci::view_3d factnum_, + mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_3d factnum_, MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, @@ -592,7 +591,6 @@ void call_hetfrz_compute_tendencies( using view_1d = typename KokkosTypes::template view_1d; view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; @@ -600,13 +598,12 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - //for (int icol=0; icol<1; icol++){ - // Set up an atmosphere, surface, diagnostics, pronostics and - // tendencies class. + // for (int icol=0; icol<1; icol++){ + // Set up an atmosphere, surface, diagnostics, pronostics and + // tendencies class. haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); @@ -626,9 +623,11 @@ void call_hetfrz_compute_tendencies( // BALLI mam4::Diagnostics diags(nlev); - diags.activation_fraction[accum_idx] = ekat::subview(factnum_,icol, accum_idx); + diags.activation_fraction[accum_idx] = + ekat::subview(factnum_, icol, accum_idx); - diags.activation_fraction[coarse_idx] = ekat::subview(factnum_, icol, coarse_idx); + diags.activation_fraction[coarse_idx] = + ekat::subview(factnum_, icol, coarse_idx); // These are the output tendencies from heterogeneous freezing that need // to be added correctly to the cloud-micorphysics scheme. @@ -682,11 +681,6 @@ void call_hetfrz_compute_tendencies( diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - //------------------------------------------------------------- // Heterogeneous freezing // frzimm, frzcnt, frzdep are the outputs of @@ -698,8 +692,8 @@ void call_hetfrz_compute_tendencies( const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; const Real t = 0; //, dt = 0; - hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, - diags, tends); + hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, + progs, diags, tends); }); //} } @@ -722,7 +716,9 @@ void MAMAci::set_grids( // Allocate memory for the class members // Kokkos::resize only works on host to allocates memory - Kokkos::resize(rho_, ncol_, nlev_);// FIXME: rho_ is only used internally in compute_w0_and_rho + Kokkos::resize( + rho_, ncol_, + nlev_); // FIXME: rho_ is only used internally in compute_w0_and_rho Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -734,7 +730,7 @@ void MAMAci::set_grids( Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); - for(int i = 0; i < drop_scratch_; ++i) { + for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } // Define the different field layouts that will be used for this process @@ -768,9 +764,9 @@ void MAMAci::set_grids( add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Temperature[K] at midpoints - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // Total pressure [Pa] at midpoints add_field("p_int", scalar3d_layout_int, Pa, @@ -782,58 +778,71 @@ void MAMAci::set_grids( grid_name); // planetary boundary layer height // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process - add_field("cldfrac_tot", scalar3d_layout_mid, nondim, - grid_name); + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + + // ======================================================================== + // Output from this whole process + // ======================================================================== // ------------------------------------------------------------------------ - // Output from this process + // Output from ice nucleation process // ------------------------------------------------------------------------ + auto m3_inv = 1 / m / m / m; // inverse of m3 + // number conc of ice nuclei due to heterogeneous freezing [1/m3] + add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, + grid_name); - // BALLI:??? - add_field("icenuc_num_hetfrz", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to - // BALLI:??? // heterogeneous freezing [1/m3] - add_field("icenuc_num_immfrz", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to immersion - // BALLI:??? // freezing (hetero nuc) [1/m3] - add_field("icenuc_num_depnuc", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to - // deposition nucleation (hetero nuc)[1/m3] - // BALLI:??? - add_field( - "icenuc_num_meydep", scalar3d_layout_mid, 1 / m / m / m, - grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] + // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] + add_field("icenuc_num_immfrz", scalar3d_layout_mid, m3_inv, + grid_name); + // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + add_field("icenuc_num_depnuc", scalar3d_layout_mid, m3_inv, + grid_name); // + + // number conc of ice nuclei due to meyers deposition [1/m3] + add_field("icenuc_num_meydep", scalar3d_layout_mid, m3_inv, + grid_name); + + // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] + add_field("num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, + n_unit, grid_name); + + // number of activated aerosol for ice nucleation[#/kg] + add_field("num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, + grid_name); + + // ------------------------------------------------------------------------ + // Output from droplet activation process (dropmixnuc) + // ------------------------------------------------------------------------ // BALLI:??? - add_field( - "num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, - grid_name); // number of activated aerosol for ice nucleation - // (homogeneous freezing only) [#/kg] - // BALLI:??? - add_field( - "num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, - grid_name); // number of activated aerosol for ice nucleation[#/kg] - // BALLI:??? + // FIXME: THis looks like an internal variable for dropmixnuc, why we need it + // here??? add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); // cloud droplet number mixing ratio [#/kg] // BALLI:??? + // FIXME:This array should update the mmrs and nmrs + // tendencies for interstitial and cloud borne aerosols [#/kg] add_field( "ptend_q", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::aero_model::pcnst}}, - n_unit, grid_name); // tendencies for interstitial and cloud borne - // aerosols [#/kg] - // BALLI:??? - add_field( - "tendnd", scalar3d_layout_mid, n_unit / s, - grid_name); // tendency in droplet number mixing ratio [#/kg/s] - // BALLI:??? + n_unit, grid_name); + + // tendency in droplet number mixing ratio [#/kg/s] + add_field("tendnd", scalar3d_layout_mid, n_unit / s, grid_name); + + // activation fraction for aerosol number [fraction] add_field( "factnum", FieldLayout{{COL, NMODES, LEV}, - {ncol_, mam_coupling::num_aero_modes(),nlev_}}, - nondim, grid_name); // activation fraction for aerosol number [fraction] - // BALLI:??? - add_field("ndropcol", scalar3d_layout_mid, n_unit / s, + {ncol_, mam_coupling::num_aero_modes(), nlev_}}, + nondim, grid_name); + + auto inv_m2 = 1 / m / m; // units of number mixing ratios of tracers + inv_m2.set_string("#/m2"); + // BALLI:??? FIXME: This is internal diagnostic variable + // column-integrated droplet number [#/m2] + add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); // // BALLI:??? add_field("ndropmix", scalar3d_layout_mid, n_unit / s, @@ -944,10 +953,10 @@ void MAMAci::set_grids( auto cm = m / 100; - // units of number mixing ratios of tracers + // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - + // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -1207,9 +1216,6 @@ void MAMAci::run_impl(const double dt) { cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. - - - copy_mam4xx_array_to_scream( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1219,10 +1225,10 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_cw_outp_, coltend_cw_, nlev_); call_hetfrz_compute_tendencies( - team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, - factnum_, hetfrz_immersion_nucleation_tend_, - hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, - naai_hom_, naai_, diagnostic_scratch_, nlev_, dt); + team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, factnum_, + hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, + hetfrz_depostion_nucleation_tend_, naai_hom_, naai_, diagnostic_scratch_, + nlev_, dt); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index be7d116fe5e6..a73e04b8575d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -95,8 +95,8 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d nact_; view_3d mact_; - static constexpr int drop_scratch_ = 15; - view_2d dropmixnuc_scratch_mem_[drop_scratch_]; + static constexpr int dropmix_scratch_ = 15; + view_2d dropmixnuc_scratch_mem_[dropmix_scratch_]; view_2d stratiform_cloud_fraction_; view_2d activation_fraction_accum_idx_; From 3c90273c9a8f13970ee1e37f8e86e7ffef0ca163 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 22:18:52 -0700 Subject: [PATCH 285/476] Further cleanup and code lines moved to match I/O for the process --- .../mam/eamxx_mam_aci_process_interface.cpp | 182 +++++++++--------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 25afa6012e49..636aee9433f9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -780,10 +780,74 @@ void MAMAci::set_grids( // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and + // BALLI:??? + // Vertical velocity variance (wp2) at midpoints + add_field("w_sec", scalar3d_layout_int, m2 / s2, grid_name); + + // BALLI:??? + // FIXME:liq_strat_cld_frac may also need OLD time + // Liquid stratiform cloud fraction at midpoints + add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, + grid_name); + // BALLI:??? + add_field("kvh", scalar3d_layout_int, m2 / s, + grid_name); // Eddy diffusivity for heat + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + const int num_aero_modes = mam_coupling::num_aero_modes(); + FieldLayout scalar4d_layout_mid{ + {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points + // BALLI:??? + add_field("dgnum", scalar4d_layout_mid, m, + grid_name); // dry diameter of aerosols + // ======================================================================== // Output from this whole process // ======================================================================== + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(mode, a); + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + // ------------------------------------------------------------------------ // Output from ice nucleation process // ------------------------------------------------------------------------ @@ -838,88 +902,47 @@ void MAMAci::set_grids( {ncol_, mam_coupling::num_aero_modes(), nlev_}}, nondim, grid_name); - auto inv_m2 = 1 / m / m; // units of number mixing ratios of tracers + auto inv_m2 = 1 / m / m; inv_m2.set_string("#/m2"); // BALLI:??? FIXME: This is internal diagnostic variable // column-integrated droplet number [#/m2] add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); // - // BALLI:??? - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, - grid_name); // droplet number mixing ratio tendency due - // to mixing [#/kg/s] - // BALLI:??? - add_field( - "nsource", scalar3d_layout_mid, n_unit / s, - grid_name); // droplet number mixing ratio source tendency [#/kg/s] - // BALLI:??? - add_field("wtke", scalar3d_layout_mid, n_unit / s, grid_name); // + // BALLI:??? FIXME: This is internal diagnostic variable + // droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); + // BALLI:??? FIXME: This is internal diagnostic variable + // droplet number mixing ratio source tendency [#/kg/s] + add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); - // BALLI:??? + // BALLI:??? FIXME: This is internal diagnostic variable + // subgrid vertical velocity [m/s] + add_field("wtke", scalar3d_layout_mid, m / s, grid_name); + + // BALLI:??? FIXME: This is internal diagnostic variable + // number conc of aerosols activated at supersat [#/m^3] + // note: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud + // [#/m^2/s]] + // / [aero. number conc. in updraft, just below + // cloudbase [#/m^3]] add_field( "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, - n_unit, - grid_name); // number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud - // [#/m^2/s]] - // / [aero. number conc. in updraft, just below - // cloudbase [#/m^3]] - // BALLI:??? + m3_inv, grid_name); + + // BALLI:??? FIXME: This is internal diagnostic variable + // column tendency for diagnostic output add_field( "coltend", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); // column tendency for diagnostic output - // BALLI:??? + nondim, grid_name); + + // BALLI:??? FIXME: This is internal diagnostic variable + // column tendency add_field( "coltend_cw", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); // column tendency - - // MUST FIXME: The aerosols has a wet mixing ratio, we should convert that to - // dry - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = - mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_nmr_field_name = - mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(mode, a); - if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); - } - } - } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + nondim, grid_name); // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM // These outputs should come from the cloud macrophysics process (e.g., SHOC) @@ -928,29 +951,6 @@ void MAMAci::set_grids( auto s2 = s * s; s2.set_string("s^2"); - // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and - // liq_strat_cld_frac may also need OLD time - // BALLI:??? - add_field( - "w_sec", scalar3d_layout_int, m2 / s2, - grid_name); // Vertical velocity variance (wp2) at midpoints - - // BALLI:??? - add_field( - "liq_strat_cld_frac", scalar3d_layout_mid, nondim, - grid_name); // Liquid stratiform cloud fraction at midpoints - // BALLI:??? - add_field("kvh", scalar3d_layout_int, m2 / s, - grid_name); // Eddy diffusivity for heat - - // Layout for 4D (2d horiz X 1d vertical x number of modes) variables - const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ - {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points - // BALLI:??? - add_field("dgnum", scalar4d_layout_mid, m, - grid_name); // dry diameter of aerosols - auto cm = m / 100; // units of number mixing ratios of tracers From 08430d5249a0dfde88eac5f6f60250d960838f43 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 1 Apr 2024 22:54:43 -0700 Subject: [PATCH 286/476] Adds eddy diff of heat as an output for SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 26 ++++++------ .../shoc/eamxx_shoc_process_interface.cpp | 41 +++++++++++-------- .../src/physics/shoc/impl/shoc_main_impl.hpp | 7 +++- .../eamxx/src/physics/shoc/shoc_functions.hpp | 2 + .../tests/single-process/mam/aci/input.yaml | 2 +- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 636aee9433f9..840d3eb181ae 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -780,6 +780,12 @@ void MAMAci::set_grids( // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); + // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM + // These outputs should come from the cloud macrophysics process (e.g., SHOC) + auto m2 = m * m; + m2.set_string("m^2"); + auto s2 = s * s; + s2.set_string("s^2"); // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and // BALLI:??? // Vertical velocity variance (wp2) at midpoints @@ -791,7 +797,7 @@ void MAMAci::set_grids( add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // BALLI:??? - add_field("kvh", scalar3d_layout_int, m2 / s, + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables @@ -944,13 +950,6 @@ void MAMAci::set_grids( FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, nondim, grid_name); - // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - // These outputs should come from the cloud macrophysics process (e.g., SHOC) - auto m2 = m * m; - m2.set_string("m^2"); - auto s2 = s * s; - s2.set_string("s^2"); - auto cm = m / 100; // units of number mixing ratios of tracers @@ -994,13 +993,12 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { void MAMAci::initialize_impl(const RunType run_type) { w_sec_ = get_field_in("w_sec").get_view(); - dgnum_ = get_field_in("dgnum") - .get_view(); // MUST FIXME: is it an input, can - // we compute it using calcsize??? + // MUST FIXME: is it an input, should we invoke calcsize here?? + dgnum_ = get_field_in("dgnum").get_view(); liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); - kvh_ = get_field_in("kvh") - .get_view(); // MUST FIXME: See if scream has it, - // it should com from the land model + + // MUST FIXME: This comes from shoc + kvh_ = get_field_in("eddy_diff_mom").get_view(); nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 6204421c1ea5..4c44bc32fe7e 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -23,6 +23,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids { using namespace ekat::units; + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto Qunit = kg/kg; + Qunit.set_string("kg/kg"); + auto nondim = Units::nondimensional(); + m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -50,22 +56,21 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids constexpr int ps = Spack::n; - const auto nondim = Units::nondimensional(); - const auto m2 = pow(m,2); - const auto s2 = pow(s,2); + const auto m2 = m*m; + const auto s2 = s*s; // These variables are needed by the interface, but not actually passed to shoc_main. add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); add_field("surf_sens_flux", scalar2d , W/m2, grid_name); add_field("surf_mom_flux", vector2d , N/m2, grid_name); - add_field("surf_evap", scalar2d , kg/(m2*s), grid_name); - add_field ("T_mid", scalar3d_mid, K, grid_name, ps); - add_field ("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); + add_field("surf_evap", scalar2d , kg/m2/s, grid_name); + add_field ("T_mid", scalar3d_mid, K, grid_name, ps); + add_field ("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { - add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); } // Input variables @@ -75,23 +80,24 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, pow(kg/kg,2), grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d, kg/(m2*s), grid_name); + add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); add_field("water_flux", scalar2d, m/s, grid_name); add_field("ice_flux", scalar2d, m/s, grid_name); add_field("heat_flux", scalar2d, W/m2, grid_name); @@ -252,6 +258,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); const auto& tk = get_field_out("eddy_diff_mom").get_view(); + const auto& tkh = get_field_out("eddy_diff_heat").get_view(); const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); const auto& phis = get_field_in("phis").get_view(); @@ -285,6 +292,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial){ Kokkos::deep_copy(sgs_buoy_flux,0.0); Kokkos::deep_copy(tk,0.0); + Kokkos::deep_copy(tkh,0.0); Kokkos::deep_copy(tke,0.0004); Kokkos::deep_copy(tke_copy,0.0004); Kokkos::deep_copy(cldfrac_liq,0.0); @@ -321,6 +329,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) input_output.wthv_sec = sgs_buoy_flux; input_output.qtracers = shoc_preprocess.qtracers; input_output.tk = tk; + input_output.tkh = tkh; input_output.shoc_cldfrac = cldfrac_liq; input_output.shoc_ql = qc_copy; diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 13a93992cff1..bb19677eb273 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -114,6 +114,7 @@ void Functions::shoc_main_internal( const uview_1d& wthv_sec, const uview_2d& qtracers, const uview_1d& tk, + const uview_1d& tkh, const uview_1d& shoc_cldfrac, const uview_1d& shoc_ql, // Output Variables @@ -366,6 +367,7 @@ void Functions::shoc_main_internal( const view_2d& wthv_sec, const view_3d& qtracers, const view_2d& tk, + const view_2d& tkh, const view_2d& shoc_cldfrac, const view_2d& shoc_ql, // Output Variables @@ -640,6 +642,7 @@ Int Functions::shoc_main( const auto qw_s = ekat::subview(shoc_input_output.qw, i); const auto wthv_sec_s = ekat::subview(shoc_input_output.wthv_sec, i); const auto tk_s = ekat::subview(shoc_input_output.tk, i); + const auto tkh_s = ekat::subview(shoc_input_output.tkh, i); const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); @@ -672,7 +675,7 @@ Int Functions::shoc_main( wtracer_sfc_s, inv_exner_s, phis_s, // Input workspace, // Workspace host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output - wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output + wthv_sec_s, qtracers_s, tk_s, tkh_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output pblh_s, shoc_ql2_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables @@ -696,7 +699,7 @@ Int Functions::shoc_main( shoc_input.wtracer_sfc, shoc_input.inv_exner, shoc_input.phis, // Input workspace_mgr, // Workspace Manager shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output - shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output + shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.tkh, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output shoc_output.pblh, shoc_output.shoc_ql2, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index d02d498c5a43..2d9744b9aed9 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -147,6 +147,8 @@ struct Functions view_3d qtracers; // eddy coefficient for momentum [m2/s] view_2d tk; + // eddy coefficient for heat [m2/s] + view_2d tkh; // Cloud fraction [-] view_2d shoc_cldfrac; // cloud liquid mixing ratio [kg/kg] diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index b63060e04a58..f21bde0ce3f0 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -31,7 +31,7 @@ initial_conditions: #we should get the following variables from other processes dgnum: 1e-3 - kvh: 1e-3 + eddy_diff_heat: 1e-3 liq_strat_cld_frac: 1e-3 w_sec: 1e-3 w_updraft: 1e-3 From bb0e696853682e92fa0d4fd6b5d6286f1e8bb921 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Apr 2024 12:01:53 -0700 Subject: [PATCH 287/476] Adds tkh-eddy diffusivity of heat in SHOC-2nd attempt --- .../shoc/eamxx_shoc_process_interface.cpp | 10 ++++---- .../src/physics/shoc/impl/shoc_main_impl.hpp | 24 ++++++++++++------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 6 +++-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 4c44bc32fe7e..a988948c0f11 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -84,13 +84,13 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -258,7 +258,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); const auto& tk = get_field_out("eddy_diff_mom").get_view(); - const auto& tkh = get_field_out("eddy_diff_heat").get_view(); const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); const auto& phis = get_field_in("phis").get_view(); @@ -292,7 +291,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial){ Kokkos::deep_copy(sgs_buoy_flux,0.0); Kokkos::deep_copy(tk,0.0); - Kokkos::deep_copy(tkh,0.0); Kokkos::deep_copy(tke,0.0004); Kokkos::deep_copy(tke_copy,0.0004); Kokkos::deep_copy(cldfrac_liq,0.0); @@ -329,13 +327,13 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) input_output.wthv_sec = sgs_buoy_flux; input_output.qtracers = shoc_preprocess.qtracers; input_output.tk = tk; - input_output.tkh = tkh; input_output.shoc_cldfrac = cldfrac_liq; input_output.shoc_ql = qc_copy; // Output Variables output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; + output.tkh = get_field_out("eddy_diff_heat").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index bb19677eb273..23348a19f50a 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -114,12 +114,12 @@ void Functions::shoc_main_internal( const uview_1d& wthv_sec, const uview_2d& qtracers, const uview_1d& tk, - const uview_1d& tkh, const uview_1d& shoc_cldfrac, const uview_1d& shoc_ql, // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, + const uview_1d& tkh_out, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -266,7 +266,7 @@ void Functions::shoc_main_internal( check_tke(team,nlev,tke); } - // End SHOC parameterization + // End SHOC parameterization // Use SHOC outputs to update the host model // temperature @@ -311,6 +311,12 @@ void Functions::shoc_main_internal( workspace, // Workspace pblh); // Output + // Assign tkh to tkh output variable + const Int nlev_pack = ekat::npack(nlev); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { + tkh_out(k) = tkh(k); + }); + // Release temporary variables from the workspace workspace.template release_many_contiguous<6>( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); @@ -367,12 +373,12 @@ void Functions::shoc_main_internal( const view_2d& wthv_sec, const view_3d& qtracers, const view_2d& tk, - const view_2d& tkh, const view_2d& shoc_cldfrac, const view_2d& shoc_ql, // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, + const view_2d& tkh_out, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, @@ -567,6 +573,8 @@ void Functions::shoc_main_internal( kbfs,shoc_cldfrac, // Input workspace_mgr, // Workspace mgr pblh); // Output + + tkh_out = tkh; } #endif @@ -642,10 +650,10 @@ Int Functions::shoc_main( const auto qw_s = ekat::subview(shoc_input_output.qw, i); const auto wthv_sec_s = ekat::subview(shoc_input_output.wthv_sec, i); const auto tk_s = ekat::subview(shoc_input_output.tk, i); - const auto tkh_s = ekat::subview(shoc_input_output.tkh, i); const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); + const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); const auto shoc_mix_s = ekat::subview(shoc_history_output.shoc_mix, i); const auto w_sec_s = ekat::subview(shoc_history_output.w_sec, i); const auto thl_sec_s = ekat::subview(shoc_history_output.thl_sec, i); @@ -675,9 +683,9 @@ Int Functions::shoc_main( wtracer_sfc_s, inv_exner_s, phis_s, // Input workspace, // Workspace host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output - wthv_sec_s, qtracers_s, tk_s, tkh_s, shoc_cldfrac_s, // Input/Output + wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, // Output + pblh_s, shoc_ql2_s, tkh_out_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -699,9 +707,9 @@ Int Functions::shoc_main( shoc_input.wtracer_sfc, shoc_input.inv_exner, shoc_input.phis, // Input workspace_mgr, // Workspace Manager shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output - shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.tkh, shoc_input_output.shoc_cldfrac, // Input/Output + shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, // Output + shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 2d9744b9aed9..ccc941bfd055 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -147,8 +147,6 @@ struct Functions view_3d qtracers; // eddy coefficient for momentum [m2/s] view_2d tk; - // eddy coefficient for heat [m2/s] - view_2d tkh; // Cloud fraction [-] view_2d shoc_cldfrac; // cloud liquid mixing ratio [kg/kg] @@ -163,6 +161,8 @@ struct Functions view_1d pblh; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; + // eddy coefficient for heat [m2/s] + view_2d tkh; }; // This struct stores output views for SHOC diagnostics for shoc_main. @@ -890,6 +890,7 @@ struct Functions // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, + const uview_1d& tkh_out, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -961,6 +962,7 @@ struct Functions // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, + const view_2d& tkh_out, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, From 7fc6b61f2b555e1e98760313e1e01398e7dfe51f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 3 Apr 2024 16:44:17 -0700 Subject: [PATCH 288/476] Adds a multi process test --- .../mam/eamxx_mam_aci_process_interface.cpp | 4 +- .../multi-process/physics_only/CMakeLists.txt | 1 + .../physics_only/mam/shoc_aci/CMakeLists.txt | 41 ++++++++++ .../physics_only/mam/shoc_aci/input.yaml | 74 +++++++++++++++++++ .../physics_only/mam/shoc_aci/output.yaml | 57 ++++++++++++++ .../tests/single-process/mam/aci/input.yaml | 2 +- .../tests/single-process/mam/aci/output.yaml | 1 + 7 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 840d3eb181ae..2d30f9a2bcd7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -797,7 +797,7 @@ void MAMAci::set_grids( add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, grid_name); // BALLI:??? - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Eddy diffusivity for heat // Layout for 4D (2d horiz X 1d vertical x number of modes) variables @@ -998,7 +998,7 @@ void MAMAci::initialize_impl(const RunType run_type) { liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); // MUST FIXME: This comes from shoc - kvh_ = get_field_in("eddy_diff_mom").get_view(); + kvh_ = get_field_in("eddy_diff_heat").get_view(); nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 61b224f71ae6..046bcecbdece 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -4,6 +4,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_spa_p3_rrtmgp) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) + add_subdirectory(mam/shoc_aci) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt new file mode 100644 index 000000000000..ea13508e3e58 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_aci) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 scream_rrtmgp mam + LABELS shoc cld p3 rrtmgp physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 rrtmgp physics PEM mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml new file mode 100644 index 000000000000..bf9f952ab1de --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml @@ -0,0 +1,74 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic,rrtmgp] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + orbital_year: 1990 + do_aerosol_rad: false + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + aero_g_sw: 0.0 + aero_ssa_sw: 0.0 + aero_tau_sw: 0.0 + aero_tau_lw: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml new file mode 100644 index 000000000000..a8b22c16ba09 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml @@ -0,0 +1,57 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index f21bde0ce3f0..a4d006853131 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -25,11 +25,11 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - pbl_height : 1.0 phis : 1.0 #These should come from the input file #we should get the following variables from other processes + pbl_height : 1.0 dgnum: 1e-3 eddy_diff_heat: 1e-3 liq_strat_cld_frac: 1e-3 diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml index cb54b742c5a6..670667576c39 100644 --- a/components/eamxx/tests/single-process/mam/aci/output.yaml +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -6,6 +6,7 @@ Fields: Physics: Field Names: - T_mid + - dgnum output_control: Frequency: 2 From 4f0e0850ca0f9ca9a6716ca7b325a8ce7544bd0b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Apr 2024 10:30:11 -0700 Subject: [PATCH 289/476] Adds two multi-process tests and changes to SHOC for copying tkh_out --- .../eamxx/src/physics/shoc/CMakeLists.txt | 1 + .../shoc/disp/shoc_assign_2d_view_disp.cpp | 29 +++++++++ .../src/physics/shoc/impl/shoc_main_impl.hpp | 5 +- .../eamxx/src/physics/shoc/shoc_functions.hpp | 7 +++ .../multi-process/physics_only/CMakeLists.txt | 3 +- .../shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 41 +++++++++++++ .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 61 +++++++++++++++++++ .../mam/shoc_cldfrac_mam4_aci_p3/output.yaml | 44 +++++++++++++ .../CMakeLists.txt | 2 +- .../input.yaml | 0 .../output.yaml | 0 11 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml rename components/eamxx/tests/multi-process/physics_only/mam/{shoc_aci => shoc_cldfrac_mam4_aci_p3_rrtmgp}/CMakeLists.txt (96%) rename components/eamxx/tests/multi-process/physics_only/mam/{shoc_aci => shoc_cldfrac_mam4_aci_p3_rrtmgp}/input.yaml (100%) rename components/eamxx/tests/multi-process/physics_only/mam/{shoc_aci => shoc_cldfrac_mam4_aci_p3_rrtmgp}/output.yaml (100%) diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index e37379095d00..f57506fe0ed0 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -85,6 +85,7 @@ set(SHOC_SK_SRCS disp/shoc_diag_third_shoc_moments_disp.cpp disp/shoc_assumed_pdf_disp.cpp disp/shoc_update_host_dse_disp.cpp + disp/shoc_assign_2d_view_disp.cpp ) if (NOT SCREAM_DEBUG) diff --git a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp new file mode 100644 index 000000000000..8f5592a38360 --- /dev/null +++ b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp @@ -0,0 +1,29 @@ +#include "shoc_functions.hpp" + +#include "ekat/kokkos/ekat_subview_utils.hpp" + +namespace scream { +namespace shoc { + +template<> +void Functions +::shoc_assign_2d_view_disp( + const Int& shcol, + const Int& nlev, + const view_2d& input_view, + const view_2d& output_view) +{ + /*using ExeSpace = typename KT::ExeSpace; + + const auto nlev_packs = ekat::npack(nlev); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { + const Int i = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { + output_view (i,k) = input_view(i,k); + }); + });*/ +} + +} // namespace shoc +} // namespace scream diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 23348a19f50a..34a0455c14ff 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -311,7 +311,7 @@ void Functions::shoc_main_internal( workspace, // Workspace pblh); // Output - // Assign tkh to tkh output variable + // Assign tkh to the tkh output variable const Int nlev_pack = ekat::npack(nlev); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { tkh_out(k) = tkh(k); @@ -574,7 +574,8 @@ void Functions::shoc_main_internal( workspace_mgr, // Workspace mgr pblh); // Output - tkh_out = tkh; + // Assign tkh to the tkh output variable + shoc_assign_2d_view_disp(shcol, nlev, tkh_out,tkh); } #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index ccc941bfd055..0b6512630644 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -1205,6 +1205,13 @@ struct Functions const view_2d& tk, const view_2d& tkh, const view_2d& isotropy); + + static void shoc_assign_2d_view_disp( + const Int& shcol, + const Int& nlev, + const view_2d& input_view, + const view_2d& output_view); + #endif }; // struct Functions diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 046bcecbdece..8247f178561c 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -4,7 +4,8 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_spa_p3_rrtmgp) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) - add_subdirectory(mam/shoc_aci) + add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) + add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt new file mode 100644 index 000000000000..946d68cb7501 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_mam4_aci_p3) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 mam + LABELS shoc cld p3 physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 physics PEM mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml new file mode 100644 index 000000000000..76efb136c00d --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml new file mode 100644 index 000000000000..f4aec357e9f6 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml @@ -0,0 +1,44 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + - T_mid +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt similarity index 96% rename from components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt index ea13508e3e58..bb30c120d38b 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt @@ -1,6 +1,6 @@ INCLUDE (ScreamUtils) -set (TEST_BASE_NAME shoc_aci) +set (TEST_BASE_NAME shoc_cldfrac_mam4_aci_p3_rrtmgp) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml similarity index 100% rename from components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/input.yaml rename to components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml similarity index 100% rename from components/eamxx/tests/multi-process/physics_only/mam/shoc_aci/output.yaml rename to components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml From dc5cec22b75e343b6c1b1bdc093f3fe0fa1d57df Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Apr 2024 11:15:30 -0700 Subject: [PATCH 290/476] Adds shoc and aci multi process test --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../mam/shoc_mam4_aci/CMakeLists.txt | 41 ++++++++++++++ .../physics_only/mam/shoc_mam4_aci/input.yaml | 56 +++++++++++++++++++ .../mam/shoc_mam4_aci/output.yaml | 17 ++++++ 4 files changed, 115 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 8247f178561c..2b63cd44d373 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -4,6 +4,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_spa_p3_rrtmgp) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) + add_subdirectory(mam/shoc_mam4_aci) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt new file mode 100644 index 000000000000..af6fff36b0a8 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_mam4_aci) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc mam + LABELS shoc physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc physics mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml new file mode 100644 index 000000000000..d34390e56d5f --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -0,0 +1,56 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,mam4_aci] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml new file mode 100644 index 000000000000..590cd86ab7ea --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml @@ -0,0 +1,17 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From ad8ec235bc44df7a36e8c3ed1c1ca54e1d56f4b1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 4 Apr 2024 11:52:39 -0700 Subject: [PATCH 291/476] Adds another multi-process test with optics --- .../multi-process/physics_only/CMakeLists.txt | 1 + .../CMakeLists.txt | 41 +++++++++ .../input.yaml | 83 +++++++++++++++++++ .../output.yaml | 57 +++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index 2b63cd44d373..b9c13234796b 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -7,6 +7,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_mam4_aci) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) + add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..c956df2b8fd8 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 scream_rrtmgp mam + LABELS shoc cld p3 rrtmgp physics mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 rrtmgp physics PEM mam + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml new file mode 100644 index 000000000000..5bd15fa7d5b8 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -0,0 +1,83 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_optics, rrtmgp] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + orbital_year: 1990 + do_aerosol_rad: false + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_sens_flux: 0.0 + surf_evap: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + dgnum: 1e-3 + liq_strat_cld_frac: 1e-3 + w_sec: 1e-3 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml new file mode 100644 index 000000000000..a8b22c16ba09 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml @@ -0,0 +1,57 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cld_p3_rrtmgp_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 929da35210ad4c92de0a792f107ac55c6939531f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Apr 2024 12:14:55 -0700 Subject: [PATCH 292/476] Adds w_variance in SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 17 ++++++++++++----- .../shoc/eamxx_shoc_process_interface.cpp | 15 ++++++++------- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 4 ++-- .../physics_only/mam/shoc_mam4_aci/input.yaml | 1 - 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2d30f9a2bcd7..193e30ee52e5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -786,10 +786,17 @@ void MAMAci::set_grids( m2.set_string("m^2"); auto s2 = s * s; s2.set_string("s^2"); - // MUST FIXME: w_sec, is at OLD time step; strat_cld_frac and - // BALLI:??? - // Vertical velocity variance (wp2) at midpoints - add_field("w_sec", scalar3d_layout_int, m2 / s2, grid_name); + + // NOTE: w_variance im microp_aero_run.F90 is at "itim_old" dynamics time step + // Since, we are using SE dycore, itim_old is 1 which is equivalent to the + // current time step. For other dycores (such as EUL), it may be different + // and we might need to revisit this + + //FIXME: w_variance in microp_aero_run.F90 is at the interfaces but + // SHOC provides it at the midpoints. Verify how it is being used. + + // Vertical velocity variance at midpoints + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // BALLI:??? // FIXME:liq_strat_cld_frac may also need OLD time @@ -991,7 +998,7 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { } void MAMAci::initialize_impl(const RunType run_type) { - w_sec_ = get_field_in("w_sec").get_view(); + w_sec_ = get_field_in("w_variance").get_view(); // MUST FIXME: is it an input, should we invoke calcsize here?? dgnum_ = get_field_in("dgnum").get_view(); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index a988948c0f11..70404b7997a3 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -80,17 +80,18 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables add_field("pbl_height", scalar2d_layout_col, m, grid_name); add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -338,7 +339,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; history_output.isotropy = m_buffer.isotropy; - history_output.w_sec = m_buffer.w_sec; + history_output.w_sec = get_field_out("w_variance").get_view(); history_output.thl_sec = m_buffer.thl_sec; history_output.qw_sec = m_buffer.qw_sec; history_output.qwthl_sec = m_buffer.qwthl_sec; diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 76efb136c00d..ca0a99c50cb4 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -52,8 +52,8 @@ initial_conditions: precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 + #liq_strat_cld_frac: 1e-3 + #w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index d34390e56d5f..84212175a3d7 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -48,7 +48,6 @@ initial_conditions: surf_evap: 0.0 dgnum: 1e-3 liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 # The parameters for I/O control Scorpio: From 0e2b470a66f8e156636569917e2f2b2309e34d32 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Apr 2024 17:42:53 -0700 Subject: [PATCH 293/476] Add old and new cloud fraction inputs from SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 46 ++++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 + .../shoc/eamxx_shoc_process_interface.cpp | 12 +++-- .../shoc/eamxx_shoc_process_interface.hpp | 10 +++- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 8 +++- .../physics_only/mam/shoc_mam4_aci/input.yaml | 5 +- 6 files changed, 52 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 193e30ee52e5..10e41db4e430 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -299,13 +299,11 @@ void compute_nucleate_ice_tendencies( }); } KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction(const haero::ThreadTeam &team, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - MAMAci::const_view_2d qc, - MAMAci::const_view_2d qi, - MAMAci::const_view_2d liqcldf, const int icol, - const int top_lev) { +void store_liquid_cloud_fraction( + const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac_new, + MAMAci::view_2d cloud_frac_old, MAMAci::const_view_2d qc, + MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, + MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev) { //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value @@ -323,8 +321,7 @@ void store_liquid_cloud_fraction(const haero::ThreadTeam &team, const Real qcld = qc(icol, kk) + qi(icol, kk); if(qcld > qsmall) { cloud_frac_new(icol, kk) = liqcldf(icol, kk); - cloud_frac_old(icol, kk) = - liqcldf(icol, kk); // FIXME should be liqcldf_old + cloud_frac_old(icol, kk) = liqcldf_prev(icol, kk); } else { cloud_frac_new(icol, kk) = 0; cloud_frac_old(icol, kk) = 0; @@ -336,6 +333,7 @@ void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d cloud_frac_old, mam_coupling::WetAtmosphere &wet_atmosphere, MAMAci::const_view_2d liqcldf, + MAMAci::const_view_2d liqcldf_prev, const int top_lev) { MAMAci::const_view_2d qc = wet_atmosphere.qc; MAMAci::const_view_2d qi = wet_atmosphere.qi; @@ -343,7 +341,7 @@ void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, - qi, liqcldf, icol, top_lev); + qi, liqcldf, liqcldf_prev, icol, top_lev); }); } KOKKOS_INLINE_FUNCTION @@ -786,22 +784,27 @@ void MAMAci::set_grids( m2.set_string("m^2"); auto s2 = s * s; s2.set_string("s^2"); - + // NOTE: w_variance im microp_aero_run.F90 is at "itim_old" dynamics time step - // Since, we are using SE dycore, itim_old is 1 which is equivalent to the + // Since, we are using SE dycore, itim_old is 1 which is equivalent to the // current time step. For other dycores (such as EUL), it may be different // and we might need to revisit this - //FIXME: w_variance in microp_aero_run.F90 is at the interfaces but - // SHOC provides it at the midpoints. Verify how it is being used. + // FIXME: w_variance in microp_aero_run.F90 is at the interfaces but + // SHOC provides it at the midpoints. Verify how it is being used. // Vertical velocity variance at midpoints add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); - // BALLI:??? - // FIXME:liq_strat_cld_frac may also need OLD time - // Liquid stratiform cloud fraction at midpoints - add_field("liq_strat_cld_frac", scalar3d_layout_mid, nondim, + // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is + // equivalent + // to "alst" in the shoc_intr.F90. In the C++ code, it is used as + // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" + + // Liquid stratiform cloud fraction at midpoints + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name); + + add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name); // BALLI:??? add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, @@ -1001,8 +1004,9 @@ void MAMAci::initialize_impl(const RunType run_type) { w_sec_ = get_field_in("w_variance").get_view(); // MUST FIXME: is it an input, should we invoke calcsize here?? - dgnum_ = get_field_in("dgnum").get_view(); - liqcldf_ = get_field_in("liq_strat_cld_frac").get_view(); + dgnum_ = get_field_in("dgnum").get_view(); + liqcldf_ = get_field_in("cldfrac_liq").get_view(); + liqcldf_prev_ = get_field_in("cldfrac_liq_prev").get_view(); // MUST FIXME: This comes from shoc kvh_ = get_field_in("eddy_diff_heat").get_view(); @@ -1192,7 +1196,7 @@ void MAMAci::run_impl(const double dt) { dry_atm_, aitken_dry_dia_, nlev_); store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atm_, liqcldf_, top_lev_); + wet_atm_, liqcldf_, liqcldf_prev_, top_lev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index a73e04b8575d..6873ccd33975 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -65,6 +65,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d naai_hom_; view_2d naai_; const_view_2d liqcldf_; + const_view_2d liqcldf_prev_; const_view_2d kvh_; view_2d cloud_frac_new_; diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 70404b7997a3..6059a4369ed9 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,10 +88,11 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d_layout_col, m, grid_name); - add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); - add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); + add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -257,6 +258,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) const auto& qv = get_field_out("qv").get_view(); const auto& tke = get_field_out("tke").get_view(); const auto& cldfrac_liq = get_field_out("cldfrac_liq").get_view(); + const auto& cldfrac_liq_prev = get_field_out("cldfrac_liq_prev").get_view(); const auto& sgs_buoy_flux = get_field_out("sgs_buoy_flux").get_view(); const auto& tk = get_field_out("eddy_diff_mom").get_view(); const auto& inv_qc_relvar = get_field_out("inv_qc_relvar").get_view(); @@ -301,7 +303,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) T_mid,p_mid,p_int,pseudo_density,omega,phis,surf_sens_flux,surf_evap, surf_mom_flux,qtracers,qv,qc,qc_copy,tke,tke_copy,z_mid,z_int, dse,rrho,rrho_i,thv,dz,zt_grid,zi_grid,wpthlp_sfc,wprtp_sfc,upwp_sfc,vpwp_sfc, - wtracer_sfc,wm_zt,inv_exner,thlm,qw); + wtracer_sfc,wm_zt,inv_exner,thlm,qw, cldfrac_liq, cldfrac_liq_prev); // Input Variables: input.zt_grid = shoc_preprocess.zt_grid; diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index d3f5b05f88da..1bc0d14afe02 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -83,6 +83,9 @@ class SHOCMacrophysics : public scream::AtmosphereProcess Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { + + cldfrac_liq_prev(i,k)=cldfrac_liq(i,k); + const auto range = ekat::range(k*Spack::n); const Smask in_nlev_range = (range < nlev); @@ -198,6 +201,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess view_2d thlm; view_2d qw; view_2d cloud_frac; + view_2d cldfrac_liq; + view_2d cldfrac_liq_prev; // Assigning local variables void set_variables(const int ncol_, const int nlev_, const int num_qtracers_, @@ -213,7 +218,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess const view_2d& dse_, const view_2d& rrho_, const view_2d& rrho_i_, const view_2d& thv_, const view_2d& dz_,const view_2d& zt_grid_,const view_2d& zi_grid_, const view_1d& wpthlp_sfc_, const view_1d& wprtp_sfc_,const view_1d& upwp_sfc_,const view_1d& vpwp_sfc_, const view_2d& wtracer_sfc_, - const view_2d& wm_zt_,const view_2d& inv_exner_,const view_2d& thlm_,const view_2d& qw_) + const view_2d& wm_zt_,const view_2d& inv_exner_,const view_2d& thlm_,const view_2d& qw_, + const view_2d& cldfrac_liq_, const view_2d& cldfrac_liq_prev_) { ncol = ncol_; nlev = nlev_; @@ -254,6 +260,8 @@ class SHOCMacrophysics : public scream::AtmosphereProcess inv_exner = inv_exner_; thlm = thlm_; qw = qw_; + cldfrac_liq=cldfrac_liq_; + cldfrac_liq_prev=cldfrac_liq_prev_; } // set_variables }; // SHOCPreprocess /* --------------------------------------------------------------------------------------------*/ diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index ca0a99c50cb4..f41b961a4f5c 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -47,13 +47,17 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variable required for shoc surf_sens_flux: 0.0 surf_evap: 0.0 + + #variable required for p3 precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci dgnum: 1e-3 - #liq_strat_cld_frac: 1e-3 - #w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index 84212175a3d7..1d6d4f48eba7 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -44,10 +44,13 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables needed for SHOC surf_sens_flux: 0.0 surf_evap: 0.0 + + #variables needed for mam4-aci dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 # The parameters for I/O control Scorpio: From 968f5f3b5708816ad6a4b4dbb4e87f598f62aa66 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 9 Apr 2024 15:57:15 -0700 Subject: [PATCH 294/476] A lot of cleanup and changed Fieldlayour for dgnum --- .../mam/eamxx_mam_aci_process_interface.cpp | 188 +++++++++--------- .../eamxx/src/share/field/field_tag.hpp | 38 ++++ .../input.yaml | 8 +- .../input.yaml | 10 +- .../tests/single-process/mam/aci/input.yaml | 6 +- 5 files changed, 145 insertions(+), 105 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 10e41db4e430..5b9a2b1d2f00 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -210,7 +210,7 @@ void compute_aitken_dry_diameter(const haero::ThreadTeam &team, const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol, kk) = dgnum(icol, kk, aitken_idx); + aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); }); } void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, @@ -230,7 +230,7 @@ void compute_nucleate_ice_tendencies( MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, mam_coupling::AerosolState &dry_aerosol_state, mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, - const int nlev) { + const int nlev, const double dt) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation @@ -248,17 +248,13 @@ void compute_nucleate_ice_tendencies( // Set up an atmosphere, surface, diagnostics, pronostics and tendencies // class. - Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), - ekat::subview(qv_dry, icol), dummy, dummy, dummy, dummy, dummy, - dummy, dummy, ekat::subview(cldfrac, icol), - ekat::subview(w_updraft, icol), pblh); - // set surface state data haero::Surface surf{}; mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + // nucleation doesn't use any diagnostics, so it's okay to leave this // alone for now mam4::Diagnostics diags(nlev); @@ -283,7 +279,7 @@ void compute_nucleate_ice_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0, dt = 0; + const Real t = 0; /* NOTE:"state_q" is a combination of subset of tracers added by "int_mmr_field_name" and "int_nmr_field_name". Only output we care @@ -294,8 +290,8 @@ void compute_nucleate_ice_tendencies( ! input rho, wsubice, strat_cld_frac, dgnum, & ! input naai, naai_hom) ! output */ - nucleate_ice.compute_tendencies(aero_config, team, t, dt, atmos, surf, - progs, diags, tends); + nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, + surf, progs, diags, tends); }); } KOKKOS_INLINE_FUNCTION @@ -605,12 +601,6 @@ void call_hetfrz_compute_tendencies( haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); - /*Real pblh = 0; - haero::Atmosphere atmos( - nlev, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), dummy, - ekat::subview(qc, icol), ekat::subview(nc, icol), dummy, dummy, - dummy, dummy, dummy, dummy, dummy, pblh);*/ - // set surface state data haero::Surface surf{}; mam4::Prognostics progs = mam_coupling::aerosols_for_column(dry_aerosol_state, icol); @@ -707,16 +697,14 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) void MAMAci::set_grids( const std::shared_ptr grids_manager) { grid_ = grids_manager->get_grid("Physics"); // Use physics grid - const auto &grid_name = grid_->name(); + const auto &grid_name = grid_->name(); // Name of the grid ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Allocate memory for the class members // Kokkos::resize only works on host to allocates memory - Kokkos::resize( - rho_, ncol_, - nlev_); // FIXME: rho_ is only used internally in compute_w0_and_rho + Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); Kokkos::resize(wsub_, ncol_, nlev_); @@ -735,60 +723,69 @@ void MAMAci::set_grids( using namespace ShortFieldTagsNames; // Layout for 3D (2d horiz X 1d vertical) variables - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // mid points - FieldLayout scalar3d_layout_int{{COL, ILEV}, - {ncol_, nlev_ + 1}}; // interfaces + // mid points + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + // interfaces + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); + auto n_unit = 1 / kg; // units of number mixing ratios of tracers n_unit.set_string("#/kg"); + auto nondim = ekat::units::Units::nondimensional(); + // atmospheric quantities - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud ice number mixing ratio [1/kg] - - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // Temperature[K] at midpoints + // specific humidity [kg/kg] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_layout_mid, K, grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // Total pressure [Pa] at midpoints - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // Total pressure [Pa] at interfaces - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // Layer thickness(pdel) [Pa] at midpoints + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); - add_field("pbl_height", scalar2d_layout_col, m, - grid_name); // planetary boundary layer height + // planetary boundary layer height + add_field("pbl_height", scalar2d_layout_col, m, grid_name); // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); - // Inputs (atmospheric quantities) for aci codes that existed in PBUF in EAM - // These outputs should come from the cloud macrophysics process (e.g., SHOC) auto m2 = m * m; m2.set_string("m^2"); auto s2 = s * s; s2.set_string("s^2"); - // NOTE: w_variance im microp_aero_run.F90 is at "itim_old" dynamics time step - // Since, we are using SE dycore, itim_old is 1 which is equivalent to the - // current time step. For other dycores (such as EUL), it may be different - // and we might need to revisit this + // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time + // step Since, we are using SE dycore, itim_old is 1 which is equivalent to + // the current time step. For other dycores (such as EUL), it may be different + // and we might need to revisit this. // FIXME: w_variance in microp_aero_run.F90 is at the interfaces but // SHOC provides it at the midpoints. Verify how it is being used. @@ -797,26 +794,26 @@ void MAMAci::set_grids( add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is - // equivalent - // to "alst" in the shoc_intr.F90. In the C++ code, it is used as - // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" + // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as + // "shoc_cldfrac" and in the F90 code it is called "cloud_frac" // Liquid stratiform cloud fraction at midpoints add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name); + // Previous value of liquid stratiform cloud fraction at midpoints add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name); - // BALLI:??? - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, - grid_name); // Eddy diffusivity for heat + + // Eddy diffusivity for heat + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{ - {COL, LEV, NMODES}, {ncol_, nlev_, num_aero_modes}}; // mid points - // BALLI:??? - add_field("dgnum", scalar4d_layout_mid, m, - grid_name); // dry diameter of aerosols + FieldLayout scalar4d_layout_mid{{COL, NMODES, LEV}, + {ncol_, num_aero_modes, nlev_}}; + + // dry diameter of aerosols [m] + add_field("dgnum", scalar4d_layout_mid, m, grid_name); // ======================================================================== // Output from this whole process @@ -895,56 +892,50 @@ void MAMAci::set_grids( // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ - // BALLI:??? - // FIXME: THis looks like an internal variable for dropmixnuc, why we need it - // here??? - add_field("qcld", scalar3d_layout_mid, n_unit, - grid_name); // cloud droplet number mixing ratio [#/kg] - // BALLI:??? - // FIXME:This array should update the mmrs and nmrs - // tendencies for interstitial and cloud borne aerosols [#/kg] - add_field( - "ptend_q", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::aero_model::pcnst}}, - n_unit, grid_name); + + constexpr int pcnst = mam4::aero_model::pcnst; + FieldLayout scalar4d_layout_nconst_mid{{COL, LEV, MAM_NCNST}, + {ncol_, nlev_, pcnst}}; + + // tendencies for interstitial and cloud borne aerosols [kg/kg or #/kg] + add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); // tendency in droplet number mixing ratio [#/kg/s] add_field("tendnd", scalar3d_layout_mid, n_unit / s, grid_name); // activation fraction for aerosol number [fraction] - add_field( - "factnum", - FieldLayout{{COL, NMODES, LEV}, - {ncol_, mam_coupling::num_aero_modes(), nlev_}}, - nondim, grid_name); + add_field("factnum", scalar4d_layout_mid, nondim, grid_name); + + // NOTE: Here is a series of internal dropmixnuc variables; + // maybe we should move them to diagnostics later + + // cloud droplet number mixing ratio [#/kg] + add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); auto inv_m2 = 1 / m / m; inv_m2.set_string("#/m2"); - // BALLI:??? FIXME: This is internal diagnostic variable + // column-integrated droplet number [#/m2] - add_field("ndropcol", scalar3d_layout_mid, inv_m2, - grid_name); // - // BALLI:??? FIXME: This is internal diagnostic variable + add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); + // droplet number mixing ratio tendency due to mixing [#/kg/s] add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable + // droplet number mixing ratio source tendency [#/kg/s] add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable // number conc of aerosols activated at supersat [#/m^3] - // note: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud - // [#/m^2/s]] - // / [aero. number conc. in updraft, just below - // cloudbase [#/m^3]] - add_field( - "ccn", FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::psat}}, - m3_inv, grid_name); + // NOTE: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + + constexpr int psat = mam4::ndrop::psat; + FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, + {ncol_, nlev_, psat}}; + add_field("ccn", scalar4d_layout_psat_mid, m3_inv, grid_name); // BALLI:??? FIXME: This is internal diagnostic variable // column tendency for diagnostic output @@ -1174,6 +1165,7 @@ void MAMAci::run_impl(const double dt) { // All the inputs are available to compute w0 and rho // Convert from omega to w (vertical velocity) // Negative omega means rising motion + // FIXME: Why wet_atm_ used here????? We should use dry_atm compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, dry_atm_, top_lev_, nlev_); @@ -1193,8 +1185,8 @@ void MAMAci::run_impl(const double dt) { // FIXME: Find out in-outs of the following call! compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, - dry_atm_, aitken_dry_dia_, nlev_); - + dry_atm_, aitken_dry_dia_, nlev_, dt); + // FIXME: Why wet_atm_ used here????? We should use dry_atm store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, wet_atm_, liqcldf_, liqcldf_prev_, top_lev_); @@ -1232,7 +1224,7 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_outp_, coltend_, nlev_); copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - + // FIXME: Why wet_atm_ used here????? We should use dry_atm call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, factnum_, hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 673208d6ae86..79515420045d 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,6 +34,14 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, + num_modes, + num_constituents, + psat, + // + nrefindex_real, + nrefindex_im, + ncoef_number, + mode }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -50,6 +58,15 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; + constexpr auto NMODES = FieldTag::num_modes; + constexpr auto MAM_NCNST = FieldTag::num_constituents; + constexpr auto MAM_PSAT = FieldTag::psat; + // + constexpr auto NREFINDEX_REAL = FieldTag::nrefindex_real; + constexpr auto NREFINDEX_IM = FieldTag::nrefindex_im; + + constexpr auto NCOEF_NUMBER = FieldTag::ncoef_number; + constexpr auto MODE = FieldTag::mode; } inline std::string e2str (const FieldTag ft) { @@ -80,6 +97,27 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; + case FieldTag::num_modes: + name = "num_modes"; + break; + case FieldTag::num_constituents: + name = "num_constituents"; + break; + case FieldTag::psat: + name = "psat"; + break; + case FieldTag::nrefindex_real: + name = "refindex_real"; + break; + case FieldTag::nrefindex_im: + name = "refindex_im"; + break; + case FieldTag::ncoef_number: + name = "coef_number"; + break; + case FieldTag::mode: + name = "mode"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml index 5bd15fa7d5b8..f3198b1308a0 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -69,13 +69,17 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables required for shoc surf_sens_flux: 0.0 surf_evap: 0.0 + + #variables required for p3 precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml index bf9f952ab1de..d61f1318f8f4 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml @@ -56,17 +56,23 @@ initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables required for shoc surf_sens_flux: 0.0 surf_evap: 0.0 + + #variables required for p3 precip_ice_surf_mass: 0.0 precip_liq_surf_mass: 0.0 + + #variables required for rrtmgp aero_g_sw: 0.0 aero_ssa_sw: 0.0 aero_tau_sw: 0.0 aero_tau_lw: 0.0 + + #variables required for mam4_aci dgnum: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index a4d006853131..dc1b779875a8 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -32,9 +32,9 @@ initial_conditions: pbl_height : 1.0 dgnum: 1e-3 eddy_diff_heat: 1e-3 - liq_strat_cld_frac: 1e-3 - w_sec: 1e-3 - w_updraft: 1e-3 + cldfrac_liq: 1e-3 + cldfrac_liq_prev: 0.1 + w_variance: 1e-3 # The parameters for I/O control Scorpio: From 5f152af650b8a4d7006fe3efcad4d8906e09daff Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 9 Apr 2024 18:01:01 -0700 Subject: [PATCH 295/476] Clean up round for initialize impl --- .../mam/eamxx_mam_aci_process_interface.cpp | 179 +++++++++--------- .../eamxx/src/share/field/field_tag.hpp | 5 + 2 files changed, 98 insertions(+), 86 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 5b9a2b1d2f00..b6dfbacc45d0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -693,9 +693,12 @@ const Real wsubmin = 1; MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) {} -// set grid for all the inputs and outputs +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMAci::set_grids( const std::shared_ptr grids_manager) { + // set grid for all the inputs and outputs grid_ = grids_manager->get_grid("Physics"); // Use physics grid const auto &grid_name = grid_->name(); // Name of the grid @@ -853,18 +856,19 @@ void MAMAci::set_grids( add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } - } + } // end for loop num species + } // end for loop for num modes + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } + } // end for loop num gases // ------------------------------------------------------------------------ // Output from ice nucleation process // ------------------------------------------------------------------------ - auto m3_inv = 1 / m / m / m; // inverse of m3 + const auto m3_inv = 1 / m / m / m; // inverse of m3 // number conc of ice nuclei due to heterogeneous freezing [1/m3] add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, grid_name); @@ -875,7 +879,7 @@ void MAMAci::set_grids( // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] add_field("icenuc_num_depnuc", scalar3d_layout_mid, m3_inv, - grid_name); // + grid_name); // number conc of ice nuclei due to meyers deposition [1/m3] add_field("icenuc_num_meydep", scalar3d_layout_mid, m3_inv, @@ -927,36 +931,37 @@ void MAMAci::set_grids( // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); + constexpr int psat = mam4::ndrop::psat; + FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, + {ncol_, nlev_, psat}}; // number conc of aerosols activated at supersat [#/m^3] // NOTE: activation fraction fluxes are defined as // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - - constexpr int psat = mam4::ndrop::psat; - FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, - {ncol_, nlev_, psat}}; add_field("ccn", scalar4d_layout_psat_mid, m3_inv, grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable + constexpr int num_aero_const = mam4::ndrop::ncnst_tot; + FieldLayout scalar4d_layout_naero_const_mid{{COL, LEV, MAM_AERO_NCNST}, + {ncol_, nlev_, num_aero_const}}; + // column tendency for diagnostic output - add_field( - "coltend", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); + add_field("coltend", scalar4d_layout_naero_const_mid, nondim, + grid_name); - // BALLI:??? FIXME: This is internal diagnostic variable // column tendency - add_field( - "coltend_cw", - FieldLayout{{COL, LEV, CMP}, {ncol_, nlev_, mam4::ndrop::ncnst_tot}}, - nondim, grid_name); + add_field("coltend_cw", scalar4d_layout_naero_const_mid, nondim, + grid_name); - auto cm = m / 100; + const auto cm = m / 100; // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); + // ------------------------------------------------------------------------ + // Output from hetrozenous freezing + // ------------------------------------------------------------------------ + // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -968,16 +973,11 @@ void MAMAci::set_grids( // heterogeous freezing by deposition nucleation [cm^-3 s^-1] add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); - // - /* - * NOTE on other inputs for the aci process: - * 1. reciprocal of pseudo_density (rpdel): computed from the pseudo_density - * 2. geopotential height at midpoints: computed geopotential height at - * interfaces, which inturn is computed using pseudo_density, p_mid, T_mid and - * qv_mid (see dry_static_energy.cpp's "compute_diagnostic_impl" function). - * qv_mid can be obtained from "get_field_in" call - */ -} // set_grids ends +} // function set_grids ends + +// ================================================================ +// INIT_BUFFERS +// ================================================================ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( @@ -989,18 +989,59 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMMicrophysics."); -} +} // function init_buffers ends +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMAci::initialize_impl(const RunType run_type) { - w_sec_ = get_field_in("w_variance").get_view(); - - // MUST FIXME: is it an input, should we invoke calcsize here?? + // ------------------------------------------------------------------------ + // Input fields read in from IC file, namelist or other processes + // ------------------------------------------------------------------------ + w_sec_ = get_field_in("w_variance").get_view(); dgnum_ = get_field_in("dgnum").get_view(); liqcldf_ = get_field_in("cldfrac_liq").get_view(); liqcldf_prev_ = get_field_in("cldfrac_liq_prev").get_view(); + kvh_ = get_field_in("eddy_diff_heat").get_view(); + + // store fields only to be converted to dry mmrs in wet_atm_ + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + // store rest fo the atm fields in dry_atm_in + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - // MUST FIXME: This comes from shoc - kvh_ = get_field_in("eddy_diff_heat").get_view(); + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // pbl_height + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + + // geometric thickness of layers (m) + dry_atm_.dz = buffer_.dz; + + // geopotential height above surface at interface levels (m) + dry_atm_.z_iface = buffer_.z_iface; + + // geopotential height above surface at mid levels (m) + dry_atm_.z_mid = buffer_.z_mid; + + // total cloud fraction + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + + // computed updraft velocity + dry_atm_.w_updraft = buffer_.w_updraft; nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); @@ -1008,6 +1049,7 @@ void MAMAci::initialize_impl(const RunType run_type) { nimey_ = get_field_out("icenuc_num_meydep").get_view(); naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); + naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); @@ -1020,36 +1062,6 @@ void MAMAci::initialize_impl(const RunType run_type) { ccn_ = get_field_out("ccn").get_view(); coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); - - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_int = get_field_in("p_int").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; - - // pbl_height from SHOC - dry_atm_.pblh = get_field_in("pbl_height").get_view(); - - dry_atm_.dz = buffer_.dz; // geometric thickness of layers (m) - dry_atm_.z_iface = buffer_.z_iface; // geopotential height above - // surface at interface levels (m) - dry_atm_.z_mid = - buffer_.z_mid; // geopotential height above surface at mid levels (m) - - dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); - dry_atm_.w_updraft = buffer_.w_updraft; - hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = @@ -1057,14 +1069,6 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - // allocate work - // FIXME: store 25 into a const var or get from MAM4xx - for(int icnst = 0; icnst < 25; ++icnst) { - qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); - } - // FIXME :store 40 in a const int or get from MAM4xx - state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, 40); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { @@ -1089,9 +1093,6 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - std::cout << m << ":" << a << ":" << int_mmr_field_name << ":" - << dry_aero_.int_aero_mmr[m][a](0, 0) << ":" - << wet_aero_.int_aero_mmr[m][a](0, 0) << std::endl; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -1111,6 +1112,13 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Allocate work arrays + for(int icnst = 0; icnst < mam4::ndrop::ncnst_tot; ++icnst) { + qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); + } + state_q_work_ = + view_3d("state_q_work_", ncol_, nlev_, mam4::aero_model::pcnst); + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. @@ -1118,16 +1126,16 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); // MUST FIXME:Do we need this? + Kokkos::resize(ptend_q_[i], ncol_, nlev_); } for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { // MUST FIXME:store 2 in a const variable + for(int j = 0; j < 2; ++j) { Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); } } - for(int i = 0; i < 42; ++i) // MUST FIXME:store 42 in a const var + for(int i = 0; i < 42; ++i) Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); // nact : fractional aero. number activation rate [/s] @@ -1147,11 +1155,12 @@ void MAMAci::initialize_impl(const RunType run_type) { // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); -} +} // end function initialize_impl +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMAci::run_impl(const double dt) { - m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI run"); - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1162,8 +1171,6 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - // All the inputs are available to compute w0 and rho - // Convert from omega to w (vertical velocity) // Negative omega means rising motion // FIXME: Why wet_atm_ used here????? We should use dry_atm compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 79515420045d..7e6b0b83412a 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -36,6 +36,7 @@ enum class FieldTag { TimeLevel, num_modes, num_constituents, + num_aero_const, psat, // nrefindex_real, @@ -60,6 +61,7 @@ namespace ShortFieldTagsNames { constexpr auto CMP = FieldTag::Component; constexpr auto NMODES = FieldTag::num_modes; constexpr auto MAM_NCNST = FieldTag::num_constituents; + constexpr auto MAM_AERO_NCNST = FieldTag::num_aero_const; constexpr auto MAM_PSAT = FieldTag::psat; // constexpr auto NREFINDEX_REAL = FieldTag::nrefindex_real; @@ -103,6 +105,9 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::num_constituents: name = "num_constituents"; break; + case FieldTag::num_aero_const: + name = "num_aero_const"; + break; case FieldTag::psat: name = "psat"; break; From 9ce18afbe35907d3d10ca7c3d404990d6d7eeb76 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 9 Apr 2024 18:18:49 -0700 Subject: [PATCH 296/476] Moves oemga to DryAtmosphere --- .../mam/eamxx_mam_aci_process_interface.cpp | 14 +++++++------- .../eamxx_mam_microphysics_process_interface.cpp | 3 ++- .../mam/eamxx_mam_optics_process_interface.cpp | 3 ++- components/eamxx/src/physics/mam/mam_coupling.hpp | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b6dfbacc45d0..e6d0fb5f8f78 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -63,7 +63,7 @@ void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, const int top_lev, const int nlev) { // Get physical constants using C = physics::Constants; - MAMAci::const_view_2d omega = wet_atmosphere.omega; + MAMAci::const_view_2d omega = dry_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for( @@ -1005,18 +1005,18 @@ void MAMAci::initialize_impl(const RunType run_type) { kvh_ = get_field_in("eddy_diff_heat").get_view(); // store fields only to be converted to dry mmrs in wet_atm_ - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); // store rest fo the atm fields in dry_atm_in dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.qv = buffer_.qv_dry; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 8bc727c325e9..6ae368353025 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -201,7 +201,7 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { wet_atm_.nc = get_field_out("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); @@ -209,6 +209,7 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.dz = buffer_.dz; dry_atm_.z_iface = buffer_.z_iface; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index a55bf84ca17a..442a6500f215 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -140,7 +140,7 @@ void MAMOptics::initialize_impl(const RunType run_type) { wet_atm_.nc = get_field_in("nc").get_view(); wet_atm_.qi = get_field_in("qi").get_view(); wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); @@ -153,6 +153,7 @@ void MAMOptics::initialize_impl(const RunType run_type) { .get_view(); // FIXME: tot or liq? dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.z_mid = buffer_.z_mid; dry_atm_.dz = buffer_.dz; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 90b8a50ea489..6b5cfbd186d1 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -260,7 +260,6 @@ struct WetAtmosphere { const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] - const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores multi-column views related to the dry atmospheric state @@ -283,6 +282,7 @@ struct DryAtmosphere { view_2d w_updraft; // updraft velocity [m/s] const_view_1d pblh; // planetary boundary layer height [m] const_view_1d phis; // surface geopotential [m2/s2] + const_view_2d omega; // vertical pressure velocity [Pa/s] }; // This type stores aerosol number and mass mixing ratios evolved by MAM. It @@ -625,7 +625,7 @@ void compute_updraft_velocities(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { dry_atm.dz(i,k) = PF::calculate_dz(dry_atm.p_del(i,k), dry_atm.p_mid(i,k), dry_atm.T_mid(i,k), wet_atm.qv(i,k)); const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); - dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); + dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(dry_atm.omega(i,k), rho); }); } From 5c1c7a68ee1cf56a2f5953aca63589e59c722f04 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 10 Apr 2024 17:48:25 -0700 Subject: [PATCH 297/476] Fixes cld frac zero out bug and removes wet_atm_ from run_impl --- .../mam/eamxx_mam_aci_process_interface.cpp | 96 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- 2 files changed, 45 insertions(+), 55 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e6d0fb5f8f78..4a7f438e89f6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -58,7 +58,6 @@ void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, } void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, MAMAci::view_2d rho, - mam_coupling::WetAtmosphere &wet_atmosphere, mam_coupling::DryAtmosphere &dry_atmosphere, const int top_lev, const int nlev) { // Get physical constants @@ -296,48 +295,42 @@ void compute_nucleate_ice_tendencies( } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, MAMAci::const_view_2d qc, + const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, MAMAci::const_view_2d qc, MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, - MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev) { + MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, + const int nlev) { //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value - - // MUST FIXME NOTE: We need old and new liquid cloud fractions here. - // We have the new liquid cloud fraction (liq_strat_cld_frac) but we need to - // store the old (liq_strat_cld_frac_old) before we call SHOC. For now, we - // will make a note of it and use the new cloud fraction for the old cloud - // fraction. //------------------------------------------------------------- - static constexpr auto qsmall = - 1e-18; // cut-off for cloud amount (ice or liquid) + // cut-off for cloud amount (ice or liquid) + static constexpr auto qsmall = 1e-18; Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - const Real qcld = qc(icol, kk) + qi(icol, kk); - if(qcld > qsmall) { - cloud_frac_new(icol, kk) = liqcldf(icol, kk); - cloud_frac_old(icol, kk) = liqcldf_prev(icol, kk); + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + if((qc(icol, kk) + qi(icol, kk)) > qsmall) { + cloud_frac(icol, kk) = liqcldf(icol, kk); + cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); } else { - cloud_frac_new(icol, kk) = 0; - cloud_frac_old(icol, kk) = 0; + cloud_frac(icol, kk) = 0; + cloud_frac_prev(icol, kk) = 0; } }); } void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, - mam_coupling::WetAtmosphere &wet_atmosphere, + MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, + mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::const_view_2d liqcldf, MAMAci::const_view_2d liqcldf_prev, - const int top_lev) { - MAMAci::const_view_2d qc = wet_atmosphere.qc; - MAMAci::const_view_2d qi = wet_atmosphere.qi; + const int top_lev, const int nlev) { + MAMAci::const_view_2d qc = dry_atmosphere.qc; + MAMAci::const_view_2d qi = dry_atmosphere.qi; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, cloud_frac_new, cloud_frac_old, qc, - qi, liqcldf, liqcldf_prev, icol, top_lev); + store_liquid_cloud_fraction(team, cloud_frac, cloud_frac_prev, qc, qi, + liqcldf, liqcldf_prev, icol, top_lev, nlev); }); } KOKKOS_INLINE_FUNCTION @@ -374,8 +367,8 @@ void call_function_dropmixnuc( MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, MAMAci::const_view_2d nc, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, - MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac_new, - MAMAci::view_2d cloud_frac_old, MAMAci::view_2d tendnd, + MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, MAMAci::view_3d mact, @@ -519,14 +512,14 @@ void call_function_dropmixnuc( icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] - ekat::subview(cloud_frac_new, icol), lspectype_amode, - specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, + ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, + spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, alogsig, aten, mam_idx, mam_cnst_idx, - ekat::subview(qcld, icol), // out - ekat::subview(wsub, icol), // in - ekat::subview(cloud_frac_old, icol), // in - qqcw_view, // inout + ekat::subview(qcld, icol), // out + ekat::subview(wsub, icol), // in + ekat::subview(cloud_frac_prev, icol), // in + qqcw_view, // inout ptend_q_view, ekat::subview(tendnd, icol), ekat::subview(factnum, icol), ekat::subview(ndropcol, icol), ekat::subview(ndropmix, icol), ekat::subview(nsource, icol), @@ -575,7 +568,6 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::AerosolState &dry_aero_, - mam_coupling::WetAtmosphere &wet_atm_, mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_3d factnum_, MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, @@ -587,7 +579,6 @@ void call_hetfrz_compute_tendencies( mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; - mam_coupling::WetAtmosphere wet_atmosphere = wet_atm_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; @@ -714,8 +705,8 @@ void MAMAci::set_grids( Kokkos::resize(wsubice_, ncol_, nlev_); Kokkos::resize(wsig_, ncol_, nlev_); Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(cloud_frac_new_, ncol_, nlev_); - Kokkos::resize(cloud_frac_old_, ncol_, nlev_); + Kokkos::resize(cloud_frac_, ncol_, nlev_); + Kokkos::resize(cloud_frac_prev_, ncol_, nlev_); Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); @@ -1171,10 +1162,8 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - // Negative omega means rising motion - // FIXME: Why wet_atm_ used here????? We should use dry_atm - compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, wet_atm_, - dry_atm_, top_lev_, nlev_); + compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, dry_atm_, + top_lev_, nlev_); compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); @@ -1193,9 +1182,10 @@ void MAMAci::run_impl(const double dt) { compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atm_, aitken_dry_dia_, nlev_, dt); - // FIXME: Why wet_atm_ used here????? We should use dry_atm - store_liquid_cloud_fraction(team_policy, cloud_frac_new_, cloud_frac_old_, - wet_atm_, liqcldf_, liqcldf_prev_, top_lev_); + + store_liquid_cloud_fraction(team_policy, cloud_frac_, cloud_frac_prev_, + dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, + nlev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1221,7 +1211,7 @@ void MAMAci::run_impl(const double dt) { team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atm_.p_int, dry_atm_.p_del, rpdel_, state_q_work_, dry_atm_.nc, kvh_, qcld_, wsub_, - cloud_frac_new_, cloud_frac_old_, tendnd_, factnum_, ndropcol_, ndropmix_, + cloud_frac_, cloud_frac_prev_, tendnd_, factnum_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. @@ -1231,12 +1221,12 @@ void MAMAci::run_impl(const double dt) { team_policy, coltend_outp_, coltend_, nlev_); copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - // FIXME: Why wet_atm_ used here????? We should use dry_atm - call_hetfrz_compute_tendencies( - team_policy, hetfrz_, dry_aero_, wet_atm_, dry_atm_, factnum_, - hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, naai_hom_, naai_, diagnostic_scratch_, - nlev_, dt); + + call_hetfrz_compute_tendencies(team_policy, hetfrz_, dry_aero_, dry_atm_, + factnum_, hetfrz_immersion_nucleation_tend_, + hetfrz_contact_nucleation_tend_, + hetfrz_depostion_nucleation_tend_, naai_hom_, + naai_, diagnostic_scratch_, nlev_, dt); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 6873ccd33975..27342467bc77 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -68,8 +68,8 @@ class MAMAci final : public scream::AtmosphereProcess { const_view_2d liqcldf_prev_; const_view_2d kvh_; - view_2d cloud_frac_new_; - view_2d cloud_frac_old_; + view_2d cloud_frac_; + view_2d cloud_frac_prev_; view_2d qcld_; view_2d tendnd_; // ptend_q_ is just ptend_q_output_ reformatted. From 4ba9252a0dd70bf92692ca16fb705bbb0bb3ce80 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 10 Apr 2024 18:00:09 -0700 Subject: [PATCH 298/476] Reaaranges args for store_liquid_cloud_fraction and add const --- .../mam/eamxx_mam_aci_process_interface.cpp | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4a7f438e89f6..1eb5452219de 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -295,11 +295,12 @@ void compute_nucleate_ice_tendencies( } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, MAMAci::const_view_2d qc, - MAMAci::const_view_2d qi, MAMAci::const_view_2d liqcldf, - MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, - const int nlev) { + const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, + const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { //------------------------------------------------------------- // Get old and new liquid cloud fractions when amount of cloud // is above qsmall threshold value @@ -317,20 +318,22 @@ void store_liquid_cloud_fraction( } }); } -void store_liquid_cloud_fraction(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, - mam_coupling::DryAtmosphere &dry_atmosphere, - MAMAci::const_view_2d liqcldf, - MAMAci::const_view_2d liqcldf_prev, - const int top_lev, const int nlev) { +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int top_lev, const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { MAMAci::const_view_2d qc = dry_atmosphere.qc; MAMAci::const_view_2d qi = dry_atmosphere.qi; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, cloud_frac, cloud_frac_prev, qc, qi, - liqcldf, liqcldf_prev, icol, top_lev, nlev); + store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, + top_lev, nlev, + // output + cloud_frac, cloud_frac_prev); }); } KOKKOS_INLINE_FUNCTION @@ -1183,9 +1186,10 @@ void MAMAci::run_impl(const double dt) { nidep_, nimey_, naai_hom_, naai_, dry_aero_, dry_atm_, aitken_dry_dia_, nlev_, dt); - store_liquid_cloud_fraction(team_policy, cloud_frac_, cloud_frac_prev_, - dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, - nlev_); + store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, + top_lev_, nlev_, + // output + cloud_frac_, cloud_frac_prev_); // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- From 1c3f93076dac66f0fadba8197c5a8b2faf6d7fff Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 13:27:13 -0700 Subject: [PATCH 299/476] Fixes k loop range, adds wsubmin as input for nuc ice and re-orders in/out of funcs --- .../mam/eamxx_mam_aci_process_interface.cpp | 267 ++++++++++-------- 1 file changed, 148 insertions(+), 119 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 1eb5452219de..875f3bec8831 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -35,16 +35,18 @@ void copy_scream_array_to_mam4xx( } KOKKOS_INLINE_FUNCTION -void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, - MAMAci::view_2d rho, MAMAci::const_view_2d omega, - MAMAci::const_view_2d T_mid, - MAMAci::const_view_2d p_mid, const int icol, - const int top_lev, const int nlev) { +void compute_w0_and_rho(const haero::ThreadTeam &team, + const MAMAci::const_view_2d omega, + const MAMAci::const_view_2d T_mid, + const MAMAci::const_view_2d p_mid, const int icol, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { // Get physical constants using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - static constexpr auto rair = - C::Rair; // Gas constant for dry air [J/(kg*K) or J/Kg/K] + // Gas constant for dry air [J/(kg*K) or J/Kg/K] + static constexpr auto rair = C::Rair; Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { w0(icol, kk) = 0; @@ -56,27 +58,29 @@ void compute_w0_and_rho(const haero::ThreadTeam &team, MAMAci::view_2d w0, w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); }); } -void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w0, - MAMAci::view_2d rho, - mam_coupling::DryAtmosphere &dry_atmosphere, - const int top_lev, const int nlev) { - // Get physical constants - using C = physics::Constants; +void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { MAMAci::const_view_2d omega = dry_atmosphere.omega; MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_w0_and_rho(team, w0, rho, omega, T_mid, p_mid, icol, top_lev, - nlev); + compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, + // output + w0, rho); }); } KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec(const haero::ThreadTeam &team, MAMAci::view_2d tke, - MAMAci::const_view_2d w_sec, const int icol, - const int nlev) { +void compute_tke_using_w_sec(const haero::ThreadTeam &team, + const MAMAci::const_view_2d w_sec, const int icol, + const int nlev, + // output + MAMAci::view_2d tke) { // FIXME Is this the correct boundary condition for tke at the surface? // TKE seems to be at interfaces but w_sec is at cell centers so this // descrepensy needs to be worked out. @@ -85,19 +89,23 @@ void compute_tke_using_w_sec(const haero::ThreadTeam &team, MAMAci::view_2d tke, KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); } void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d tke, MAMAci::const_view_2d w_sec, - const int nlev) { + const MAMAci::const_view_2d w_sec, const int nlev, + // output + MAMAci::view_2d tke) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_tke_using_w_sec(team, tke, w_sec, icol, nlev); + compute_tke_using_w_sec(team, w_sec, icol, nlev, + // output + tke); }); } KOKKOS_INLINE_FUNCTION void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, - const Real wsubmin, const int icol, const int top_lev, const int nlev) { + const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, + const Real wsubmin, const int icol, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. Kokkos::parallel_for( @@ -118,14 +126,17 @@ void compute_subgrid_scale_velocities( }); } void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, MAMAci::view_2d wsub, - MAMAci::view_2d wsubice, MAMAci::view_2d wsig, MAMAci::const_view_2d tke, - const Real wsubmin, const int top_lev, const int nlev) { + haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, + const Real wsubmin, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, wsub, wsubice, wsig, tke, - wsubmin, icol, top_lev, nlev); + compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, + nlev, + // output + wsub, wsubice, wsig); }); } @@ -183,115 +194,124 @@ Real subgrid_mean_updraft(const Real w0, const Real wsig) { } KOKKOS_INLINE_FUNCTION void compute_subgrid_mean_updraft_velocities(const haero::ThreadTeam &team, - MAMAci::view_2d w2, - MAMAci::const_view_2d w0, - MAMAci::const_view_2d wsig, - const int icol, const int nlev) { + const MAMAci::const_view_2d w0, + const MAMAci::const_view_2d wsig, + const int icol, const int nlev, + // output + MAMAci::view_2d w2) { Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { w2(icol, kk) = subgrid_mean_updraft(w0(icol, kk), wsig(icol, kk)); }); } void compute_subgrid_mean_updraft_velocities( - haero::ThreadTeamPolicy team_policy, MAMAci::view_2d w2, - MAMAci::const_view_2d w0, MAMAci::const_view_2d wsig, const int nlev) { + haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d w0, + const MAMAci::const_view_2d wsig, const int nlev, + // output + MAMAci::view_2d w2) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_subgrid_mean_updraft_velocities(team, w2, w0, wsig, icol, nlev); + compute_subgrid_mean_updraft_velocities(team, w0, wsig, icol, nlev, + // output + w2); }); } KOKKOS_INLINE_FUNCTION void compute_aitken_dry_diameter(const haero::ThreadTeam &team, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, const int icol, - const int top_lev) { + const MAMAci::const_view_3d dgnum, + const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); }); } void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d aitken_dry_dia, - MAMAci::const_view_3d dgnum, - const int top_lev) { + const MAMAci::const_view_3d dgnum, + const int top_lev, const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, aitken_dry_dia, dgnum, icol, top_lev); + compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, + // output + aitken_dry_dia); }); } void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const mam_coupling::AerosolState &dry_aerosol_state, + const MAMAci::view_2d wsubice, const MAMAci::view_2d aitken_dry_dia, + const int nlev, const double dt, + // output MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, - MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, - mam_coupling::AerosolState &dry_aerosol_state, - mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d aitken_dry_dia, - const int nlev, const double dt) { + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation //------------------------------------------------------------- - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d qv_dry = dry_atmosphere.qv; - MAMAci::const_view_2d cldfrac = dry_atmosphere.cldfrac; - MAMAci::const_view_2d w_updraft = dry_atmosphere.w_updraft; - using view_1d = typename KokkosTypes::template view_1d; - view_1d dummy("DummyView", nlev); - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - // Set up an atmosphere, surface, diagnostics, pronostics and tendencies - // class. - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - - // nucleation doesn't use any diagnostics, so it's okay to leave this - // alone for now - mam4::Diagnostics diags(nlev); - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = - ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai filds are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0; - /* - NOTE:"state_q" is a combination of subset of tracers added by - "int_mmr_field_name" and "int_nmr_field_name". Only output we care - about is "naai", "naai_hom" is never used anywhere - - Fortran code: - call nucleate_ice_cam_calc(ncol, lchnk, temperature, state_q, pmid, & - ! input rho, wsubice, strat_cld_frac, dgnum, & ! input naai, - naai_hom) ! output - */ - nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, - surf, progs, diags, tends); - }); + // Kokkos::parallel_for( + // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + // const int icol = team.league_rank(); + for(int icol = 0; icol < 218; ++icol) { + //--------------------------------------------------------------------- + // Set up surface, pronostics atmosphere, diagnostics, and tendencies + // classes. + //--------------------------------------------------------------------- + + // For compute_tendecies interface only, this structure is empty + haero::Surface surf{}; + + // Store interstitial and cld borne aerosols in "progrs" struture + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + + // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + + // Update the updraft velocity needed by nucleation to be "wsubice" + // in the haero_atm object + haero_atm.updraft_vel_ice_nucleation = ekat::subview(wsubice, icol); + + // All the output from this process is diagnotics; creates "diags" with + // nlev column length + mam4::Diagnostics diags(nlev); + + // Aitken mode index + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); // not used + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, + surf, progs, diags, tends); + //}); + } } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( @@ -577,9 +597,6 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], const int nlev, const double dt) { - using view_1d = typename KokkosTypes::template view_1d; - view_1d dummy("DummyView", nlev); - mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; @@ -1165,26 +1182,38 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - compute_w0_and_rho(team_policy, w0_ /*output*/, rho_ /*output*/, dry_atm_, - top_lev_, nlev_); + compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, + // output + w0_, rho_); - compute_tke_using_w_sec(team_policy, tke_ /*output*/, w_sec_, nlev_); + compute_tke_using_w_sec(team_policy, w_sec_, nlev_, + // output + tke_); Kokkos::fence(); // wait for for tke_ to be computed. - compute_subgrid_scale_velocities(team_policy, wsub_, wsubice_, wsig_, tke_, - wsubmin, top_lev_, nlev_); + compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, + // output + wsub_, wsubice_, wsig_); Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w2_, w0_, wsig_, nlev_); + compute_subgrid_mean_updraft_velocities(team_policy, w0_, wsig_, nlev_, + // output + w2_); - compute_aitken_dry_diameter(team_policy, aitken_dry_dia_, dgnum_, top_lev_); + compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, + // output + aitken_dry_dia_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - // FIXME: Find out in-outs of the following call! - compute_nucleate_ice_tendencies(nucleate_ice_, team_policy, nihf_, niim_, - nidep_, nimey_, naai_hom_, naai_, dry_aero_, - dry_atm_, aitken_dry_dia_, nlev_, dt); + // Compute Ice nucleation + // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent + // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct + compute_nucleate_ice_tendencies( + nucleate_ice_, team_policy, dry_atm_, dry_aero_, wsubice_, + aitken_dry_dia_, nlev_, dt, + // output + nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, From b4d191d8b29a47beb5a44de024d008837bb593e0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 13:51:18 -0700 Subject: [PATCH 300/476] Fixes in/outs of store_liquid_cloud_fraction --- .../mam/eamxx_mam_aci_process_interface.cpp | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 875f3bec8831..d583a37d894a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -358,22 +358,26 @@ void store_liquid_cloud_fraction( } KOKKOS_INLINE_FUNCTION void compute_recipical_pseudo_density(const haero::ThreadTeam &team, - MAMAci::view_2d rpdel, - MAMAci::const_view_2d pdel, - const int icol, const int nlev) { + const MAMAci::const_view_2d pdel, + const int icol, const int nlev, + // output + MAMAci::view_2d rpdel) { // FIXME: Add an assert to ensure pdel is non-zero Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); } void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d rpdel, MAMAci::const_view_2d pdel, - const int nlev) { + const int nlev, + // output + MAMAci::view_2d rpdel) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, rpdel, pdel, icol, nlev); + compute_recipical_pseudo_density(team, pdel, icol, nlev, + // output + rpdel); }); } @@ -1237,8 +1241,16 @@ void MAMAci::run_impl(const double dt) { rho(:ncol,:) enddo */ - compute_recipical_pseudo_density(team_policy, rpdel_, dry_atm_.p_del, nlev_); + compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, + // output + rpdel_); + Kokkos::fence(); // wait for rpdel_ to be computed. + /*call dropmixnuc(lchnk, ncol, deltatin, temperature, pmid, pint, pdel, rpdel, + zm, & ! in + state_q, nc, kvh, wsub, lcldn, lcldo, & ! in + qqcw, & ! inout + ptend, nctend_mixnuc, factnum) !out*/ call_function_dropmixnuc( team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, From 449af2c98652c0cfcbb374f940db29697551c492 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 14:25:54 -0700 Subject: [PATCH 301/476] Removes atm vars from the dropmixnuc arg list --- .../mam/eamxx_mam_aci_process_interface.cpp | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d583a37d894a..475f131cd2ea 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -390,11 +390,9 @@ void call_function_dropmixnuc( MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], - MAMAci::const_view_2d p_int, MAMAci::const_view_2d pdel, - MAMAci::view_2d rpdel, MAMAci::view_3d state_q_work_, - MAMAci::const_view_2d nc, MAMAci::const_view_2d kvh, MAMAci::view_2d qcld, - MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d rpdel, + MAMAci::view_3d state_q_work_, MAMAci::const_view_2d kvh, + MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev, MAMAci::view_2d tendnd, MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, @@ -405,6 +403,9 @@ void call_function_dropmixnuc( MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; MAMAci::const_view_2d zm = dry_atmosphere.z_mid; + MAMAci::const_view_2d pdel = dry_atmosphere.p_del; + MAMAci::const_view_2d p_int = dry_atmosphere.p_int; + MAMAci::const_view_2d nc = dry_atmosphere.nc; MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; @@ -1250,14 +1251,14 @@ void MAMAci::run_impl(const double dt) { zm, & ! in state_q, nc, kvh, wsub, lcldn, lcldo, & ! in qqcw, & ! inout - ptend, nctend_mixnuc, factnum) !out*/ - - call_function_dropmixnuc( - team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, raercol_, - qqcw_fld_work_, ptend_q_, coltend_, coltend_cw_, dry_atm_.p_int, - dry_atm_.p_del, rpdel_, state_q_work_, dry_atm_.nc, kvh_, qcld_, wsub_, - cloud_frac_, cloud_frac_prev_, tendnd_, factnum_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + ptend, nctend_mixnuc, factnum) !out*/ + + call_function_dropmixnuc(team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, + raercol_, qqcw_fld_work_, ptend_q_, coltend_, + coltend_cw_, rpdel_, state_q_work_, kvh_, qcld_, + wsub_, cloud_frac_, cloud_frac_prev_, tendnd_, + factnum_, ndropcol_, ndropmix_, nsource_, wtke_, + ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From e2c12786df6d94570a6a1d3be630bef709980883 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 15:47:35 -0700 Subject: [PATCH 302/476] Moves some i/o vars in dropmixnuc call, not done yet --- .../mam/eamxx_mam_aci_process_interface.cpp | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 475f131cd2ea..f6b14f71d70e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -382,22 +382,26 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, } void call_function_dropmixnuc( - haero::ThreadTeamPolicy team_policy, - mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aerosol_state, const Real dt, + haero::ThreadTeamPolicy team_policy, const Real dt, + mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d rpdel, + MAMAci::const_view_2d kvh, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, + MAMAci::view_2d cloud_frac_prev, + const mam_coupling::AerosolState &dry_aerosol_state, + + // output + MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, + MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + + // work arrays MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d rpdel, - MAMAci::view_3d state_q_work_, MAMAci::const_view_2d kvh, - MAMAci::view_2d qcld, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, MAMAci::view_2d tendnd, - MAMAci::view_3d factnum, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - MAMAci::view_3d nact, MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[15], const int nlev) { + MAMAci::view_3d state_q_work_, MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, + MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], + const int nlev) { // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; @@ -1253,12 +1257,14 @@ void MAMAci::run_impl(const double dt) { qqcw, & ! inout ptend, nctend_mixnuc, factnum) !out*/ - call_function_dropmixnuc(team_policy, dry_atm_, dry_aero_, dt, raercol_cw_, - raercol_, qqcw_fld_work_, ptend_q_, coltend_, - coltend_cw_, rpdel_, state_q_work_, kvh_, qcld_, - wsub_, cloud_frac_, cloud_frac_prev_, tendnd_, - factnum_, ndropcol_, ndropmix_, nsource_, wtke_, - ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + call_function_dropmixnuc( + team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, + cloud_frac_prev_, dry_aero_, + // output + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, + // work arrays + raercol_cw_, raercol_, state_q_work_, qcld_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From ab242612ac8ca3392c976c1242dfb86a4d7a5f5e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 16:01:41 -0700 Subject: [PATCH 303/476] Moves ndrop init functions outside of kokkos loop --- .../mam/eamxx_mam_aci_process_interface.cpp | 77 ++++++++++--------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f6b14f71d70e..637469d0678f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -386,7 +386,7 @@ void call_function_dropmixnuc( mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d rpdel, MAMAci::const_view_2d kvh, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev, - const mam_coupling::AerosolState &dry_aerosol_state, + const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, // output MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], @@ -400,8 +400,7 @@ void call_function_dropmixnuc( MAMAci::view_3d state_q_work_, MAMAci::view_2d qcld, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, - MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15], - const int nlev) { + MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15]) { // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; @@ -434,48 +433,56 @@ void call_function_dropmixnuc( MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; - for(int i = 0; i < mam4::ndrop::pver; ++i) - for(int j = 0; j < 2; ++j) loc_raercol_cw[i][j] = raercol_cw[i][j]; - for(int i = 0; i < mam4::ndrop::pver; ++i) - for(int j = 0; j < 2; ++j) loc_raercol[i][j] = raercol[i][j]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) loc_coltend[i] = coltend[i]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + loc_raercol_cw[i][j] = raercol_cw[i][j]; + loc_raercol[i][j] = raercol[i][j]; + } + } + + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + loc_coltend[i] = coltend[i]; loc_coltend_cw[i] = coltend_cw[i]; + } + + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; MAMAci::view_3d state_q_work_loc = state_q_work_; + //--------------------------------------------------------------------------- + // ## Initialize the ndrop class. + //--------------------------------------------------------------------------- + const int ntot_amode = mam_coupling::num_aero_modes(); + const int maxd_aspectype = mam4::ndrop::maxd_aspectype; + const int nspec_max = mam4::ndrop::nspec_max; + int nspec_amode[ntot_amode] = {}; + int lspectype_amode[maxd_aspectype][ntot_amode] = {}; + int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; + int numptr_amode[ntot_amode] = {}; + int mam_idx[ntot_amode][nspec_max] = {}; + int mam_cnst_idx[ntot_amode][nspec_max] = {}; + Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, + num2vol_ratio_min_nmodes[ntot_amode] = {}, + num2vol_ratio_max_nmodes[ntot_amode] = {}; + Real aten = 0; + mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, + numptr_amode, specdens_amode, spechygro, + mam_idx, mam_cnst_idx); + mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes); + //--------------------------------------------------------------------------- + // ## END (Initialize the ndrop class.) + //--------------------------------------------------------------------------- Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // Initialize the ndrop class. - const int ntot_amode = mam_coupling::num_aero_modes(); - const int maxd_aspectype = mam4::ndrop::maxd_aspectype; - const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode] = {}; - int lspectype_amode[maxd_aspectype][ntot_amode] = {}; - int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; - int numptr_amode[ntot_amode] = {}; - int mam_idx[ntot_amode][nspec_max] = {}; - int mam_cnst_idx[ntot_amode][nspec_max] = {}; - Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, - num2vol_ratio_min_nmodes[ntot_amode] = {}, - num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; - mam4::ndrop::get_e3sm_parameters( - nspec_amode, lspectype_amode, lmassptr_amode, numptr_amode, - specdens_amode, spechygro, mam_idx, mam_cnst_idx); - mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, - num2vol_ratio_min_nmodes, // voltonumbhi_amode - num2vol_ratio_max_nmodes); // voltonumblo_amode - mam4::ndrop::View1D raercol_cw_view[mam4::ndrop::pver][2]; mam4::ndrop::View1D raercol_view[mam4::ndrop::pver][2]; for(int i = 0; i < mam4::ndrop::pver; ++i) { @@ -1259,12 +1266,12 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc( team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, - cloud_frac_prev_, dry_aero_, + cloud_frac_prev_, dry_aero_, nlev_, // output qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, // work arrays raercol_cw_, raercol_, state_q_work_, qcld_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_, nlev_); + nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From 8c9685533a0c74b5783942fb341f21b1a430abe0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 18:16:53 -0700 Subject: [PATCH 304/476] Moves some variables into output and moves some code around in dropmixnuc --- .../mam/eamxx_mam_aci_process_interface.cpp | 105 ++++++++++-------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 637469d0678f..00addc2c5ea8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -383,24 +383,25 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, - mam_coupling::DryAtmosphere &dry_atmosphere, MAMAci::view_2d rpdel, - MAMAci::const_view_2d kvh, MAMAci::view_2d wsub, MAMAci::view_2d cloud_frac, - MAMAci::view_2d cloud_frac_prev, + mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, + const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, + const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, // output MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, // work arrays MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_3d state_q_work_, MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, - MAMAci::view_2d wtke, MAMAci::view_3d ccn, MAMAci::view_3d nact, - MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15]) { + MAMAci::view_3d state_q_work_, MAMAci::view_2d ndropmix, + MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[15]) { // FIXME: why can't we use MAMAci::dropmix_scratch_ above MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; @@ -410,22 +411,10 @@ void call_function_dropmixnuc( MAMAci::const_view_2d p_int = dry_atmosphere.p_int; MAMAci::const_view_2d nc = dry_atmosphere.nc; - MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; - MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; - MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; - MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; - MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; - MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; - MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; - MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; - MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; - MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; - MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; - MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; - MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; - MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; - MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; - + //---------------------------------------------------------------------- + // ## Declare local variables for class variables + //(FIXME: GPU hack, revisit this) + //---------------------------------------------------------------------- MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; @@ -452,6 +441,27 @@ void call_function_dropmixnuc( qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; MAMAci::view_3d state_q_work_loc = state_q_work_; + + //---------------------------------------------------------------------- + // ## Assign scratch memory for work variables + //---------------------------------------------------------------------- + + MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; + MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; + MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; + MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; + MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; + MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; + MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; + MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; + MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; + MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; + MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; + MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; + MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; + MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; + MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; + //--------------------------------------------------------------------------- // ## Initialize the ndrop class. //--------------------------------------------------------------------------- @@ -476,7 +486,6 @@ void call_function_dropmixnuc( mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes); //--------------------------------------------------------------------------- - // ## END (Initialize the ndrop class.) //--------------------------------------------------------------------------- Kokkos::parallel_for( @@ -506,34 +515,31 @@ void call_function_dropmixnuc( coltend_cw_view[i] = ekat::subview(loc_coltend_cw[i], icol); } - /* - NOTE: "deltain" is the model time step. "state_q" is a combination of - tracers fields with "int_mmr_field_name" and "int_nmr_field_name". - "z_mid" is computed. "qqcw" is the combination of cld_mmr_field_name and - cld_nmr_field_name. The output "ptend" will have tendencies for - interstitial and cloud borne aerosols. - - Fortan code: - call dropmixnuc(lchnk, ncol, deltatin, T_mid, p_mid, p_int, p_del, - rpdel, z_mid, & ! in state_q, nc, kvh, wsub, lcldn, lcldo, & ! in - qqcw, & ! inout - ptend, nctend_mixnuc, factnum) !out - */ + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + // get prognostics mam4::Prognostics progs_at_col = aerosols_for_column(dry_aerosol_state, icol); + + // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + // Construct state_q (interstitial) and qqcw (cloud borne) arrays for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = - {}; // use pcnst here - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // use pcnst here + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a gri cell (col,lev) mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); + // get qqcw at a gri cell (col,lev) mam4::utils::extract_qqcw_from_prognostics(progs_at_col, qqcw_at_lev_col, klev); + + // create colum views of state_q and qqcw for(int icnst = 15; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; // FIXME: ensure that indices are @@ -1264,14 +1270,15 @@ void MAMAci::run_impl(const double dt) { qqcw, & ! inout ptend, nctend_mixnuc, factnum) !out*/ - call_function_dropmixnuc( - team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, - cloud_frac_prev_, dry_aero_, nlev_, - // output - qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, - // work arrays - raercol_cw_, raercol_, state_q_work_, qcld_, ndropcol_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, dropmixnuc_scratch_mem_); + call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, + cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, + // output + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, + coltend_, coltend_cw_, qcld_, ndropcol_, + // work arrays + raercol_cw_, raercol_, state_q_work_, ndropmix_, + nsource_, wtke_, ccn_, nact_, mact_, + dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. copy_mam4xx_array_to_scream( From 5c221ab09f0e746f3a98547273c6f0c164d515ca Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 11 Apr 2024 18:28:39 -0700 Subject: [PATCH 305/476] Moves some more fields to output --- .../mam/eamxx_mam_aci_process_interface.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 00addc2c5ea8..bb419847e45f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -393,17 +393,17 @@ void call_function_dropmixnuc( MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, // work arrays MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], - MAMAci::view_2d raercol[mam4::ndrop::pver][2], - MAMAci::view_3d state_q_work_, MAMAci::view_2d ndropmix, - MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, + MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[15]) { - // FIXME: why can't we use MAMAci::dropmix_scratch_ above + // FIXME: why can't we use MAMAci::dropmix_scratch_ above instead of 15? + // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; MAMAci::const_view_2d zm = dry_atmosphere.z_mid; @@ -440,7 +440,7 @@ void call_function_dropmixnuc( for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; - MAMAci::view_3d state_q_work_loc = state_q_work_; + MAMAci::view_3d state_q_work_loc = state_q_work; //---------------------------------------------------------------------- // ## Assign scratch memory for work variables @@ -1274,10 +1274,10 @@ void MAMAci::run_impl(const double dt) { cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output qqcw_fld_work_, ptend_q_, factnum_, tendnd_, - coltend_, coltend_cw_, qcld_, ndropcol_, + coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, + nsource_, wtke_, ccn_, // work arrays - raercol_cw_, raercol_, state_q_work_, ndropmix_, - nsource_, wtke_, ccn_, nact_, mact_, + raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. From 9235c84948e5b6f484b55ee73fa574fcd9a448c5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Apr 2024 09:13:42 -0700 Subject: [PATCH 306/476] Adds wsubmin namelist and also a lot of debugging cout stamenents for Fortran verification --- .../cime_config/namelist_defaults_scream.xml | 4 + .../mam/eamxx_mam_aci_process_interface.cpp | 139 +++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 5 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 9 ++ .../tests/single-process/mam/aci/input.yaml | 15 +- 5 files changed, 129 insertions(+), 43 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 9a33879b01d8..8aa4c0ea6f74 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -234,6 +234,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.1 + + + 0.001 + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index bb419847e45f..158f88dcd9d2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -388,15 +388,26 @@ void call_function_dropmixnuc( const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, - // output - MAMAci::view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot], - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], MAMAci::view_3d factnum, - MAMAci::view_2d tendnd, MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + // ## outputs ## + // qqcw_fld_work should be directly assigned to the cloud borne aerosols + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + + // ptend_q are the tendencies to the interstitial aerosols + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + + // factnum is used by the hetrozenous freezing + MAMAci::view_3d factnum, + + // tendnd is used by microphysics scheme (e.g. P3) + MAMAci::view_2d tendnd, + + // Following outputs are all diagnostics + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // work arrays + // ## work arrays ## MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, @@ -471,11 +482,12 @@ void call_function_dropmixnuc( int nspec_amode[ntot_amode] = {}; int lspectype_amode[maxd_aspectype][ntot_amode] = {}; int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; int numptr_amode[ntot_amode] = {}; int mam_idx[ntot_amode][nspec_max] = {}; int mam_cnst_idx[ntot_amode][nspec_max] = {}; + + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, num2vol_ratio_min_nmodes[ntot_amode] = {}, num2vol_ratio_max_nmodes[ntot_amode] = {}; @@ -552,11 +564,9 @@ void call_function_dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), ekat::subview(pdel, icol), ekat::subview(rpdel, icol), - ekat::subview( - zm, - icol), // ! in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(state_q_work_loc, icol), ekat::subview(nc, icol), - ekat::subview(kvh, icol), // kvh[kk+1] + // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] + ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), + ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, @@ -612,23 +622,24 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, - mam_coupling::AerosolState &dry_aero_, - mam_coupling::DryAtmosphere &dry_atm_, MAMAci::view_3d factnum_, + mam_coupling::DryAtmosphere &dry_atm_, + mam_coupling::AerosolState &dry_aero_, MAMAci::view_3d factnum_, + const double dt, const int nlev, + // output MAMAci::view_2d hetfrz_immersion_nucleation_tend, MAMAci::view_2d hetfrz_contact_nucleation_tend, - MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d naai_hom, - MAMAci::view_2d naai, MAMAci::view_2d diagnostic_scratch_[], const int nlev, - const double dt) { + MAMAci::view_2d hetfrz_depostion_nucleation_tend, + MAMAci::view_2d diagnostic_scratch_[]) { mam4::Hetfrz hetfrz = hetfrz_; mam_coupling::AerosolState dry_aerosol_state = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; + MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for (int icol=0; icol<1; icol++){ // Set up an atmosphere, surface, diagnostics, pronostics and // tendencies class. @@ -641,7 +652,6 @@ void call_hetfrz_compute_tendencies( const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - // BALLI mam4::Diagnostics diags(nlev); diags.activation_fraction[accum_idx] = @@ -712,19 +722,19 @@ void call_hetfrz_compute_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0; //, dt = 0; + const Real t = 0; hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); - //} } } // namespace -// FIXME: The following variables are namelist variables -const Real wsubmin = 1; - MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) - : AtmosphereProcess(comm, params) {} + : AtmosphereProcess(comm, params) { + // Asserts for the runtime or namelist options + EKAT_REQUIRE_MSG(m_params.isParameter("wsubmin"), + "ERROR: wsubmin is missing from mam_optics parameter list."); +} // ================================================================ // SET_GRIDS @@ -1028,6 +1038,12 @@ void MAMAci::init_buffers(const ATMBufferManager &buffer_manager) { // INITIALIZE_IMPL // ================================================================ void MAMAci::initialize_impl(const RunType run_type) { + // ------------------------------------------------------------------------ + // ## Runtime options + // ------------------------------------------------------------------------ + + ACIRuntime_.wsubmin = m_params.get("wsubmin"); + // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes // ------------------------------------------------------------------------ @@ -1204,30 +1220,79 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + const int kb = 66; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); + // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; + // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; + std::cout << "wsec:" << w_sec_(0, kb) << std::endl; + std::cout << "w0:" << w0_(0, kb) << std::endl; + std::cout << " rho: " << rho_(0, kb) << std::endl; compute_tke_using_w_sec(team_policy, w_sec_, nlev_, // output tke_); + const Real tke_b[73] = { + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, + 0.00062854116332895, 0.00067291680645447, 0.00101664413803513, + 0.00232786652593969, 0.00569557505990274, 0.01000238209664803, + 0.01519680083369616, 0.02170550249412443, 0.02401690416836010, + 0.02366486534737443, 0.02153664341694223, 0.01578233182683715, + 0.01145733753135718, 0.01010923067444657, 0.01243858801725962, + 0.02323111869706078, 0.04796154961104696, 0.08572647178140329, + 0.12086080769994784, 0.14941194987050846, 0.15643944255461628, + 0.14719266313675605}; + for(int kk = 0; kk < 73; ++kk) tke_(0, kk) = tke_b[kk]; + + std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. - + const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); + std::cout << "WSUB:" << wsub_(0, kb) << std::endl; + std::cout << "WICE:" << wsubice_(0, kb) << std::endl; + std::cout << "WSIG:" << wsig_(0, kb) << std::endl; + Kokkos::fence(); // wait for wsig_ to be computed. compute_subgrid_mean_updraft_velocities(team_policy, w0_, wsig_, nlev_, // output w2_); + std::cout << "W2_:" << w2_(0, kb) << std::endl; + /* wo 9.784350381198358E-003 + rho: 1.07565908612486 + TKE: 2.323111869706078E-002 4.796154961104696E-002 + wsub: 0.154048334739574 + wsubi: 0.200000000000000 + wsig: 0.154048334739574 + w2: 6.571878680006606E-002*/ + compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct @@ -1264,11 +1329,6 @@ void MAMAci::run_impl(const double dt) { rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. - /*call dropmixnuc(lchnk, ncol, deltatin, temperature, pmid, pint, pdel, rpdel, - zm, & ! in - state_q, nc, kvh, wsub, lcldn, lcldo, & ! in - qqcw, & ! inout - ptend, nctend_mixnuc, factnum) !out*/ call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, @@ -1288,12 +1348,17 @@ void MAMAci::run_impl(const double dt) { copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - call_hetfrz_compute_tendencies(team_policy, hetfrz_, dry_aero_, dry_atm_, - factnum_, hetfrz_immersion_nucleation_tend_, - hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, naai_hom_, - naai_, diagnostic_scratch_, nlev_, dt); + /* call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, + & ! in qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in + frzimm, frzcnt, frzdep) */ + call_hetfrz_compute_tendencies( + team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, + // output + hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, + hetfrz_depostion_nucleation_tend_, + // work arrays + diagnostic_scratch_); Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 27342467bc77..3159fa3f0be9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -32,9 +32,12 @@ class MAMAci final : public scream::AtmosphereProcess { using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; - // FIXME:B: Should the following variables be public? They are like that in + // FIXME: Should the following variables be public? They are like that in // micriphysics and optics codes + // ACI runtime ( or namelist) options + mam_coupling::MAM4ACIRuntime ACIRuntime_; + // rho is air density [kg/m3] view_2d rho_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6b5cfbd186d1..3c01f4690926 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -252,6 +252,15 @@ const char* gas_mmr_field_name(const int gas) { return const_cast(gas_species_name(gas)); } + // This struct stores runtime options for MAM4xx-ACI + struct MAM4ACIRuntime { + MAM4ACIRuntime() = default; + // Runtime options for MAM4xx aerosol cloud interactions code + + Real wsubmin; //Minimum diagnostic sub-grid vertical velocity + }; + + // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index dc1b779875a8..4941b2d259b4 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -10,6 +10,8 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_aci] + mam4_aci: + wsubmin: 0.001 grids_manager: Type: Mesh Free @@ -30,11 +32,14 @@ initial_conditions: #we should get the following variables from other processes pbl_height : 1.0 - dgnum: 1e-3 - eddy_diff_heat: 1e-3 - cldfrac_liq: 1e-3 - cldfrac_liq_prev: 0.1 - w_variance: 1e-3 + p_mid: 86389.8729712384 + omega: -0.103206160508178 + T_mid: 279.796493114264 + dgnum: 1.326982247461267E-007 + eddy_diff_heat: 2.82379273906116 + cldfrac_liq: 0.999966505294370 + cldfrac_liq_prev: 0.999999418697814 + w_variance: 1.548741246470719E-002 # The parameters for I/O control Scorpio: From f179ffbbeca5acd0ceb4e1e9a7ca10e95db83ccb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 13 Apr 2024 09:22:06 -0700 Subject: [PATCH 307/476] Removes subgrid_mean_updraft velocities functions as they are not used --- .../mam/eamxx_mam_aci_process_interface.cpp | 84 +------------------ 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 158f88dcd9d2..45d5d9b0bee4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -140,83 +140,6 @@ void compute_subgrid_scale_velocities( }); } -KOKKOS_INLINE_FUNCTION -Real subgrid_mean_updraft(const Real w0, const Real wsig) { - /* --------------------------------------------------------------------------------- - Purpose: Calculate the mean updraft velocity inside a GCM grid assuming the - vertical velocity distribution is Gaussian and peaks at the - GCM resolved large-scale vertical velocity. - When icenul_wsub_scheme = 2, the model uses the mean updraft - velocity as the characteristic updraft velocity to calculate the ice - nucleation rate. Author: Kai Zhang (kai.zhang@pnnl.gov) Last Modified: Oct, - 2015 - -------------------------------------------------------------------------------- -*/ - - // interface - - // in :: wsig standard deviation (m/s) - // in :: w0 large scale vertical velocity (m/s) - // out:: mean updraft velocity(m/s) -> characteristic w* - - // FIXME should nbin be a user parameter? - const int nbin = 50; - - using C = physics::Constants; - constexpr Real pi = C::Pi; - Real zz[nbin], wa = 0, ww = 0; - int kp = 0; - - const Real sigma = haero::max(0.001, wsig); - const Real wlarge = w0; - - const Real xx = 6.0 * sigma / nbin; - - for(int ibin = 0; ibin < nbin; ++ibin) { - Real yy = wlarge - 3.0 * sigma + 0.5 * xx; - yy += (ibin - 1) * xx; - // wbar = integrator < w * f(w) * dw > - zz[ibin] = - yy * - haero::exp(-1.0 * haero::square(yy - wlarge) / (2 * sigma * sigma)) / - (sigma * haero::sqrt(2 * pi)) * xx; - } - for(int ibin = 0; ibin < nbin; ++ibin) { - if(zz[ibin] > 0) ++kp, wa += zz[ibin]; - } - if(kp) { - // wbar = integrator < w * f(w) * dw > - ww = wa; - } else { - ww = 0.001; - } - return ww; -} -KOKKOS_INLINE_FUNCTION -void compute_subgrid_mean_updraft_velocities(const haero::ThreadTeam &team, - const MAMAci::const_view_2d w0, - const MAMAci::const_view_2d wsig, - const int icol, const int nlev, - // output - MAMAci::view_2d w2) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - w2(icol, kk) = subgrid_mean_updraft(w0(icol, kk), wsig(icol, kk)); - }); -} -void compute_subgrid_mean_updraft_velocities( - haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d w0, - const MAMAci::const_view_2d wsig, const int nlev, - // output - MAMAci::view_2d w2) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_mean_updraft_velocities(team, w0, wsig, icol, nlev, - // output - w2); - }); -} KOKKOS_INLINE_FUNCTION void compute_aitken_dry_diameter(const haero::ThreadTeam &team, const MAMAci::const_view_3d dgnum, @@ -449,7 +372,7 @@ void call_function_dropmixnuc( MAMAci::view_2d qqcw_fld_work_loc[25]; for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - qqcw_fld_work_loc[i] = qqcw_fld_work_[i]; + qqcw_fld_work_loc[i] = qqcw_fld_work[i]; MAMAci::view_3d state_q_work_loc = state_q_work; @@ -1274,11 +1197,6 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for wsig_ to be computed. - compute_subgrid_mean_updraft_velocities(team_policy, w0_, wsig_, nlev_, - // output - w2_); - - std::cout << "W2_:" << w2_(0, kb) << std::endl; /* wo 9.784350381198358E-003 rho: 1.07565908612486 TKE: 2.323111869706078E-002 4.796154961104696E-002 From dbbd5a549a3a5ae867b1aeb5c40813ff8417ad03 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 07:07:35 -0700 Subject: [PATCH 308/476] One grid cell validated code with all debug statements --- .../mam/eamxx_mam_aci_process_interface.cpp | 198 ++++++++++++------ .../eamxx/src/physics/mam/mam_coupling.hpp | 20 +- .../single-process/mam/aci/CMakeLists.txt | 4 +- .../tests/single-process/mam/aci/input.yaml | 13 +- 4 files changed, 154 insertions(+), 81 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 45d5d9b0bee4..1d47fdcc5592 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -184,7 +184,9 @@ void compute_nucleate_ice_tendencies( // Kokkos::parallel_for( // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { // const int icol = team.league_rank(); - for(int icol = 0; icol < 218; ++icol) { + for(int icol = 0; icol < 5; ++icol) { + //std::cout<<""<("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -774,7 +779,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1133,84 +1138,69 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { + //for (int kk=0; kk<72; ++kk){ + //std::cout << "num_cb_accum_top:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + //KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); - const int kb = 66; + const int kb = 62; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); - // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; - // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; - std::cout << "wsec:" << w_sec_(0, kb) << std::endl; - std::cout << "w0:" << w0_(0, kb) << std::endl; - std::cout << " rho: " << rho_(0, kb) << std::endl; + //std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; + //std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; + //std::cout << "wsec:" << w_sec_(0, kb) << std::endl; + //std::cout << "w0:" << w0_(0, kb) << std::endl; + //std::cout << " rho: " << rho_(0, kb) << std::endl; compute_tke_using_w_sec(team_policy, w_sec_, nlev_, // output tke_); - const Real tke_b[73] = { - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00060000000000000, 0.00060000000000000, 0.00060000000000000, - 0.00062854116332895, 0.00067291680645447, 0.00101664413803513, - 0.00232786652593969, 0.00569557505990274, 0.01000238209664803, - 0.01519680083369616, 0.02170550249412443, 0.02401690416836010, - 0.02366486534737443, 0.02153664341694223, 0.01578233182683715, - 0.01145733753135718, 0.01010923067444657, 0.01243858801725962, - 0.02323111869706078, 0.04796154961104696, 0.08572647178140329, - 0.12086080769994784, 0.14941194987050846, 0.15643944255461628, - 0.14719266313675605}; - for(int kk = 0; kk < 73; ++kk) tke_(0, kk) = tke_b[kk]; - - std::cout << "TKE:" << tke_(0, kb) << std::endl; + //std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - std::cout << "WSUB:" << wsub_(0, kb) << std::endl; - std::cout << "WICE:" << wsubice_(0, kb) << std::endl; - std::cout << "WSIG:" << wsig_(0, kb) << std::endl; + //std::cout << "WSUB:" << wsub_(0, kb) << std::endl; + //std::cout << "WICE:" << wsubice_(0, kb) << std::endl; + //std::cout << "WSIG:" << wsig_(0, kb) << std::endl; Kokkos::fence(); // wait for wsig_ to be computed. - /* wo 9.784350381198358E-003 - rho: 1.07565908612486 - TKE: 2.323111869706078E-002 4.796154961104696E-002 - wsub: 0.154048334739574 - wsubi: 0.200000000000000 - wsig: 0.154048334739574 - w2: 6.571878680006606E-002*/ + /* wo 3.343177351577457E-004 + rho: 1.28186433384113 + TKE: 1.666465903934813E-002 6.097664104873541E-002 + wsub: 0.160873967324407 + wsubi: 0.200000000000000 + wsig: 0.160873967324407 + dgnum_ait: 4.362354500358337E-008 + naai: 11464.5957222634 + naai_hom: 29838.6879763933 + */ compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); + const Real dgnum_ait_e3sm[72] = { 0.20877713336487552E-007, 0.21782230353342090E-007, 0.21688324003865861E-007, 0.21112855042342451E-007, 0.19162058462939536E-007, 0.18102979880838476E-007, 0.17906980715477606E-007, 0.20271254074583327E-007, 0.22698983422181942E-007, 0.24134835117044986E-007, 0.25498156808001372E-007, 0.29796738799905547E-007, 0.35822987394021908E-007, 0.41170963764365215E-007, 0.44892726528330642E-007, 0.47217231342203107E-007, 0.48928661807108766E-007, 0.50170939816128735E-007, 0.51078750853732200E-007, 0.52247333465736065E-007, 0.53190758580174931E-007, 0.53576491941850044E-007, 0.53915614473890715E-007, 0.54510964775236826E-007, 0.55643231691556703E-007, 0.57057811112589899E-007, 0.58177383586181116E-007, 0.58209849180850108E-007, 0.57976751598840998E-007, 0.52000000000000002E-007, 0.50728746567226150E-007, 0.49119902704480870E-007, 0.48212162162050883E-007, 0.49227715213506454E-007, 0.46876827233752246E-007, 0.45360603896257791E-007, 0.49986783979004747E-007, 0.51186879246229022E-007, 0.50009353247048599E-007, 0.48250264542204811E-007, 0.47560278748093609E-007, 0.48298089720730957E-007, 0.49095935613468768E-007, 0.49493024126912931E-007, 0.50250797590476007E-007, 0.51949267668322422E-007, 0.53778727208416418E-007, 0.53563593301099588E-007, 0.51218136771199298E-007, 0.43171429694325200E-007, 0.39019610039033895E-007, 0.36175109143257051E-007, 0.42731638777892750E-007, 0.38060728507221777E-007, 0.44046323901481340E-007, 0.39216732751330010E-007, 0.34842233953609988E-007, 0.34068804733226066E-007, 0.30636043694263528E-007, 0.28302341686131413E-007, 0.33023014309036320E-007, 0.34745748365385196E-007, 0.43623545003583371E-007, 0.48206451795644064E-007, 0.49854490325455530E-007, 0.50346335647724146E-007, 0.50661560988561763E-007, 0.50986261962838767E-007, 0.51256955985111086E-007, 0.51482578449096488E-007, 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < 218; ++icol){ + for(int kk = 0; kk < 72; ++kk){ + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + }} Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; + //std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct @@ -1225,6 +1215,9 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); + //std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; + //std::cout << "naai_:" << naai_(0, kb) << std::endl; + // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- // Save cloud borne aerosols to be used in the heterozenous @@ -1245,6 +1238,7 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); + //std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; Kokkos::fence(); // wait for rpdel_ to be computed. @@ -1258,6 +1252,17 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. + + + //std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, kb)<<" : "<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1266,10 +1271,6 @@ void MAMAci::run_impl(const double dt) { copy_mam4xx_array_to_scream( team_policy, coltend_cw_outp_, coltend_cw_, nlev_); - /* call hetfrz_classnuc_cam_calc(ncol, lchnk, temperature, pmid, rho, ast, - & ! in qc, nc, state_q, aer_cb(:,:,:,lchnk_zb), deltatin, factnum, & ! in - frzimm, frzcnt, frzdep) */ - call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // output @@ -1277,6 +1278,10 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); + std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend_(0, kb)<< std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" << hetfrz_contact_nucleation_tend_(0, kb)<< std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" << hetfrz_depostion_nucleation_tend_(0, kb)<< std::endl; + Kokkos::fence(); // wait before returning to calling function } @@ -1285,3 +1290,78 @@ void MAMAci::finalize_impl() { } } // namespace scream + //tendnd: -839.404918218856 7289315.36108503 7792958.31201634 1.666666666666667E-003 + //factnum: 0.998556176544263 0.861689826773470 0.999999999974140 0.000000000000000E+000 + /*nctend_mixnuc: -839.404918218856 + O3_tend: 0.000000000000000E+000 + H2O2_tend: 0.000000000000000E+000 + H2SO4_tend: 0.000000000000000E+000 + SO2_tend: 0.000000000000000E+000 + DMS_tend: 0.000000000000000E+000 + SOAG_tend: 0.000000000000000E+000 + so4_a1_tend: -1.384310943569190E-014 + pom_a1_tend: -2.950799002682271E-015 + soa_a1_tend: -2.283869371271150E-014 + bc_a1_tend: -2.202672700280516E-016 + dst_a1_tend: -2.194060414083922E-016 + ncl_a1_tend: -6.981124745267083E-017 + mom_a1_tend: -5.662690535048673E-018 + num_a1_tend: -1626.38730532537 + so4_a2_tend: -2.548707408341951E-017 + soa_a2_tend: -1.929990276139139E-018 + ncl_a2_tend: -6.855087738119723E-019 + mom_a2_tend: -4.571691604185304E-020 + num_a2_tend: -111.890330245159 + dst_a3_tend: -7.609611333715698E-015 + ncl_a3_tend: -1.256350252705383E-015 + so4_a3_tend: -1.320988655880598E-016 + bc_a3_tend: -1.099586316024402E-018 + pom_a3_tend: -3.309939677824050E-018 + soa_a3_tend: -2.767668335981830E-017 + mom_a3_tend: -2.254911604324925E-019 + num_a3_tend: -0.406788905791186 + pom_a4_tend: -2.375070578317096E-017 + bc_a4_tend: 4.082479713528180E-017 + mom_a4_tend: -4.901356204764327E-023 + num_a4_tend: -34.4232851017138 + so4_c1: 1322558.40616450 + pom_c1: 1.233997159684093E-011 + soa_c1: 2.393397505793018E-012 + bc_c1: 1.872520079842212E-011 + dst_c1: 1.922157644491776E-013 + ncl_c1: 2.530385880698310E-013 + mom_c1: 8.315014074126540E-014 + num_c1: 6.895143982427237E-015 + so4_c2: 142214.467855250 + soa_c2: 3.181118913367564E-014 + ncl_c2: 2.996638282752921E-015 + mom_c2: 8.055333842809433E-016 + num_c2: 5.202632275847699E-017 + dst_c3: 464.524744048298 + ncl_c3: 8.814810290885256E-012 + so4_c3: 1.417415096039318E-012 + bc_c3: 1.284022582681906E-013 + pom_c3: 1.081226492056768E-015 + soa_c3: 3.292708392236485E-015 + mom_c3: 2.584587804061707E-014 + num_c3: 2.324887367507579E-016 + pom_c4: 6.934165370527951E-014 + bc_c4: 2.703924752644086E-026 + mom_c4: 3.684880329580097E-027 + num_c4: 1.049556651713384E-032 + + frzimm: 5.651860156917002E-006 4.477615226393836E-008 5.607084004653063E-006 + frzcnt: 0.000000000000000E+000 + frzdep: 0.000000000000000E+000 + + BC: 2.686569135836302E-003 6.811694223739704E-014* 4.327927689273498E-012* + 8.647238075040541E-013* 6.734058317716400E-012* 2.685810991448475E-014* + 8.239303776951524E-014* 2.218253512777307E-015**** 477478.884677930* + 1.000000000000000E-006* + + --BC: 2.686569135836302E-003 6.734058317716400E-012 477478.884677930 4.327927689273498E-012 8.647238075040541E-013 8.239303776951524E-014 6.811694223739704E-014 + 2.685810991448475E-014 1.000000000000000E-006 + + BC:0.00209583 : 6.81169e-14 : 4.32793e-12 : 8.64724e-13 : 6.73406e-12 : 2.68581e-14 : 8.23930e-14 : 2.21825e-15 : 372488. : 1.00000e-06 + + */ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 3c01f4690926..88b442d7c2f7 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -653,11 +653,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -678,21 +678,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);// PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 2b33c95a5174..95da48b7fa44 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,8 +12,8 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 4941b2d259b4..bedc9c583c2a 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -25,23 +25,16 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - p_mid: 86389.8729712384 - omega: -0.103206160508178 - T_mid: 279.796493114264 - dgnum: 1.326982247461267E-007 - eddy_diff_heat: 2.82379273906116 - cldfrac_liq: 0.999966505294370 - cldfrac_liq_prev: 0.999999418697814 - w_variance: 1.548741246470719E-002 + dgnum: 1e-3 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] -... +... \ No newline at end of file From 9a08cc66f4ea654e42958701c89d756fc465b538 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 12:15:24 -0700 Subject: [PATCH 309/476] Fakes interfaces values for kvh and wsec; fixes test and namelist wsubmin --- .../cime_config/namelist_defaults_scream.xml | 3 +- .../mam/eamxx_mam_aci_process_interface.cpp | 262 ++++++++++++------ .../mam/eamxx_mam_aci_process_interface.hpp | 10 +- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 2 + 4 files changed, 181 insertions(+), 96 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 8aa4c0ea6f74..2eda717ee06e 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -236,8 +236,9 @@ be lost if SCREAM_HACK_XML is not enabled. - 0.001 + 0.001 + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 1d47fdcc5592..53e72fc8dcb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -185,11 +185,11 @@ void compute_nucleate_ice_tendencies( // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { // const int icol = team.league_rank(); for(int icol = 0; icol < 5; ++icol) { - //std::cout<<""<("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -779,7 +779,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -861,8 +861,7 @@ void MAMAci::set_grids( n_unit, grid_name); // number of activated aerosol for ice nucleation[#/kg] - add_field("num_act_aerosol_ice_nucle", scalar3d_layout_mid, n_unit, - grid_name); + add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) @@ -975,11 +974,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes // ------------------------------------------------------------------------ - w_sec_ = get_field_in("w_variance").get_view(); + w_sec_mid_ = get_field_in("w_variance").get_view(); dgnum_ = get_field_in("dgnum").get_view(); liqcldf_ = get_field_in("cldfrac_liq").get_view(); liqcldf_prev_ = get_field_in("cldfrac_liq_prev").get_view(); - kvh_ = get_field_in("eddy_diff_heat").get_view(); + kvh_mid_ = get_field_in("eddy_diff_heat").get_view(); // store fields only to be converted to dry mmrs in wet_atm_ wet_atm_.qv = get_field_in("qv").get_view(); @@ -1027,8 +1026,8 @@ void MAMAci::initialize_impl(const RunType run_type) { naai_hom_ = get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); - naai_ = get_field_out("num_act_aerosol_ice_nucle").get_view(); - qcld_ = get_field_out("qcld").get_view(); + naai_ = get_field_out("ni_activated").get_view(); + qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); tendnd_ = get_field_out("tendnd").get_view(); factnum_ = get_field_out("factnum").get_view(); @@ -1116,10 +1115,17 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); // nact : fractional aero. number activation rate [/s] - // mact : fractional aero. mass activation rate [/s] Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // mact : fractional aero. mass activation rate [/s] Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + // Eddy diffusivity of heat at the interfaces + Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); + + // Vertical velocity variance at the interfaces + Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); + mam4::AeroConfig aero_config; // configure the nucleation parameterization mam4::NucleateIce::Config nucleate_ice_config; @@ -1138,11 +1144,13 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - //for (int kk=0; kk<72; ++kk){ - //std::cout << "num_cb_accum_top:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + // nlev_); KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of local derivied @@ -1150,60 +1158,94 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); const int kb = 62; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); - //std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; - //std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; - //std::cout << "wsec:" << w_sec_(0, kb) << std::endl; - //std::cout << "w0:" << w0_(0, kb) << std::endl; - //std::cout << " rho: " << rho_(0, kb) << std::endl; + // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; + // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; + // std::cout << "wsec:" << w_sec_(0, kb) << std::endl; + // std::cout << "w0:" << w0_(0, kb) << std::endl; + // std::cout << " rho: " << rho_(0, kb) << std::endl; + + // compute vertical velocity variance at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + } - compute_tke_using_w_sec(team_policy, w_sec_, nlev_, + compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); - //std::cout << "TKE:" << tke_(0, kb) << std::endl; + // std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - //std::cout << "WSUB:" << wsub_(0, kb) << std::endl; - //std::cout << "WICE:" << wsubice_(0, kb) << std::endl; - //std::cout << "WSIG:" << wsig_(0, kb) << std::endl; + // std::cout << "WSUB:" << wsub_(0, kb) << std::endl; + // std::cout << "WICE:" << wsubice_(0, kb) << std::endl; + // std::cout << "WSIG:" << wsig_(0, kb) << std::endl; Kokkos::fence(); // wait for wsig_ to be computed. - /* wo 3.343177351577457E-004 - rho: 1.28186433384113 - TKE: 1.666465903934813E-002 6.097664104873541E-002 - wsub: 0.160873967324407 - wsubi: 0.200000000000000 - wsig: 0.160873967324407 - dgnum_ait: 4.362354500358337E-008 - naai: 11464.5957222634 - naai_hom: 29838.6879763933 - */ - compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - const Real dgnum_ait_e3sm[72] = { 0.20877713336487552E-007, 0.21782230353342090E-007, 0.21688324003865861E-007, 0.21112855042342451E-007, 0.19162058462939536E-007, 0.18102979880838476E-007, 0.17906980715477606E-007, 0.20271254074583327E-007, 0.22698983422181942E-007, 0.24134835117044986E-007, 0.25498156808001372E-007, 0.29796738799905547E-007, 0.35822987394021908E-007, 0.41170963764365215E-007, 0.44892726528330642E-007, 0.47217231342203107E-007, 0.48928661807108766E-007, 0.50170939816128735E-007, 0.51078750853732200E-007, 0.52247333465736065E-007, 0.53190758580174931E-007, 0.53576491941850044E-007, 0.53915614473890715E-007, 0.54510964775236826E-007, 0.55643231691556703E-007, 0.57057811112589899E-007, 0.58177383586181116E-007, 0.58209849180850108E-007, 0.57976751598840998E-007, 0.52000000000000002E-007, 0.50728746567226150E-007, 0.49119902704480870E-007, 0.48212162162050883E-007, 0.49227715213506454E-007, 0.46876827233752246E-007, 0.45360603896257791E-007, 0.49986783979004747E-007, 0.51186879246229022E-007, 0.50009353247048599E-007, 0.48250264542204811E-007, 0.47560278748093609E-007, 0.48298089720730957E-007, 0.49095935613468768E-007, 0.49493024126912931E-007, 0.50250797590476007E-007, 0.51949267668322422E-007, 0.53778727208416418E-007, 0.53563593301099588E-007, 0.51218136771199298E-007, 0.43171429694325200E-007, 0.39019610039033895E-007, 0.36175109143257051E-007, 0.42731638777892750E-007, 0.38060728507221777E-007, 0.44046323901481340E-007, 0.39216732751330010E-007, 0.34842233953609988E-007, 0.34068804733226066E-007, 0.30636043694263528E-007, 0.28302341686131413E-007, 0.33023014309036320E-007, 0.34745748365385196E-007, 0.43623545003583371E-007, 0.48206451795644064E-007, 0.49854490325455530E-007, 0.50346335647724146E-007, 0.50661560988561763E-007, 0.50986261962838767E-007, 0.51256955985111086E-007, 0.51482578449096488E-007, 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < 218; ++icol){ - for(int kk = 0; kk < 72; ++kk){ - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - }} + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < 218; ++icol) { + for(int kk = 0; kk < 72; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - //std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; - // Compute Ice nucleation - // NOTE: The Fortran version uses "ast" for cloud fraction which is equivalent - // to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct + // std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; + // Compute Ice nucleation + // NOTE: The Fortran version uses "ast" for cloud fraction which is + // equivalent to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct compute_nucleate_ice_tendencies( nucleate_ice_, team_policy, dry_atm_, dry_aero_, wsubice_, aitken_dry_dia_, nlev_, dt, @@ -1215,8 +1257,8 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - //std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; - //std::cout << "naai_:" << naai_(0, kb) << std::endl; + // std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; + // std::cout << "naai_:" << naai_(0, kb) << std::endl; // MUST FIXME: save cloud borne aerosols here!!!! //------------------------------------------------------------- @@ -1238,11 +1280,19 @@ void MAMAci::run_impl(const double dt) { compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - //std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; + // std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; Kokkos::fence(); // wait for rpdel_ to be computed. - call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_, wsub_, + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + kvh_int_(icol, kk) = kvh_mid_(icol, kk); + } + kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + } + + call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output qqcw_fld_work_, ptend_q_, factnum_, tendnd_, @@ -1252,17 +1302,18 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - - //std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, kb)<<" : "<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1278,10 +1329,13 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); - std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend_(0, kb)<< std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" << hetfrz_contact_nucleation_tend_(0, kb)<< std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" << hetfrz_depostion_nucleation_tend_(0, kb)<< std::endl; - + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend_(0, kb) << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend_(0, kb) << std::endl; + Kokkos::fence(); // wait before returning to calling function } @@ -1290,9 +1344,45 @@ void MAMAci::finalize_impl() { } } // namespace scream - //tendnd: -839.404918218856 7289315.36108503 7792958.31201634 1.666666666666667E-003 - //factnum: 0.998556176544263 0.861689826773470 0.999999999974140 0.000000000000000E+000 - /*nctend_mixnuc: -839.404918218856 + +/* +FORTRAN OUTPUT: + + ----- OUPUT AT time step: 6 6 + wo 3.343177351577457E-004 + rho: 1.28186433384113 + TKE: 1.666465903934813E-002 6.097664104873541E-002 + wsub: 0.160873967324407 + wsubi: 0.200000000000000 + wsig: 0.160873967324407 + dgnum_ait: 4.362354500358337E-008 + naai: 11464.5957222634 + naai_hom: 29838.6879763933 + aer_cb: so4_c1 4.327927689273498E-012 so4_c1 4.327927689273498E-012 + aer_cb: bc_c1 6.811694223739704E-014 bc_c1 6.811694223739704E-014 + aer_cb: pom_c1 8.647238075040541E-013 pom_c1 8.647238075040541E-013 + aer_cb: soa_c1 6.734058317716400E-012 soa_c1 6.734058317716400E-012 + aer_cb: dst_c1 8.239303776951524E-014 dst_c1 8.239303776951524E-014 + aer_cb: ncl_c1 2.685810991448475E-014 ncl_c1 2.685810991448475E-014 + aer_cb: mom_c1 2.218253512777307E-015 mom_c1 2.218253512777307E-015 + aer_cb: num_c1 477478.884677930 num_c1 477478.884677930 + aer_cb: dst_c3 2.869197497896834E-012 dst_c3 2.869197497896834E-012 + aer_cb: ncl_c3 4.636857908110365E-013 ncl_c3 4.636857908110365E-013 + aer_cb: so4_c3 4.375223642985256E-014 so4_c3 4.375223642985256E-014 + aer_cb: bc_c3 3.674860666928688E-016 bc_c3 3.674860666928688E-016 + aer_cb: pom_c3 1.114802999982066E-015 pom_c3 1.114802999982066E-015 + aer_cb: soa_c3 8.915776993183282E-015 soa_c3 8.915776993183282E-015 + aer_cb: mom_c3 7.802642470782658E-017 mom_c3 7.802642470782658E-017 + aer_cb: num_c3 151.593971049442 num_c3 151.593971049442 + aer_cb: bc_c4 1.127495264613331E-027 bc_c4 1.127495264613331E-027 + aer_cb: pom_c4 8.204883715624573E-027 pom_c4 8.204883715624573E-027 + aer_cb: mom_c4 3.249069782816687E-033 mom_c4 3.249069782816687E-033 + aer_cb: num_c4 1.345885758586041E-013 num_c4 1.345885758586041E-013 + tendnd: -839.404918218856 7289315.36108503 7792958.31201634 + 1.666666666666667E-003 + factnum: 0.998556176544263 0.861689826773470 + 0.999999999974140 0.000000000000000E+000 + nctend_mixnuc: -839.404918218856 O3_tend: 0.000000000000000E+000 H2O2_tend: 0.000000000000000E+000 H2SO4_tend: 0.000000000000000E+000 @@ -1306,12 +1396,12 @@ void MAMAci::finalize_impl() { dst_a1_tend: -2.194060414083922E-016 ncl_a1_tend: -6.981124745267083E-017 mom_a1_tend: -5.662690535048673E-018 - num_a1_tend: -1626.38730532537 + num_a1_tend: -1626.38730532537 so4_a2_tend: -2.548707408341951E-017 soa_a2_tend: -1.929990276139139E-018 ncl_a2_tend: -6.855087738119723E-019 mom_a2_tend: -4.571691604185304E-020 - num_a2_tend: -111.890330245159 + num_a2_tend: -111.890330245159 dst_a3_tend: -7.609611333715698E-015 ncl_a3_tend: -1.256350252705383E-015 so4_a3_tend: -1.320988655880598E-016 @@ -1319,12 +1409,12 @@ void MAMAci::finalize_impl() { pom_a3_tend: -3.309939677824050E-018 soa_a3_tend: -2.767668335981830E-017 mom_a3_tend: -2.254911604324925E-019 - num_a3_tend: -0.406788905791186 + num_a3_tend: -0.406788905791186 pom_a4_tend: -2.375070578317096E-017 bc_a4_tend: 4.082479713528180E-017 mom_a4_tend: -4.901356204764327E-023 - num_a4_tend: -34.4232851017138 - so4_c1: 1322558.40616450 + num_a4_tend: -34.4232851017138 + so4_c1: 1322558.40616450 pom_c1: 1.233997159684093E-011 soa_c1: 2.393397505793018E-012 bc_c1: 1.872520079842212E-011 @@ -1332,12 +1422,12 @@ void MAMAci::finalize_impl() { ncl_c1: 2.530385880698310E-013 mom_c1: 8.315014074126540E-014 num_c1: 6.895143982427237E-015 - so4_c2: 142214.467855250 + so4_c2: 142214.467855250 soa_c2: 3.181118913367564E-014 ncl_c2: 2.996638282752921E-015 mom_c2: 8.055333842809433E-016 num_c2: 5.202632275847699E-017 - dst_c3: 464.524744048298 + dst_c3: 464.524744048298 ncl_c3: 8.814810290885256E-012 so4_c3: 1.417415096039318E-012 bc_c3: 1.284022582681906E-013 @@ -1349,19 +1439,9 @@ void MAMAci::finalize_impl() { bc_c4: 2.703924752644086E-026 mom_c4: 3.684880329580097E-027 num_c4: 1.049556651713384E-032 - - frzimm: 5.651860156917002E-006 4.477615226393836E-008 5.607084004653063E-006 + + frzimm: 5.651860156917002E-006 frzcnt: 0.000000000000000E+000 frzdep: 0.000000000000000E+000 - BC: 2.686569135836302E-003 6.811694223739704E-014* 4.327927689273498E-012* - 8.647238075040541E-013* 6.734058317716400E-012* 2.685810991448475E-014* - 8.239303776951524E-014* 2.218253512777307E-015**** 477478.884677930* - 1.000000000000000E-006* - - --BC: 2.686569135836302E-003 6.734058317716400E-012 477478.884677930 4.327927689273498E-012 8.647238075040541E-013 8.239303776951524E-014 6.811694223739704E-014 - 2.685810991448475E-014 1.000000000000000E-006 - - BC:0.00209583 : 6.81169e-14 : 4.32793e-12 : 8.64724e-13 : 6.73406e-12 : 2.68581e-14 : 8.23930e-14 : 2.21825e-15 : 372488. : 1.00000e-06 - */ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 3159fa3f0be9..865cdcdc0bff 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -67,9 +67,10 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d nimey_; view_2d naai_hom_; view_2d naai_; + view_2d kvh_int_; // Eddy diffusivity of heat at the interfaces const_view_2d liqcldf_; const_view_2d liqcldf_prev_; - const_view_2d kvh_; + const_view_2d kvh_mid_; // Eddy diffusivity of heat at the midpoints view_2d cloud_frac_; view_2d cloud_frac_prev_; @@ -120,9 +121,10 @@ class MAMAci final : public scream::AtmosphereProcess { const int top_lev_ = 6; // local atmospheric state column variables - const_view_2d pdel_; // pressure thickess of layer [Pa] - view_2d rpdel_; // Inverse of pdel_ - const_view_2d w_sec_; // Vertical velocity variance + const_view_2d pdel_; // pressure thickess of layer [Pa] + view_2d rpdel_; // Inverse of pdel_ + const_view_2d w_sec_mid_; // Vertical velocity variance at midpoints + view_2d w_sec_int_; // Vertical velocity variance at interfaces // number of horizontal columns and vertical levels int ncol_, nlev_; diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index f41b961a4f5c..4a1c80ba7caf 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -32,6 +32,8 @@ atmosphere_processes: c_diag_3rd_mom: 7.0 Ckh: 0.1 Ckm: 0.1 + mam4_aci: + wsubmin: 0.001 grids_manager: Type: Mesh Free From fa3cc9189ed0784b5a94a2e1c11d99e1dc9ab3a0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 12:21:26 -0700 Subject: [PATCH 310/476] Moves all resize views codes in set_grid --- .../mam/eamxx_mam_aci_process_interface.cpp | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 53e72fc8dcb6..c4555877df84 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -693,6 +693,37 @@ void MAMAci::set_grids( for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + // These are temp arrays formatted like mam4xx wants. + // Not sure if there is a way to do this with scream. + Kokkos::resize(coltend_[i], ncol_, nlev_); + Kokkos::resize(coltend_cw_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + Kokkos::resize(ptend_q_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); + Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); + } + } + + for(int i = 0; i < 42; ++i) + Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); + + // nact : fractional aero. number activation rate [/s] + Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // mact : fractional aero. mass activation rate [/s] + Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // Eddy diffusivity of heat at the interfaces + Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); + + // Vertical velocity variance at the interfaces + Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); + // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -1095,37 +1126,6 @@ void MAMAci::initialize_impl(const RunType run_type) { state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, mam4::aero_model::pcnst); - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // These are temp arrays formatted like mam4xx wants. - // Not sure if there is a way to do this with scream. - Kokkos::resize(coltend_[i], ncol_, nlev_); - Kokkos::resize(coltend_cw_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); - Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); - } - } - - for(int i = 0; i < 42; ++i) - Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); - - // nact : fractional aero. number activation rate [/s] - Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // mact : fractional aero. mass activation rate [/s] - Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // Eddy diffusivity of heat at the interfaces - Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); - - // Vertical velocity variance at the interfaces - Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); - mam4::AeroConfig aero_config; // configure the nucleation parameterization mam4::NucleateIce::Config nucleate_ice_config; From 4d6ec50ba9399d79973e61ecd649a67603c5a0dd Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 19 Apr 2024 17:28:47 -0700 Subject: [PATCH 311/476] Connects ni_activated with P3 --- .../mam/eamxx_mam_aci_process_interface.cpp | 88 ++++++++++++++++++- .../shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 3 +- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 2 +- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index c4555877df84..6a175660f866 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1172,11 +1172,51 @@ void MAMAci::run_impl(const double dt) { // std::cout << " rho: " << rho_(0, kb) << std::endl; // compute vertical velocity variance at the interfaces + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; for(int icol = 0; icol < ncol_; ++icol) { for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); } - w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; } compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, @@ -1251,6 +1291,7 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_, nlev_, dt, // output nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); + std::cout << "naai:" << naai_(0, 62) << std::endl; store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, @@ -1284,12 +1325,51 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; // compute eddy diffusivity of heat at the interfaces for(int icol = 0; icol < ncol_; ++icol) { for(int kk = 0; kk < nlev_; ++kk) { - kvh_int_(icol, kk) = kvh_mid_(icol, kk); + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; } - kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; } call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt index 946d68cb7501..81eaa3ae3e4c 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -13,7 +13,8 @@ CreateADUnitTest(${TEST_BASE_NAME} # Set AD configurable options set (ATM_TIME_STEP 1800) -SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +#SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) # 1h 4h 24h set (RUN_T0 2021-10-12-45000) # Determine num subcycles needed to keep shoc dt<=300s diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 4a1c80ba7caf..5cc2fb8fa653 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -47,7 +47,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} #variable required for shoc From 4ea07a3b6ed15c1aa1fa70fc5cb1d4da35dcc3ce Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 09:59:25 -0700 Subject: [PATCH 312/476] nctend or tendnd is not connected with P3 --- .../mam/eamxx_mam_aci_process_interface.cpp | 134 ++++++++++-------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 6a175660f866..a4d7fc5c9209 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -181,62 +181,63 @@ void compute_nucleate_ice_tendencies( // from ice nucleation //------------------------------------------------------------- - // Kokkos::parallel_for( - // team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - // const int icol = team.league_rank(); - for(int icol = 0; icol < 5; ++icol) { - // std::cout<<""<(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = - ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); // not used - const mam4::AeroConfig aero_config; - const Real t = 0; // not used - nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, - surf, progs, diags, tends); - //}); - } + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // for(int icol = 0; icol < 5; ++icol) { + // std::cout<<""<(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); // not used + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, + surf, progs, diags, tends); + }); + //} } KOKKOS_INLINE_FUNCTION void store_liquid_cloud_fraction( @@ -906,7 +907,7 @@ void MAMAci::set_grids( add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("tendnd", scalar3d_layout_mid, n_unit / s, grid_name); + add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); // activation fraction for aerosol number [fraction] add_field("factnum", scalar4d_layout_mid, nondim, grid_name); @@ -1060,7 +1061,7 @@ void MAMAci::initialize_impl(const RunType run_type) { naai_ = get_field_out("ni_activated").get_view(); qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("tendnd").get_view(); + tendnd_ = get_field_out("nc_nuceat_tend").get_view(); factnum_ = get_field_out("factnum").get_view(); ndropcol_ = get_field_out("ndropcol").get_view(); ndropmix_ = get_field_out("ndropmix").get_view(); @@ -1275,8 +1276,8 @@ void MAMAci::run_impl(const double dt) { 0.50661560988561763E-007, 0.50986261962838767E-007, 0.51256955985111086E-007, 0.51482578449096488E-007, 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < 218; ++icol) { - for(int kk = 0; kk < 72; ++kk) { + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; } } @@ -1291,7 +1292,12 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_, nlev_, dt, // output nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); - std::cout << "naai:" << naai_(0, 62) << std::endl; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + naai_(icol, kk) =9876.0; + } + } store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, @@ -1382,7 +1388,11 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + tendnd_(icol, kk) =1.2345; + } + } // std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, // kb)<<" : "< Date: Sat, 20 Apr 2024 10:09:54 -0700 Subject: [PATCH 313/476] Adds a branched mam4xx submodule with ndrop validation changes --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 7b0be56f4c3f..0f80195e0ff3 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 7b0be56f4c3f164259c6a9d8a0d16d6f5b72f3ae +Subproject commit 0f80195e0ff30d00d9a9b651e63a4ab4e314a242 From e78203fe528debade7cfc1b54135717089bcf87d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 19:01:30 -0700 Subject: [PATCH 314/476] Removes ACIRuntime namelist struct and used wsubmin on its own --- .../mam/eamxx_mam_aci_process_interface.cpp | 139 +++++++++--------- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 9 -- 3 files changed, 74 insertions(+), 76 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index a4d7fc5c9209..a6eb50fe82e6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -671,60 +671,15 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) void MAMAci::set_grids( const std::shared_ptr grids_manager) { // set grid for all the inputs and outputs - grid_ = grids_manager->get_grid("Physics"); // Use physics grid - const auto &grid_name = grid_->name(); // Name of the grid + // use physics grid + grid_ = grids_manager->get_grid("Physics"); + + // Name of the grid + const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - // Allocate memory for the class members - // Kokkos::resize only works on host to allocates memory - Kokkos::resize(rho_, ncol_, nlev_); - Kokkos::resize(w0_, ncol_, nlev_); - Kokkos::resize(tke_, ncol_, nlev_ + 1); - Kokkos::resize(wsub_, ncol_, nlev_); - Kokkos::resize(wsubice_, ncol_, nlev_); - Kokkos::resize(wsig_, ncol_, nlev_); - Kokkos::resize(w2_, ncol_, nlev_); - Kokkos::resize(cloud_frac_, ncol_, nlev_); - Kokkos::resize(cloud_frac_prev_, ncol_, nlev_); - Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); - Kokkos::resize(rpdel_, ncol_, nlev_); - - for(int i = 0; i < dropmix_scratch_; ++i) { - Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // These are temp arrays formatted like mam4xx wants. - // Not sure if there is a way to do this with scream. - Kokkos::resize(coltend_[i], ncol_, nlev_); - Kokkos::resize(coltend_cw_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - Kokkos::resize(ptend_q_[i], ncol_, nlev_); - } - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); - Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); - } - } - - for(int i = 0; i < 42; ++i) - Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); - - // nact : fractional aero. number activation rate [/s] - Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // mact : fractional aero. mass activation rate [/s] - Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); - - // Eddy diffusivity of heat at the interfaces - Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); - - // Vertical velocity variance at the interfaces - Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); - // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -780,7 +735,7 @@ void MAMAci::set_grids( // planetary boundary layer height add_field("pbl_height", scalar2d_layout_col, m, grid_name); - // cloud fraction [nondimentional] computed by eamxx_cld_fraction_process + // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); auto m2 = m * m; @@ -789,7 +744,7 @@ void MAMAci::set_grids( s2.set_string("s^2"); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time - // step Since, we are using SE dycore, itim_old is 1 which is equivalent to + // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to // the current time step. For other dycores (such as EUL), it may be different // and we might need to revisit this. @@ -797,7 +752,7 @@ void MAMAci::set_grids( // SHOC provides it at the midpoints. Verify how it is being used. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -811,7 +766,8 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + // FIXME: It is at mid level in EAMxx, we need to compute it at the interfaces + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -907,13 +863,14 @@ void MAMAci::set_grids( add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); // tendency in droplet number mixing ratio [#/kg/s] - add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); + add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, + grid_name); // activation fraction for aerosol number [fraction] add_field("factnum", scalar4d_layout_mid, nondim, grid_name); // NOTE: Here is a series of internal dropmixnuc variables; - // maybe we should move them to diagnostics later + // maybe we should move them to diagnostics later (FIXME) // cloud droplet number mixing ratio [#/kg] add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); @@ -1001,7 +958,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // ## Runtime options // ------------------------------------------------------------------------ - ACIRuntime_.wsubmin = m_params.get("wsubmin"); + wsubmin_ = m_params.get("wsubmin"); // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes @@ -1120,6 +1077,54 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Allocate memory for the class members + // (Kokkos::resize only works on host to allocates memory) + Kokkos::resize(rho_, ncol_, nlev_); + Kokkos::resize(w0_, ncol_, nlev_); + Kokkos::resize(tke_, ncol_, nlev_ + 1); + Kokkos::resize(wsub_, ncol_, nlev_); + Kokkos::resize(wsubice_, ncol_, nlev_); + Kokkos::resize(wsig_, ncol_, nlev_); + Kokkos::resize(w2_, ncol_, nlev_); + Kokkos::resize(cloud_frac_, ncol_, nlev_); + Kokkos::resize(cloud_frac_prev_, ncol_, nlev_); + Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); + Kokkos::resize(rpdel_, ncol_, nlev_); + + for(int i = 0; i < dropmix_scratch_; ++i) { + Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + // These are temp arrays formatted like mam4xx wants. + // Not sure if there is a way to do this with scream. + // FIXME: Do we need these? + Kokkos::resize(coltend_[i], ncol_, nlev_); + Kokkos::resize(coltend_cw_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + Kokkos::resize(ptend_q_[i], ncol_, nlev_); + } + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + Kokkos::resize(raercol_cw_[i][j], ncol_, mam4::ndrop::ncnst_tot); + Kokkos::resize(raercol_[i][j], ncol_, mam4::ndrop::ncnst_tot); + } + } + + for(int i = 0; i < 42; ++i) + Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); + + // nact : fractional aero. number activation rate [/s] + Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // mact : fractional aero. mass activation rate [/s] + Kokkos::resize(mact_, ncol_, nlev_, mam_coupling::num_aero_modes()); + + // Eddy diffusivity of heat at the interfaces + Kokkos::resize(kvh_int_, ncol_, nlev_ + 1); + + // Vertical velocity variance at the interfaces + Kokkos::resize(w_sec_int_, ncol_, nlev_ + 1); // Allocate work arrays for(int icnst = 0; icnst < mam4::ndrop::ncnst_tot; ++icnst) { qqcw_fld_work_[icnst] = view_2d("qqcw_fld_work_", ncol_, nlev_); @@ -1226,8 +1231,7 @@ void MAMAci::run_impl(const double dt) { // std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. - const Real &wsubmin = ACIRuntime_.wsubmin; // runtime variable - compute_subgrid_scale_velocities(team_policy, tke_, wsubmin, top_lev_, nlev_, + compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); // std::cout << "WSUB:" << wsub_(0, kb) << std::endl; @@ -1295,7 +1299,7 @@ void MAMAci::run_impl(const double dt) { for(int icol = 0; icol < ncol_; ++icol) { for(int kk = 0; kk < nlev_; ++kk) { - naai_(icol, kk) =9876.0; + naai_(icol, kk) = 9876.0; } } @@ -1388,11 +1392,7 @@ void MAMAci::run_impl(const double dt) { raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - tendnd_(icol, kk) =1.2345; - } - } + // std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, // kb)<<" : "< 5.65186e-06) { + std::cout << "SOmethign changed!!!! :" + << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; + exit(1); + } std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; + << hetfrz_immersion_nucleation_tend_(0, kb) << ":" + << hetfrz_immersion_nucleation_tend_(0, 0) << std::endl; std::cout << "hetfrz_contact_nucleation_tend_:" << hetfrz_contact_nucleation_tend_(0, kb) << std::endl; std::cout << "hetfrz_depostion_nucleation_tend_:" diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 865cdcdc0bff..297078c87275 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -36,7 +36,7 @@ class MAMAci final : public scream::AtmosphereProcess { // micriphysics and optics codes // ACI runtime ( or namelist) options - mam_coupling::MAM4ACIRuntime ACIRuntime_; + Real wsubmin_; // rho is air density [kg/m3] view_2d rho_; diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 88b442d7c2f7..25cc9d5de9de 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -252,15 +252,6 @@ const char* gas_mmr_field_name(const int gas) { return const_cast(gas_species_name(gas)); } - // This struct stores runtime options for MAM4xx-ACI - struct MAM4ACIRuntime { - MAM4ACIRuntime() = default; - // Runtime options for MAM4xx aerosol cloud interactions code - - Real wsubmin; //Minimum diagnostic sub-grid vertical velocity - }; - - // This type stores multi-column views related specifically to the wet // atmospheric state used by EAMxx. struct WetAtmosphere { From e34a548a227f6544bd99f3b95b3858bbeba42639 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 20:36:58 -0700 Subject: [PATCH 315/476] Clean up and moves validation intput output in functions --- .../mam/eamxx_mam_aci_process_interface.cpp | 410 +++++++++--------- 1 file changed, 211 insertions(+), 199 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index a6eb50fe82e6..67184b7466d1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -3,6 +3,194 @@ namespace scream { namespace { + +void print_input(const Real t, const Real p, const Real wsec) { + std::cout << "T_mid:" << t << std::endl; + std::cout << "p_mid:" << p << std::endl; + std::cout << "wsec:" << wsec << std::endl; +} + +void print_output(const Real w0, const Real rho, const Real tke, + const Real wsub, const Real wice, const Real wsig, + const Real naai_hom, const Real naai, const Real rpdel, + MAMAci::view_3d factnum, const Real tendnd, + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + const Real hetfrz_immersion_nucleation_tend, + const Real hetfrz_contact_nucleation_tend, + const Real hetfrz_depostion_nucleation_tend, const int kb) { + std::cout << "w0:" << w0 << std::endl; + std::cout << " rho: " << rho << std::endl; + std::cout << "TKE:" << tke << std::endl; + std::cout << "WSUB:" << wsub << std::endl; + std::cout << "WICE:" << wice << std::endl; + std::cout << "WSIG:" << wsig << std::endl; + std::cout << "naai_hom_:" << naai_hom << std::endl; + std::cout << "naai_:" << naai << std::endl; + std::cout << "rpdel_:" << rpdel << std::endl; + std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) + << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) + << std::endl; + std::cout << "tendnd_:" << tendnd << std::endl; + for(int ic = 9; ic < 40; ++ic) { + std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; + } + for(int ic = 0; ic < 25; ++ic) { + std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) + << std::endl; + } + + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend << ":" + << hetfrz_immersion_nucleation_tend << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend << std::endl; +} + +void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, + const int ncol_, const int nlev_) { + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; + } + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; + } + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; + } +} +void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, + const int nlev_) { + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } +} + KOKKOS_INLINE_FUNCTION void copy_scream_array_to_mam4xx(const haero::ThreadTeam &team, const MAMAci::view_2d mam4xx_view, @@ -1150,10 +1338,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - // for (int kk=0; kk<72; ++kk){ - // std::cout << "num_cb_accum_top:"<::get_thread_range_parallel_scan_team_policy(ncol_, // nlev_); @@ -1167,127 +1356,29 @@ void MAMAci::run_impl(const double dt) { // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); - const int kb = 62; compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); - // std::cout << "T_mid:" << dry_atm_.T_mid(0, kb) << std::endl; - // std::cout << "p_mid:" << dry_atm_.p_mid(0, kb) << std::endl; - // std::cout << "wsec:" << w_sec_(0, kb) << std::endl; - // std::cout << "w0:" << w0_(0, kb) << std::endl; - // std::cout << " rho: " << rho_(0, kb) << std::endl; - - // compute vertical velocity variance at the interfaces - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); - // std::cout << "TKE:" << tke_(0, kb) << std::endl; Kokkos::fence(); // wait for for tke_ to be computed. compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - // std::cout << "WSUB:" << wsub_(0, kb) << std::endl; - // std::cout << "WICE:" << wsubice_(0, kb) << std::endl; - // std::cout << "WSIG:" << wsig_(0, kb) << std::endl; Kokkos::fence(); // wait for wsig_ to be computed. compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } + + set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - // std::cout << "aitken_dry_dia_:" << aitken_dry_dia_(0, kb) << std::endl; + // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is // equivalent to "cldfrac_tot" in FM. It is part of the "dry_atm_" struct @@ -1297,91 +1388,17 @@ void MAMAci::run_impl(const double dt) { // output nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - naai_(icol, kk) = 9876.0; - } - } - store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, // output cloud_frac_, cloud_frac_prev_); - // std::cout << "naai_hom_:" << naai_hom_(0, kb) << std::endl; - // std::cout << "naai_:" << naai_(0, kb) << std::endl; - - // MUST FIXME: save cloud borne aerosols here!!!! - //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. - //------------------------------------------------------------- - /*NOTE: We probably need to store indices for the select few cloud borne - aerosols Fortran code: lchnk_zb = lchnk - begchunk ! save copy of cloud borne - aerosols for use in heterogeneous freezing before ! we change it in dropmixnuc - do ispec = 1, ncnst - call pbuf_get_field(pbuf, hetfrz_aer_spec_idx(ispec), ptr2d) - aer_cb(:ncol,:,ispec,lchnk_zb) = ptr2d(:ncol,:) - aer_cb(:ncol,:,ispec,lchnk_zb) = aer_cb(:ncol,:,ispec,lchnk_zb) * - rho(:ncol,:) enddo - */ - compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - // std::cout << "rpdel_:" << rpdel_(0, kb) << std::endl; Kokkos::fence(); // wait for rpdel_ to be computed. - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } - call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output @@ -1393,17 +1410,13 @@ void MAMAci::run_impl(const double dt) { dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - // std::cout << "factnum_:" << factnum_(0, 0, kb)<<" : "<< factnum_(0, 1, - // kb)<<" : "<( team_policy, ptend_q_output_, ptend_q_, nlev_); @@ -1419,20 +1432,19 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); + + print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), + wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), + rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, + qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), + hetfrz_contact_nucleation_tend_(0, kb), + hetfrz_depostion_nucleation_tend_(0, kb), kb); const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { - std::cout << "SOmethign changed!!!! :" + std::cout << "Somethign changed!!!! :" << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; exit(1); } - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend_(0, kb) << ":" - << hetfrz_immersion_nucleation_tend_(0, 0) << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend_(0, kb) << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend_(0, kb) << std::endl; - Kokkos::fence(); // wait before returning to calling function } From 831ffbbcdac29db7832d59e35c4b1d881d8752ae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 20:51:50 -0700 Subject: [PATCH 316/476] further cleanup for the output to be used by other processes --- .../mam/eamxx_mam_aci_process_interface.cpp | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 67184b7466d1..0a28c55a2e60 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -502,7 +502,13 @@ void call_function_dropmixnuc( const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, - // ## outputs ## + // Following outputs are all diagnostics + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, + + // ## outputs to be used by other processes ## // qqcw_fld_work should be directly assigned to the cloud borne aerosols MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], @@ -515,12 +521,6 @@ void call_function_dropmixnuc( // tendnd is used by microphysics scheme (e.g. P3) MAMAci::view_2d tendnd, - // Following outputs are all diagnostics - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, - MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // ## work arrays ## MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, @@ -1338,9 +1338,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - + // FIXME: Remove set_input and print_input const int kb = 62; + set_input(w_sec_int_, kvh_int_, ncol_, nlev_); print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, kb)); const auto scan_policy = ekat::ExeSpaceUtils< @@ -1371,10 +1371,11 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for wsig_ to be computed. + // We need dry diameter for only aitken mode compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - + // FIXME:Remove set_dgait set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1386,8 +1387,11 @@ void MAMAci::run_impl(const double dt) { nucleate_ice_, team_policy, dry_atm_, dry_aero_, wsubice_, aitken_dry_dia_, nlev_, dt, // output - nihf_, niim_, nidep_, nimey_, naai_hom_, naai_); + nihf_, niim_, nidep_, nimey_, naai_hom_, + // ## output to be used by the other processes ## + naai_); + // Compute cloud fractions based on cloud threshold store_liquid_cloud_fraction(team_policy, dry_atm_, liqcldf_, liqcldf_prev_, top_lev_, nlev_, // output @@ -1402,9 +1406,10 @@ void MAMAci::run_impl(const double dt) { call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output - qqcw_fld_work_, ptend_q_, factnum_, tendnd_, coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, nsource_, wtke_, ccn_, + // ## output to be used by the other processes ## + qqcw_fld_work_, ptend_q_, factnum_, tendnd_, // work arrays raercol_cw_, raercol_, state_q_work_, nact_, mact_, dropmixnuc_scratch_mem_); @@ -1427,7 +1432,7 @@ void MAMAci::run_impl(const double dt) { call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, - // output + // ## output to be used by the other processes ## hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, hetfrz_depostion_nucleation_tend_, // work arrays From e7730c707ed2dbba86ca9de42bf7722b0c8cb0fd Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 20 Apr 2024 21:13:54 -0700 Subject: [PATCH 317/476] Adds some comments --- .../mam/eamxx_mam_aci_process_interface.cpp | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 0a28c55a2e60..645d520ca5c4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -476,10 +476,12 @@ void compute_recipical_pseudo_density(const haero::ThreadTeam &team, const int icol, const int nlev, // output MAMAci::view_2d rpdel) { - // FIXME: Add an assert to ensure pdel is non-zero Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), - KOKKOS_LAMBDA(int kk) { rpdel(icol, kk) = 1 / pdel(icol, kk); }); + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); } void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, MAMAci::const_view_2d pdel, @@ -1403,6 +1405,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. + // Compute activated CCN number tendency (tendnd_) and updated + // cloud borne aerosols (stored in a work array) and interstitial + // aerosols tendencies call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output @@ -1415,21 +1420,13 @@ void MAMAci::run_impl(const double dt) { dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. - // MUST FIXME: save cloud borne aerosols here!!!! - //------------------------------------------------------------- - // Save cloud borne aerosols to be used in the heterozenous - // freezing before they are changed by the droplet activation - // process. This is only a select subset of cloud borne - // aerosols, not all the cloud borne aerosols. - //------------------------------------------------------------- - - copy_mam4xx_array_to_scream( - team_policy, ptend_q_output_, ptend_q_, nlev_); - copy_mam4xx_array_to_scream( - team_policy, coltend_outp_, coltend_, nlev_); - copy_mam4xx_array_to_scream( - team_policy, coltend_cw_outp_, coltend_cw_, nlev_); + //--------------------------------------------------------------------------- + // NOTE: DO NOT UPDATE cloud borne aerosols using the qqcw_fld_work_ array + // at this point as heterozenous freezing needs to use cloud borne aerosols + // before they are changed by the droplet activation (dropmixnuc) process. + //--------------------------------------------------------------------------- + // Compute hetrozenous freezing call_hetfrz_compute_tendencies( team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // ## output to be used by the other processes ## @@ -1438,6 +1435,7 @@ void MAMAci::run_impl(const double dt) { // work arrays diagnostic_scratch_); + // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, From 7f0cf90a087ce450949c05dbfd724eef9ed66a17 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 22:08:37 -0700 Subject: [PATCH 318/476] Adds functions to update to qqcw and state --- .../mam/eamxx_mam_aci_process_interface.cpp | 180 ++++++++++++++---- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 148 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 645d520ca5c4..0c7c7c72d659 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -9,7 +9,18 @@ void print_input(const Real t, const Real p, const Real wsec) { std::cout << "p_mid:" << p << std::endl; std::cout << "wsec:" << wsec << std::endl; } - +void print_bef_ndrop(const mam_coupling::AerosolState &dry_aero, const int kb) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "Bef-ndrop-cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) + << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "Bef-ndrop-cldbrn-mmr:" + << dry_aero.cld_aero_mmr[m][a](0, kb) << std::endl; + } + } + } +} void print_output(const Real w0, const Real rho, const Real tke, const Real wsub, const Real wice, const Real wsig, const Real naai_hom, const Real naai, const Real rpdel, @@ -18,7 +29,8 @@ void print_output(const Real w0, const Real rho, const Real tke, MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const Real hetfrz_immersion_nucleation_tend, const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, const int kb) { + const Real hetfrz_depostion_nucleation_tend, + const mam_coupling::AerosolState &dry_aero, const int kb) { std::cout << "w0:" << w0 << std::endl; std::cout << " rho: " << rho << std::endl; std::cout << "TKE:" << tke << std::endl; @@ -39,6 +51,25 @@ void print_output(const Real w0, const Real rho, const Real tke, std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) << std::endl; } + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend << ":" @@ -358,9 +389,8 @@ void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, const mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aerosol_state, - const MAMAci::view_2d wsubice, const MAMAci::view_2d aitken_dry_dia, - const int nlev, const double dt, + const mam_coupling::AerosolState &dry_aero, const MAMAci::view_2d wsubice, + const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, // output MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai) { @@ -385,7 +415,7 @@ void compute_nucleate_ice_tendencies( // Store interstitial and cld borne aerosols in "progrs" struture mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aero, icol); // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) haero::Atmosphere haero_atm = @@ -502,7 +532,7 @@ void call_function_dropmixnuc( mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, - const mam_coupling::AerosolState &dry_aerosol_state, const int nlev, + const mam_coupling::AerosolState &dry_aero, const int nlev, // Following outputs are all diagnostics MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], @@ -649,32 +679,46 @@ void call_function_dropmixnuc( // and Atmosphere (water species such as qv, qc etc.) // get prognostics - mam4::Prognostics progs_at_col = - aerosols_for_column(dry_aerosol_state, icol); + mam4::Prognostics progs_at_col = aerosols_for_column(dry_aero, icol); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays + // FIXME:: Kookos for here?? for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; - // get state_q at a gri cell (col,lev) + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); - // get qqcw at a gri cell (col,lev) - mam4::utils::extract_qqcw_from_prognostics(progs_at_col, - qqcw_at_lev_col, klev); + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } - // create colum views of state_q and qqcw - for(int icnst = 15; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = - state_q_at_lev_col[icnst]; // FIXME: ensure that indices are - // right! remove "15" if possible!! - qqcw_view[icnst - 15](klev) = qqcw_at_lev_col[icnst]; + // get qqcw at a grid cell (col,lev) + // NOTE: The layout for qqcw array is based on mam_idx in dropmixnuc + // To mimic that, we are using the following for-loops + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + qqcw_view[ind_qqcw](klev) = dry_aero.cld_aero_nmr[m](icol, klev); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_mmr[m][a](icol, klev); + ++ind_qqcw; + } + } } } @@ -739,6 +783,64 @@ void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, }); } +// Update cloud borne aerosols +void update_cloud_borne_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, + // output + mam_coupling::AerosolState &dry_aero) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + dry_aero.cld_aero_nmr[m](icol, kk) = + qqcw_fld_work[ind_qqcw](icol, kk); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](icol, kk) = + qqcw_fld_work[ind_qqcw](icol, kk); + ++ind_qqcw; + } + } + } + }); + }); +} + +// Update interstitial aerosols using tendencies +void update_interstitial_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, + const Real dt, + // output + mam_coupling::AerosolState &dry_aero) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + ; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q[s_idx](icol, kk) * dt; + s_idx++; + } + } + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q[s_idx](icol, kk) * dt; + s_idx++; + } + }); + }); +} + void call_hetfrz_compute_tendencies( haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, mam_coupling::DryAtmosphere &dry_atm_, @@ -749,9 +851,9 @@ void call_hetfrz_compute_tendencies( MAMAci::view_2d hetfrz_contact_nucleation_tend, MAMAci::view_2d hetfrz_depostion_nucleation_tend, MAMAci::view_2d diagnostic_scratch_[]) { - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState dry_aerosol_state = dry_aero_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState dry_aero = dry_aero_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; MAMAci::view_2d diagnostic_scratch[42]; for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; @@ -766,7 +868,7 @@ void call_hetfrz_compute_tendencies( atmosphere_for_column(dry_atmosphere, icol); haero::Surface surf{}; mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aerosol_state, icol); + mam_coupling::aerosols_for_column(dry_aero, icol); const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); @@ -1343,7 +1445,8 @@ void MAMAci::run_impl(const double dt) { // FIXME: Remove set_input and print_input const int kb = 62; set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, kb)); + // print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, + // kb)); const auto scan_policy = ekat::ExeSpaceUtils< // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, @@ -1404,10 +1507,10 @@ void MAMAci::run_impl(const double dt) { rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. - - // Compute activated CCN number tendency (tendnd_) and updated - // cloud borne aerosols (stored in a work array) and interstitial - // aerosols tendencies + // print_bef_ndrop(dry_aero_, kb); + // Compute activated CCN number tendency (tendnd_) and updated + // cloud borne aerosols (stored in a work array) and interstitial + // aerosols tendencies call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output @@ -1435,13 +1538,28 @@ void MAMAci::run_impl(const double dt) { // work arrays diagnostic_scratch_); + //--------------------------------------------------------------- + //----------------- End of all processes ------------------------ + //--------------------------------------------------------------- + + // Update cloud borne aerosols + update_cloud_borne_aerosols(team_policy, qqcw_fld_work_, nlev_, + // output + dry_aero_); + + // Update interstitial aerosols using tendencies + update_interstitial_aerosols(team_policy, ptend_q_, nlev_, dt, + // output + dry_aero_); + // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), kb); + hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); + const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { std::cout << "Somethign changed!!!! :" @@ -1451,10 +1569,6 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait before returning to calling function } -void MAMAci::finalize_impl() { - m_atm_logger->log(ekat::logger::LogLevel::info, "calling ACI final"); -} - } // namespace scream /* diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 297078c87275..471cec42ce6d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -171,7 +171,7 @@ class MAMAci final : public scream::AtmosphereProcess { // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; - void finalize_impl() override; + void finalize_impl() const {/*DO NOTHING*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. From 19606b8c7a0f45f8e3618170534fc2f64624dcde Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 22:45:55 -0700 Subject: [PATCH 319/476] Removes finanlize from cpp and mam_copy functions --- .../mam/eamxx_mam_aci_process_interface.cpp | 139 +----------------- .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 3 insertions(+), 138 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 0c7c7c72d659..00064e0a5082 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -222,37 +222,6 @@ void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, } } -KOKKOS_INLINE_FUNCTION -void copy_scream_array_to_mam4xx(const haero::ThreadTeam &team, - const MAMAci::view_2d mam4xx_view, - MAMAci::const_view_3d scream_view, - const int icol, const int nlev, - const int view_num) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - mam4xx_view(icol, kk) = scream_view(icol, kk, view_num); - }); -} - -void copy_scream_array_to_mam4xx( - haero::ThreadTeamPolicy team_policy, - MAMAci::view_2d mam4xx_views[mam4::ndrop::ncnst_tot], - MAMAci::const_view_3d scream_view, const int nlev) { - // Localize the input arrays. - MAMAci::view_2d mam4xx[mam4::ndrop::ncnst_tot]; - for(int view_num = 0; view_num < mam4::ndrop::ncnst_tot; ++view_num) - mam4xx[view_num] = mam4xx_views[view_num]; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - for(int view_num = 0; view_num < mam4::ndrop::ncnst_tot; ++view_num) { - MAMAci::view_2d mam4xx_view = mam4xx[view_num]; - copy_scream_array_to_mam4xx(team, mam4xx_view, scream_view, icol, - nlev, view_num); - } - }); -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -300,9 +269,6 @@ void compute_tke_using_w_sec(const haero::ThreadTeam &team, const int nlev, // output MAMAci::view_2d tke) { - // FIXME Is this the correct boundary condition for tke at the surface? - // TKE seems to be at interfaces but w_sec is at cell centers so this - // descrepensy needs to be worked out. Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, nlev + 1), KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); @@ -1539,7 +1505,7 @@ void MAMAci::run_impl(const double dt) { diagnostic_scratch_); //--------------------------------------------------------------- - //----------------- End of all processes ------------------------ + // Now update interstitial and cllud borne aerosols //--------------------------------------------------------------- // Update cloud borne aerosols @@ -1569,105 +1535,4 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait before returning to calling function } -} // namespace scream - -/* -FORTRAN OUTPUT: - - ----- OUPUT AT time step: 6 6 - wo 3.343177351577457E-004 - rho: 1.28186433384113 - TKE: 1.666465903934813E-002 6.097664104873541E-002 - wsub: 0.160873967324407 - wsubi: 0.200000000000000 - wsig: 0.160873967324407 - dgnum_ait: 4.362354500358337E-008 - naai: 11464.5957222634 - naai_hom: 29838.6879763933 - aer_cb: so4_c1 4.327927689273498E-012 so4_c1 4.327927689273498E-012 - aer_cb: bc_c1 6.811694223739704E-014 bc_c1 6.811694223739704E-014 - aer_cb: pom_c1 8.647238075040541E-013 pom_c1 8.647238075040541E-013 - aer_cb: soa_c1 6.734058317716400E-012 soa_c1 6.734058317716400E-012 - aer_cb: dst_c1 8.239303776951524E-014 dst_c1 8.239303776951524E-014 - aer_cb: ncl_c1 2.685810991448475E-014 ncl_c1 2.685810991448475E-014 - aer_cb: mom_c1 2.218253512777307E-015 mom_c1 2.218253512777307E-015 - aer_cb: num_c1 477478.884677930 num_c1 477478.884677930 - aer_cb: dst_c3 2.869197497896834E-012 dst_c3 2.869197497896834E-012 - aer_cb: ncl_c3 4.636857908110365E-013 ncl_c3 4.636857908110365E-013 - aer_cb: so4_c3 4.375223642985256E-014 so4_c3 4.375223642985256E-014 - aer_cb: bc_c3 3.674860666928688E-016 bc_c3 3.674860666928688E-016 - aer_cb: pom_c3 1.114802999982066E-015 pom_c3 1.114802999982066E-015 - aer_cb: soa_c3 8.915776993183282E-015 soa_c3 8.915776993183282E-015 - aer_cb: mom_c3 7.802642470782658E-017 mom_c3 7.802642470782658E-017 - aer_cb: num_c3 151.593971049442 num_c3 151.593971049442 - aer_cb: bc_c4 1.127495264613331E-027 bc_c4 1.127495264613331E-027 - aer_cb: pom_c4 8.204883715624573E-027 pom_c4 8.204883715624573E-027 - aer_cb: mom_c4 3.249069782816687E-033 mom_c4 3.249069782816687E-033 - aer_cb: num_c4 1.345885758586041E-013 num_c4 1.345885758586041E-013 - tendnd: -839.404918218856 7289315.36108503 7792958.31201634 - 1.666666666666667E-003 - factnum: 0.998556176544263 0.861689826773470 - 0.999999999974140 0.000000000000000E+000 - nctend_mixnuc: -839.404918218856 - O3_tend: 0.000000000000000E+000 - H2O2_tend: 0.000000000000000E+000 - H2SO4_tend: 0.000000000000000E+000 - SO2_tend: 0.000000000000000E+000 - DMS_tend: 0.000000000000000E+000 - SOAG_tend: 0.000000000000000E+000 - so4_a1_tend: -1.384310943569190E-014 - pom_a1_tend: -2.950799002682271E-015 - soa_a1_tend: -2.283869371271150E-014 - bc_a1_tend: -2.202672700280516E-016 - dst_a1_tend: -2.194060414083922E-016 - ncl_a1_tend: -6.981124745267083E-017 - mom_a1_tend: -5.662690535048673E-018 - num_a1_tend: -1626.38730532537 - so4_a2_tend: -2.548707408341951E-017 - soa_a2_tend: -1.929990276139139E-018 - ncl_a2_tend: -6.855087738119723E-019 - mom_a2_tend: -4.571691604185304E-020 - num_a2_tend: -111.890330245159 - dst_a3_tend: -7.609611333715698E-015 - ncl_a3_tend: -1.256350252705383E-015 - so4_a3_tend: -1.320988655880598E-016 - bc_a3_tend: -1.099586316024402E-018 - pom_a3_tend: -3.309939677824050E-018 - soa_a3_tend: -2.767668335981830E-017 - mom_a3_tend: -2.254911604324925E-019 - num_a3_tend: -0.406788905791186 - pom_a4_tend: -2.375070578317096E-017 - bc_a4_tend: 4.082479713528180E-017 - mom_a4_tend: -4.901356204764327E-023 - num_a4_tend: -34.4232851017138 - so4_c1: 1322558.40616450 - pom_c1: 1.233997159684093E-011 - soa_c1: 2.393397505793018E-012 - bc_c1: 1.872520079842212E-011 - dst_c1: 1.922157644491776E-013 - ncl_c1: 2.530385880698310E-013 - mom_c1: 8.315014074126540E-014 - num_c1: 6.895143982427237E-015 - so4_c2: 142214.467855250 - soa_c2: 3.181118913367564E-014 - ncl_c2: 2.996638282752921E-015 - mom_c2: 8.055333842809433E-016 - num_c2: 5.202632275847699E-017 - dst_c3: 464.524744048298 - ncl_c3: 8.814810290885256E-012 - so4_c3: 1.417415096039318E-012 - bc_c3: 1.284022582681906E-013 - pom_c3: 1.081226492056768E-015 - soa_c3: 3.292708392236485E-015 - mom_c3: 2.584587804061707E-014 - num_c3: 2.324887367507579E-016 - pom_c4: 6.934165370527951E-014 - bc_c4: 2.703924752644086E-026 - mom_c4: 3.684880329580097E-027 - num_c4: 1.049556651713384E-032 - - frzimm: 5.651860156917002E-006 - frzcnt: 0.000000000000000E+000 - frzdep: 0.000000000000000E+000 - - */ \ No newline at end of file +} // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 471cec42ce6d..1fa255ccdf73 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -171,7 +171,7 @@ class MAMAci final : public scream::AtmosphereProcess { // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; - void finalize_impl() const {/*DO NOTHING*/}; + void finalize_impl(){/*DO NOTHING*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. From 9f8b2af8c1295c0e281c4623ff84c79dd5349732 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 23:01:59 -0700 Subject: [PATCH 320/476] Removes another set of unused mam copy functions --- .../mam/eamxx_mam_aci_process_interface.cpp | 49 ++++--------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 00064e0a5082..da0632fd6195 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -359,7 +359,9 @@ void compute_nucleate_ice_tendencies( const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, // output MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, - MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, MAMAci::view_2d naai) { + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, + // ## output used by other processes ## + MAMAci::view_2d naai) { //------------------------------------------------------------- // Get number of activated aerosol for ice nucleation (naai) // from ice nucleation @@ -368,9 +370,6 @@ void compute_nucleate_ice_tendencies( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // for(int icol = 0; icol < 5; ++icol) { - // std::cout<<""< -void copy_mam4xx_array_to_scream(haero::ThreadTeamPolicy team_policy, - MAMAci::view_3d scream, - MAMAci::view_2d mam4xx[len], const int nlev) { - // Localize the input arrays. - MAMAci::view_2d mam4xx_loc[len]; - for(int view_num = 0; view_num < len; ++view_num) - mam4xx_loc[view_num] = mam4xx[view_num]; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - for(int view_num = 0; view_num < len; ++view_num) { - MAMAci::view_2d mam4xx_view = mam4xx_loc[view_num]; - copy_mam4xx_array_to_scream(team, scream, mam4xx_view, icol, nlev, - view_num); - } - }); } // Update cloud borne aerosols @@ -790,7 +758,6 @@ void update_interstitial_aerosols( Kokkos::parallel_for( Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - ; for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { for(int a = 0; a < mam4::num_species_mode(m); ++a) { if(dry_aero.int_aero_mmr[m][a].data()) { @@ -909,7 +876,7 @@ void call_hetfrz_compute_tendencies( // values are store in diags above. const mam4::Tendencies tends(nlev); const mam4::AeroConfig aero_config; - const Real t = 0; + const Real t = 0; // not used hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); @@ -1085,6 +1052,11 @@ void MAMAci::set_grids( // ------------------------------------------------------------------------ // Output from ice nucleation process // ------------------------------------------------------------------------ + + // number of activated aerosol for ice nucleation[#/kg] + add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); + + // FIXME: Diagnostics output const auto m3_inv = 1 / m / m / m; // inverse of m3 // number conc of ice nuclei due to heterogeneous freezing [1/m3] add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, @@ -1106,9 +1078,6 @@ void MAMAci::set_grids( add_field("num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, n_unit, grid_name); - // number of activated aerosol for ice nucleation[#/kg] - add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); - // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ From 931278e07f6cbad7ce292ff13b3020f8e450e6bf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 21 Apr 2024 23:12:23 -0700 Subject: [PATCH 321/476] Removes nihf_ from FM --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index da0632fd6195..ce3ad0c0d97b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1058,9 +1058,6 @@ void MAMAci::set_grids( // FIXME: Diagnostics output const auto m3_inv = 1 / m / m / m; // inverse of m3 - // number conc of ice nuclei due to heterogeneous freezing [1/m3] - add_field("icenuc_num_hetfrz", scalar3d_layout_mid, m3_inv, - grid_name); // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] add_field("icenuc_num_immfrz", scalar3d_layout_mid, m3_inv, @@ -1235,7 +1232,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // computed updraft velocity dry_atm_.w_updraft = buffer_.w_updraft; - nihf_ = get_field_out("icenuc_num_hetfrz").get_view(); niim_ = get_field_out("icenuc_num_immfrz").get_view(); nidep_ = get_field_out("icenuc_num_depnuc").get_view(); nimey_ = get_field_out("icenuc_num_meydep").get_view(); @@ -1306,6 +1302,10 @@ void MAMAci::initialize_impl(const RunType run_type) { // Allocate memory for the class members // (Kokkos::resize only works on host to allocates memory) + + // number conc of ice nuclei due to heterogeneous freezing [1/m3] + Kokkos::resize(nihf_, ncol_, nlev_); + Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); Kokkos::resize(tke_, ncol_, nlev_ + 1); From 5f23711c718d6238c0cf4148cfc327b4c7ce3042 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 09:56:52 -0700 Subject: [PATCH 322/476] Removes all ice nuc diags and ccn from FM --- .../mam/eamxx_mam_aci_process_interface.cpp | 68 ++++++++----------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ce3ad0c0d97b..8636fd1cbd07 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1056,25 +1056,6 @@ void MAMAci::set_grids( // number of activated aerosol for ice nucleation[#/kg] add_field("ni_activated", scalar3d_layout_mid, n_unit, grid_name); - // FIXME: Diagnostics output - const auto m3_inv = 1 / m / m / m; // inverse of m3 - - // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] - add_field("icenuc_num_immfrz", scalar3d_layout_mid, m3_inv, - grid_name); - - // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] - add_field("icenuc_num_depnuc", scalar3d_layout_mid, m3_inv, - grid_name); - - // number conc of ice nuclei due to meyers deposition [1/m3] - add_field("icenuc_num_meydep", scalar3d_layout_mid, m3_inv, - grid_name); - - // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] - add_field("num_act_aerosol_ice_nucle_hom", scalar3d_layout_mid, - n_unit, grid_name); - // ------------------------------------------------------------------------ // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ @@ -1114,15 +1095,6 @@ void MAMAci::set_grids( // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - constexpr int psat = mam4::ndrop::psat; - FieldLayout scalar4d_layout_psat_mid{{COL, LEV, MAM_PSAT}, - {ncol_, nlev_, psat}}; - // number conc of aerosols activated at supersat [#/m^3] - // NOTE: activation fraction fluxes are defined as - // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] - // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] - add_field("ccn", scalar4d_layout_psat_mid, m3_inv, grid_name); - constexpr int num_aero_const = mam4::ndrop::ncnst_tot; FieldLayout scalar4d_layout_naero_const_mid{{COL, LEV, MAM_AERO_NCNST}, {ncol_, nlev_, num_aero_const}}; @@ -1232,12 +1204,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // computed updraft velocity dry_atm_.w_updraft = buffer_.w_updraft; - niim_ = get_field_out("icenuc_num_immfrz").get_view(); - nidep_ = get_field_out("icenuc_num_depnuc").get_view(); - nimey_ = get_field_out("icenuc_num_meydep").get_view(); - naai_hom_ = - get_field_out("num_act_aerosol_ice_nucle_hom").get_view(); - + // ------------------------------------------------------------------------ + // Output fields to be used by other processes + // ------------------------------------------------------------------------ naai_ = get_field_out("ni_activated").get_view(); qcld_ = get_field_out("qcld").get_view(); ptend_q_output_ = get_field_out("ptend_q").get_view(); @@ -1247,7 +1216,6 @@ void MAMAci::initialize_impl(const RunType run_type) { ndropmix_ = get_field_out("ndropmix").get_view(); nsource_ = get_field_out("nsource").get_view(); wtke_ = get_field_out("wtke").get_view(); - ccn_ = get_field_out("ccn").get_view(); coltend_outp_ = get_field_out("coltend").get_view(); coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); hetfrz_immersion_nucleation_tend_ = @@ -1300,11 +1268,10 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + //--------------------------------------------------------------------------------- // Allocate memory for the class members // (Kokkos::resize only works on host to allocates memory) - - // number conc of ice nuclei due to heterogeneous freezing [1/m3] - Kokkos::resize(nihf_, ncol_, nlev_); + //--------------------------------------------------------------------------------- Kokkos::resize(rho_, ncol_, nlev_); Kokkos::resize(w0_, ncol_, nlev_); @@ -1318,6 +1285,31 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(aitken_dry_dia_, ncol_, nlev_); Kokkos::resize(rpdel_, ncol_, nlev_); + //--------------------------------------------------------------------------------- + // Diagnotics variables from the ice nucleation scheme + //--------------------------------------------------------------------------------- + + // number conc of ice nuclei due to heterogeneous freezing [1/m3] + Kokkos::resize(nihf_, ncol_, nlev_); + + // number conc of ice nuclei due to immersionfreezing (hetero nuc) [1/m3] + Kokkos::resize(niim_, ncol_, nlev_); + + // number conc of ice nuclei due to deposition nucleation (hetero nuc)[1/m3] + Kokkos::resize(nidep_, ncol_, nlev_); + + // number conc of ice nuclei due to meyers deposition [1/m3] + Kokkos::resize(nimey_, ncol_, nlev_); + + // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] + Kokkos::resize(naai_hom_, ncol_, nlev_); + + // number conc of aerosols activated at supersat [#/m^3] + // NOTE: activation fraction fluxes are defined as + // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] + // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] + Kokkos::resize(ccn_, ncol_, nlev_, mam4::ndrop::psat); + for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } From f4b01a6cda3bad56d1a64b9965eb30e24ef3492b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 11:40:35 -0700 Subject: [PATCH 323/476] Removes some unused variables from FM --- .../mam/eamxx_mam_aci_process_interface.cpp | 48 ++++++------------- .../mam/eamxx_mam_aci_process_interface.hpp | 4 -- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 8636fd1cbd07..4577f0732345 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1060,13 +1060,6 @@ void MAMAci::set_grids( // Output from droplet activation process (dropmixnuc) // ------------------------------------------------------------------------ - constexpr int pcnst = mam4::aero_model::pcnst; - FieldLayout scalar4d_layout_nconst_mid{{COL, LEV, MAM_NCNST}, - {ncol_, nlev_, pcnst}}; - - // tendencies for interstitial and cloud borne aerosols [kg/kg or #/kg] - add_field("ptend_q", scalar4d_layout_nconst_mid, q_unit, grid_name); - // tendency in droplet number mixing ratio [#/kg/s] add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); @@ -1095,28 +1088,15 @@ void MAMAci::set_grids( // subgrid vertical velocity [m/s] add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - constexpr int num_aero_const = mam4::ndrop::ncnst_tot; - FieldLayout scalar4d_layout_naero_const_mid{{COL, LEV, MAM_AERO_NCNST}, - {ncol_, nlev_, num_aero_const}}; - - // column tendency for diagnostic output - add_field("coltend", scalar4d_layout_naero_const_mid, nondim, - grid_name); - - // column tendency - add_field("coltend_cw", scalar4d_layout_naero_const_mid, nondim, - grid_name); + // ------------------------------------------------------------------------ + // Output from hetrozenous freezing + // ------------------------------------------------------------------------ const auto cm = m / 100; // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - - // ------------------------------------------------------------------------ - // Output from hetrozenous freezing - // ------------------------------------------------------------------------ - // heterogeous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -1207,17 +1187,15 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ // Output fields to be used by other processes // ------------------------------------------------------------------------ - naai_ = get_field_out("ni_activated").get_view(); - qcld_ = get_field_out("qcld").get_view(); - ptend_q_output_ = get_field_out("ptend_q").get_view(); - tendnd_ = get_field_out("nc_nuceat_tend").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); - coltend_outp_ = get_field_out("coltend").get_view(); - coltend_cw_outp_ = get_field_out("coltend_cw").get_view(); + naai_ = get_field_out("ni_activated").get_view(); + qcld_ = get_field_out("qcld").get_view(); + tendnd_ = get_field_out("nc_nuceat_tend").get_view(); + factnum_ = get_field_out("factnum").get_view(); + ndropcol_ = get_field_out("ndropcol").get_view(); + ndropmix_ = get_field_out("ndropmix").get_view(); + nsource_ = get_field_out("nsource").get_view(); + wtke_ = get_field_out("wtke").get_view(); + hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = @@ -1317,7 +1295,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // These are temp arrays formatted like mam4xx wants. // Not sure if there is a way to do this with scream. // FIXME: Do we need these? + // column tendency for diagnostic output Kokkos::resize(coltend_[i], ncol_, nlev_); + // column tendency Kokkos::resize(coltend_cw_[i], ncol_, nlev_); } for(int i = 0; i < mam4::aero_model::pcnst; ++i) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 1fa255ccdf73..c71988185720 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -76,9 +76,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d cloud_frac_prev_; view_2d qcld_; view_2d tendnd_; - // ptend_q_ is just ptend_q_output_ reformatted. view_2d ptend_q_[mam4::aero_model::pcnst]; - view_3d ptend_q_output_; view_3d factnum_; const_view_3d qqcw_input_; view_2d qqcw_[mam4::ndrop::ncnst_tot]; @@ -87,9 +85,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d nsource_; view_2d wtke_; view_3d ccn_; - view_3d coltend_outp_; view_2d coltend_[mam4::ndrop::ncnst_tot]; - view_3d coltend_cw_outp_; view_2d coltend_cw_[mam4::ndrop::ncnst_tot]; // raercol_cw_ and raercol_ are work arrays for dropmixnuc, allocated on the From 4d3b83dcf5dc9a36b7e4b8d81611f6de1459ef5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 11:59:59 -0700 Subject: [PATCH 324/476] Removed all diagnostics from FM --- .../mam/eamxx_mam_aci_process_interface.cpp | 93 ++++++++++--------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4577f0732345..b18984931cd2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1064,30 +1064,6 @@ void MAMAci::set_grids( add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); - // activation fraction for aerosol number [fraction] - add_field("factnum", scalar4d_layout_mid, nondim, grid_name); - - // NOTE: Here is a series of internal dropmixnuc variables; - // maybe we should move them to diagnostics later (FIXME) - - // cloud droplet number mixing ratio [#/kg] - add_field("qcld", scalar3d_layout_mid, n_unit, grid_name); - - auto inv_m2 = 1 / m / m; - inv_m2.set_string("#/m2"); - - // column-integrated droplet number [#/m2] - add_field("ndropcol", scalar3d_layout_mid, inv_m2, grid_name); - - // droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); - - // droplet number mixing ratio source tendency [#/kg/s] - add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); - - // subgrid vertical velocity [m/s] - add_field("wtke", scalar3d_layout_mid, m / s, grid_name); - // ------------------------------------------------------------------------ // Output from hetrozenous freezing // ------------------------------------------------------------------------ @@ -1187,21 +1163,11 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ // Output fields to be used by other processes // ------------------------------------------------------------------------ - naai_ = get_field_out("ni_activated").get_view(); - qcld_ = get_field_out("qcld").get_view(); - tendnd_ = get_field_out("nc_nuceat_tend").get_view(); - factnum_ = get_field_out("factnum").get_view(); - ndropcol_ = get_field_out("ndropcol").get_view(); - ndropmix_ = get_field_out("ndropmix").get_view(); - nsource_ = get_field_out("nsource").get_view(); - wtke_ = get_field_out("wtke").get_view(); + // ice nucleation output + naai_ = get_field_out("ni_activated").get_view(); - hetfrz_immersion_nucleation_tend_ = - get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - hetfrz_contact_nucleation_tend_ = - get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = - get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + // droplet activation output + tendnd_ = get_field_out("nc_nuceat_tend").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -1246,6 +1212,14 @@ void MAMAci::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // hetrozenous freezing outputs + hetfrz_immersion_nucleation_tend_ = + get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + hetfrz_contact_nucleation_tend_ = + get_field_out("hetfrz_contact_nucleation_tend").get_view(); + hetfrz_depostion_nucleation_tend_ = + get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + //--------------------------------------------------------------------------------- // Allocate memory for the class members // (Kokkos::resize only works on host to allocates memory) @@ -1282,19 +1256,39 @@ void MAMAci::initialize_impl(const RunType run_type) { // number of activated aerosol for ice nucleation(homogeneous frz only)[#/kg] Kokkos::resize(naai_hom_, ncol_, nlev_); + //--------------------------------------------------------------------------------- + // Diagnotics variables from the droplet activation scheme + //--------------------------------------------------------------------------------- + + // activation fraction for aerosol number [fraction] + const int num_aero_modes = mam_coupling::num_aero_modes(); + Kokkos::resize(factnum_, ncol_, num_aero_modes, nlev_); + + // cloud droplet number mixing ratio [#/kg] + Kokkos::resize(qcld_, ncol_, nlev_); + // number conc of aerosols activated at supersat [#/m^3] // NOTE: activation fraction fluxes are defined as // fluxn = [flux of activated aero. number into cloud[#/m^2/s]] // / [aero. number conc. in updraft, just below cloudbase [#/m^3]] Kokkos::resize(ccn_, ncol_, nlev_, mam4::ndrop::psat); + // column-integrated droplet number [#/m2] + Kokkos::resize(ndropcol_, ncol_, nlev_); + + // droplet number mixing ratio tendency due to mixing [#/kg/s] + Kokkos::resize(ndropmix_, ncol_, nlev_); + + // droplet number mixing ratio source tendency [#/kg/s] + Kokkos::resize(nsource_, ncol_, nlev_); + + // subgrid vertical velocity [m/s] + Kokkos::resize(wtke_, ncol_, nlev_); + for(int i = 0; i < dropmix_scratch_; ++i) { Kokkos::resize(dropmixnuc_scratch_mem_[i], ncol_, nlev_); } for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - // These are temp arrays formatted like mam4xx wants. - // Not sure if there is a way to do this with scream. - // FIXME: Do we need these? // column tendency for diagnostic output Kokkos::resize(coltend_[i], ncol_, nlev_); // column tendency @@ -1310,9 +1304,6 @@ void MAMAci::initialize_impl(const RunType run_type) { } } - for(int i = 0; i < 42; ++i) - Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); - // nact : fractional aero. number activation rate [/s] Kokkos::resize(nact_, ncol_, nlev_, mam_coupling::num_aero_modes()); @@ -1331,6 +1322,17 @@ void MAMAci::initialize_impl(const RunType run_type) { state_q_work_ = view_3d("state_q_work_", ncol_, nlev_, mam4::aero_model::pcnst); + //--------------------------------------------------------------------------------- + // Diagnotics variables from the hetrozenous ice nucleation scheme + //--------------------------------------------------------------------------------- + + for(int i = 0; i < 42; ++i) + Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); + + //--------------------------------------------------------------------------------- + // Initialize the processes + //--------------------------------------------------------------------------------- + mam4::AeroConfig aero_config; // configure the nucleation parameterization mam4::NucleateIce::Config nucleate_ice_config; @@ -1340,6 +1342,9 @@ void MAMAci::initialize_impl(const RunType run_type) { mam4::Hetfrz::Config hetfrz_config; hetfrz_.init(aero_config, hetfrz_config); + //--------------------------------------------------------------------------------- + // Setup preprocessing + //--------------------------------------------------------------------------------- // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); From bf2e4a5c599908f3bd048205be439bf3016cf6bb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 12:07:10 -0700 Subject: [PATCH 325/476] Replaces a for with parallel for for klev --- .../mam/eamxx_mam_aci_process_interface.cpp | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b18984931cd2..d372a2e68f11 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -648,41 +648,45 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays - // FIXME:: Kookos for here?? - for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + KOKKOS_LAMBDA(int klev) { + // for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } - // get qqcw at a grid cell (col,lev) - // NOTE: The layout for qqcw array is based on mam_idx in dropmixnuc - // To mimic that, we are using the following for-loops - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - qqcw_view[ind_qqcw](klev) = dry_aero.cld_aero_nmr[m](icol, klev); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { + // get qqcw at a grid cell (col,lev) + // NOTE: The layout for qqcw array is based on mam_idx in + // dropmixnuc To mimic that, we are using the following for-loops + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { qqcw_view[ind_qqcw](klev) = - dry_aero.cld_aero_mmr[m][a](icol, klev); + dry_aero.cld_aero_nmr[m](icol, klev); ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_mmr[m][a](icol, klev); + ++ind_qqcw; + } + } } - } - } - } + }); mam4::ndrop::dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), From 5a9c5ec50056517ac4aa1d2a0872d5788e4fd553 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 18:34:32 -0700 Subject: [PATCH 326/476] Some more cleanup --- .../mam/eamxx_mam_aci_process_interface.cpp | 30 ++++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 7 ++--- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d372a2e68f11..ba718835844a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -417,7 +417,7 @@ void compute_nucleate_ice_tendencies( const mam4::Tendencies tends(nlev); // not used const mam4::AeroConfig aero_config; const Real t = 0; // not used - nucleate_ice.compute_tendencies(aero_config, /*team,*/ t, dt, haero_atm, + nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, progs, diags, tends); }); } @@ -792,8 +792,8 @@ void call_hetfrz_compute_tendencies( mam_coupling::AerosolState dry_aero = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; - MAMAci::view_2d diagnostic_scratch[42]; - for(int i = 0; i < 42; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; + MAMAci::view_2d diagnostic_scratch[43]; + for(int i = 0; i < 43; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -869,7 +869,16 @@ void call_hetfrz_compute_tendencies( diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + diags.stratiform_cloud_fraction = + ekat::subview(diagnostic_scratch[42], icol); + // assign cloud fraction + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + KOKKOS_LAMBDA(int klev) { + diags.stratiform_cloud_fraction(klev) = + haero_atm.cloud_fraction(klev); + }); //------------------------------------------------------------- // Heterogeneous freezing // frzimm, frzcnt, frzdep are the outputs of @@ -1330,7 +1339,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // Diagnotics variables from the hetrozenous ice nucleation scheme //--------------------------------------------------------------------------------- - for(int i = 0; i < 42; ++i) + for(int i = 0; i < 43; ++i) Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); //--------------------------------------------------------------------------------- @@ -1453,7 +1462,7 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); - + std::cout << " FACTNUM-2 :" << factnum_(0, 0, kb) << std::endl; //--------------------------------------------------------------- // Now update interstitial and cllud borne aerosols //--------------------------------------------------------------- @@ -1467,7 +1476,7 @@ void MAMAci::run_impl(const double dt) { update_interstitial_aerosols(team_policy, ptend_q_, nlev_, dt, // output dry_aero_); - + std::cout << " FACTNUM-1 :" << factnum_(0, 0, kb) << std::endl; // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), @@ -1475,13 +1484,20 @@ void MAMAci::run_impl(const double dt) { qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - + std::cout << " FACTNUM-3 :" << factnum_(0, 0, kb) << std::endl; const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { std::cout << "Somethign changed!!!! :" << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; exit(1); } + std::cout << " FACTNUM-4 :" << factnum_(0, 0, kb) << std::endl; + if((factnum_(0, 0, kb) < 0.998557 || factnum_(0, 0, kb) > 0.998559) || + (factnum_(0, 1, kb) < 0.861768 || factnum_(0, 1, kb) > 0.861770)) { + std::cout << "Somethign changed FACTNUM!!!! :" << factnum_(0, 0, kb) + << std::endl; + exit(1); + } Kokkos::fence(); // wait before returning to calling function } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index c71988185720..678ebfafbf06 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -59,14 +59,13 @@ class MAMAci final : public scream::AtmosphereProcess { // aerosol dry diameter const_view_3d dgnum_; - // FIXME: Make sure these output are used somewhere and add comments about - // these view_2d nihf_; view_2d niim_; view_2d nidep_; view_2d nimey_; view_2d naai_hom_; view_2d naai_; + view_2d kvh_int_; // Eddy diffusivity of heat at the interfaces const_view_2d liqcldf_; const_view_2d liqcldf_prev_; @@ -108,7 +107,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; view_2d hetfrz_depostion_nucleation_tend_; - view_2d diagnostic_scratch_[42]; + view_2d diagnostic_scratch_[43]; // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; @@ -136,7 +135,7 @@ class MAMAci final : public scream::AtmosphereProcess { // A view array to carry cloud borne aerosol mmrs/nmrs // FIXME: 25 should be a const int - view_2d qqcw_fld_work_[25]; + view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot]; // A view to carry interstitial aerosol mmrs/nmrs view_3d state_q_work_; From 387cbf8d1664757ded7c5d81089346a1990f09c0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 18:34:49 -0700 Subject: [PATCH 327/476] Adds updated submodule --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 0f80195e0ff3..00ae716b4188 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 0f80195e0ff30d00d9a9b651e63a4ab4e314a242 +Subproject commit 00ae716b41881eaf47e9c437bdfdd6fc0db93952 From 918786959803cc852a12e678ab17f7ba97e317d2 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 19:46:31 -0700 Subject: [PATCH 328/476] Adds the post processing step --- .../mam/eamxx_mam_aci_process_interface.cpp | 12 +++++- .../mam/eamxx_mam_aci_process_interface.hpp | 39 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ba718835844a..e4a925b98354 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1358,9 +1358,13 @@ void MAMAci::initialize_impl(const RunType run_type) { //--------------------------------------------------------------------------------- // Setup preprocessing //--------------------------------------------------------------------------------- - // set up our preprocess functor + // set up our preprocess and postprocess functors preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + } // end function initialize_impl // ================================================================ @@ -1476,7 +1480,11 @@ void MAMAci::run_impl(const double dt) { update_interstitial_aerosols(team_policy, ptend_q_, nlev_, dt, // output dry_aero_); - std::cout << " FACTNUM-1 :" << factnum_(0, 0, kb) << std::endl; + + // call post processing to convert dry mixing ratios to wet mixing ratios + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); + // FIXME: Remove the following print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 678ebfafbf06..d058a17c0b6e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -212,9 +212,48 @@ class MAMAci final : public scream::AtmosphereProcess { mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; }; // MAMAci::Preprocess + // Atmosphere processes often have a post-processing step prepares output + // from this process for the Field Manager. This functor implements this + // step, which is called during run_impl. + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMMicrophysics::Postprocess + private: // pre- and postprocessing scratch pads Preprocess preprocess_; + Postprocess postprocess_; }; // MAMAci From dc3e08f3a472001c1a9d03ce1d9a8fa4346a525e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 20:44:43 -0700 Subject: [PATCH 329/476] Adds top_lev as a namelist --- .../cime_config/namelist_defaults_scream.xml | 1 + .../mam/eamxx_mam_aci_process_interface.cpp | 19 ++++++---- .../mam/eamxx_mam_aci_process_interface.hpp | 37 ++++++++++++------- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 1 + .../tests/single-process/mam/aci/input.yaml | 1 + 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 2eda717ee06e..49244880280e 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -237,6 +237,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0.001 + 6 diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e4a925b98354..983eeb7dd8d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -521,9 +521,7 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[15]) { - // FIXME: why can't we use MAMAci::dropmix_scratch_ above instead of 15? - + MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -792,8 +790,9 @@ void call_hetfrz_compute_tendencies( mam_coupling::AerosolState dry_aero = dry_aero_; mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; - MAMAci::view_2d diagnostic_scratch[43]; - for(int i = 0; i < 43; ++i) diagnostic_scratch[i] = diagnostic_scratch_[i]; + MAMAci::view_2d diagnostic_scratch[MAMAci::hetro_scratch_]; + for(int i = 0; i < MAMAci::hetro_scratch_; ++i) + diagnostic_scratch[i] = diagnostic_scratch_[i]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -901,6 +900,9 @@ MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) // Asserts for the runtime or namelist options EKAT_REQUIRE_MSG(m_params.isParameter("wsubmin"), "ERROR: wsubmin is missing from mam_aci parameter list."); + EKAT_REQUIRE_MSG( + m_params.isParameter("top_level_mam4xx"), + "ERROR: top_level_mam4xx is missing from mam_aci parameter list."); } // ================================================================ @@ -1124,6 +1126,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // ------------------------------------------------------------------------ wsubmin_ = m_params.get("wsubmin"); + top_lev_ = m_params.get("top_level_mam4xx"); // ------------------------------------------------------------------------ // Input fields read in from IC file, namelist or other processes @@ -1339,7 +1342,7 @@ void MAMAci::initialize_impl(const RunType run_type) { // Diagnotics variables from the hetrozenous ice nucleation scheme //--------------------------------------------------------------------------------- - for(int i = 0; i < 43; ++i) + for(int i = 0; i < hetro_scratch_; ++i) Kokkos::resize(diagnostic_scratch_[i], ncol_, nlev_); //--------------------------------------------------------------------------------- @@ -1492,14 +1495,14 @@ void MAMAci::run_impl(const double dt) { qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - std::cout << " FACTNUM-3 :" << factnum_(0, 0, kb) << std::endl; + std::cout << " FACTNUM-0 :" << factnum_(0, 0, kb) << std::endl; const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); if(ans < 5.65184e-06 || ans > 5.65186e-06) { std::cout << "Somethign changed!!!! :" << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; exit(1); } - std::cout << " FACTNUM-4 :" << factnum_(0, 0, kb) << std::endl; + std::cout << " FACTNUM-1 :" << factnum_(0, 1, kb) << std::endl; if((factnum_(0, 0, kb) < 0.998557 || factnum_(0, 0, kb) > 0.998559) || (factnum_(0, 1, kb) < 0.861768 || factnum_(0, 1, kb) > 0.861770)) { std::cout << "Somethign changed FACTNUM!!!! :" << factnum_(0, 0, kb) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index d058a17c0b6e..0bfbefc118fa 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -19,6 +19,12 @@ namespace scream { class MAMAci final : public scream::AtmosphereProcess { + public: + // declare some constant scratch space lengths + static constexpr int hetro_scratch_ = 43; + static constexpr int dropmix_scratch_ = 15; + + private: using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; @@ -32,11 +38,16 @@ class MAMAci final : public scream::AtmosphereProcess { using const_view_2d = scream::mam_coupling::const_view_2d; using const_view_3d = scream::mam_coupling::const_view_3d; - // FIXME: Should the following variables be public? They are like that in - // micriphysics and optics codes - + //------------------------------------------------------------------------ // ACI runtime ( or namelist) options - Real wsubmin_; + //------------------------------------------------------------------------ + + Real wsubmin_; // Minimum subgrid vertical velocity + int top_lev_; // Top level for MAM4xx + + //------------------------------------------------------------------------ + // END: ACI runtime ( or namelist) options + //------------------------------------------------------------------------ // rho is air density [kg/m3] view_2d rho_; @@ -59,13 +70,17 @@ class MAMAci final : public scream::AtmosphereProcess { // aerosol dry diameter const_view_3d dgnum_; + // ice nucleation diagnostic variables view_2d nihf_; view_2d niim_; view_2d nidep_; view_2d nimey_; view_2d naai_hom_; + + // ice nucleation output for FM view_2d naai_; + // droplet activation inputs and outputs view_2d kvh_int_; // Eddy diffusivity of heat at the interfaces const_view_2d liqcldf_; const_view_2d liqcldf_prev_; @@ -74,7 +89,6 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d cloud_frac_; view_2d cloud_frac_prev_; view_2d qcld_; - view_2d tendnd_; view_2d ptend_q_[mam4::aero_model::pcnst]; view_3d factnum_; const_view_3d qqcw_input_; @@ -94,26 +108,21 @@ class MAMAci final : public scream::AtmosphereProcess { view_3d nact_; view_3d mact_; - - static constexpr int dropmix_scratch_ = 15; view_2d dropmixnuc_scratch_mem_[dropmix_scratch_]; - view_2d stratiform_cloud_fraction_; - view_2d activation_fraction_accum_idx_; - view_2d activation_fraction_coarse_idx_; + // droplet activation output for the FM + view_2d tendnd_; // These are the output tendencies from heterogeneous freezing that need to be // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; view_2d hetfrz_depostion_nucleation_tend_; - view_2d diagnostic_scratch_[43]; + + view_2d diagnostic_scratch_[hetro_scratch_]; // Subgrid scale velocities view_2d wsub_, wsubice_, wsig_, w2_; - // Top level for troposphere cloud physics - // FIXME: This should be read in to make user selectable. - const int top_lev_ = 6; // local atmospheric state column variables const_view_2d pdel_; // pressure thickess of layer [Pa] diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index 5cc2fb8fa653..a8cac4b1a6c7 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -34,6 +34,7 @@ atmosphere_processes: Ckm: 0.1 mam4_aci: wsubmin: 0.001 + top_level_mam4xx: 6 grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index bedc9c583c2a..3e778315bdfc 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -12,6 +12,7 @@ atmosphere_processes: atm_procs_list: [mam4_aci] mam4_aci: wsubmin: 0.001 + top_level_mam4xx: 6 grids_manager: Type: Mesh Free From d6f8a940e768db8315922010ef82e44d115d92f3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 21:10:40 -0700 Subject: [PATCH 330/476] Repaces parallel_for copy logic with deep_copy --- .../mam/eamxx_mam_aci_process_interface.cpp | 49 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.hpp | 1 - 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 983eeb7dd8d4..4689e88ca078 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -71,6 +71,16 @@ void print_output(const Real w0, const Real rho, const Real tke, } } + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + std::cout << "hetfrz_immersion_nucleation_tend_:" << hetfrz_immersion_nucleation_tend << ":" << hetfrz_immersion_nucleation_tend << std::endl; @@ -725,26 +735,17 @@ void update_cloud_borne_aerosols( const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, // output mam_coupling::AerosolState &dry_aero) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - dry_aero.cld_aero_nmr[m](icol, kk) = - qqcw_fld_work[ind_qqcw](icol, kk); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](icol, kk) = - qqcw_fld_work[ind_qqcw](icol, kk); - ++ind_qqcw; - } - } - } - }); - }); + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + } + } + } } // Update interstitial aerosols using tendencies @@ -1381,17 +1382,13 @@ void MAMAci::run_impl(const double dt) { // kb)); const auto scan_policy = ekat::ExeSpaceUtils< - // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, - // nlev_); - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); - + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 0bfbefc118fa..7538451db6ef 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -143,7 +143,6 @@ class MAMAci final : public scream::AtmosphereProcess { std::shared_ptr grid_; // A view array to carry cloud borne aerosol mmrs/nmrs - // FIXME: 25 should be a const int view_2d qqcw_fld_work_[mam4::ndrop::ncnst_tot]; // A view to carry interstitial aerosol mmrs/nmrs From c76b4260fa706a6adbf748a7d258b16a058d03a7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 22 Apr 2024 22:29:48 -0700 Subject: [PATCH 331/476] Add function call to get interface values for wsec and kvh --- .../mam/eamxx_mam_aci_process_interface.cpp | 240 ++++-------------- .../tests/single-process/mam/aci/input.yaml | 7 +- 2 files changed, 51 insertions(+), 196 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4689e88ca078..b98421ca2335 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -4,23 +4,6 @@ namespace scream { namespace { -void print_input(const Real t, const Real p, const Real wsec) { - std::cout << "T_mid:" << t << std::endl; - std::cout << "p_mid:" << p << std::endl; - std::cout << "wsec:" << wsec << std::endl; -} -void print_bef_ndrop(const mam_coupling::AerosolState &dry_aero, const int kb) { - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "Bef-ndrop-cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) - << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "Bef-ndrop-cldbrn-mmr:" - << dry_aero.cld_aero_mmr[m][a](0, kb) << std::endl; - } - } - } -} void print_output(const Real w0, const Real rho, const Real tke, const Real wsub, const Real wice, const Real wsig, const Real naai_hom, const Real naai, const Real rpdel, @@ -90,148 +73,6 @@ void print_output(const Real w0, const Real rho, const Real tke, << hetfrz_depostion_nucleation_tend << std::endl; } -void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, - const int ncol_, const int nlev_) { - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } -} -void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, - const int nlev_) { - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -273,6 +114,29 @@ void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, }); } +void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_2d var_mid, + const MAMAci::view_2d dz, const int nlev_, + // output + MAMAci::view_2d var_int) { + using CO = scream::ColumnOps; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + const auto var_mid_col = ekat::subview(var_mid, icol); + const auto var_int_col = ekat::subview(var_int, icol); + const auto dz_col = ekat::subview(dz, icol); + + const Real bc_top = var_mid_col(0); + const Real bc_bot = var_mid_col(nlev_ - 1); + + CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, + bc_top, bc_bot, var_int_col); + }); +} + KOKKOS_INLINE_FUNCTION void compute_tke_using_w_sec(const haero::ThreadTeam &team, const MAMAci::const_view_2d w_sec, const int icol, @@ -989,11 +853,8 @@ void MAMAci::set_grids( // the current time step. For other dycores (such as EUL), it may be different // and we might need to revisit this. - // FIXME: w_variance in microp_aero_run.F90 is at the interfaces but - // SHOC provides it at the midpoints. Verify how it is being used. - // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -1007,8 +868,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - // FIXME: It is at mid level in EAMxx, we need to compute it at the interfaces - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1360,7 +1220,7 @@ void MAMAci::initialize_impl(const RunType run_type) { hetfrz_.init(aero_config, hetfrz_config); //--------------------------------------------------------------------------------- - // Setup preprocessing + // Setup preprocessing and post processing //--------------------------------------------------------------------------------- // set up our preprocess and postprocess functors preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, @@ -1375,14 +1235,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - // FIXME: Remove set_input and print_input - const int kb = 62; - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - // print_input(dry_atm_.T_mid(0, kb), dry_atm_.p_mid(0, kb), w_sec_int_(0, - // kb)); - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); @@ -1390,10 +1245,19 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + const int itop = + (dry_atm_.p_mid(0, 0) < dry_atm_.p_mid(0, nlev_ - 1)) ? 0 : nlev_ - 1; + std::cout << "ITOP::::::::" << itop << std::endl; + compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); + // Get w_sec_int_ from w_sec_mid_ + compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + // output + w_sec_int_); + compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); @@ -1409,8 +1273,6 @@ void MAMAci::run_impl(const double dt) { compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - // FIXME:Remove set_dgait - set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1436,7 +1298,12 @@ void MAMAci::run_impl(const double dt) { rpdel_); Kokkos::fence(); // wait for rpdel_ to be computed. - // print_bef_ndrop(dry_aero_, kb); + + // Get kvh_int_ from kvh_mid_ + compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + // output + kvh_int_); + // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies @@ -1466,9 +1333,9 @@ void MAMAci::run_impl(const double dt) { hetfrz_depostion_nucleation_tend_, // work arrays diagnostic_scratch_); - std::cout << " FACTNUM-2 :" << factnum_(0, 0, kb) << std::endl; + //--------------------------------------------------------------- - // Now update interstitial and cllud borne aerosols + // Now update interstitial and cloud borne aerosols //--------------------------------------------------------------- // Update cloud borne aerosols @@ -1483,30 +1350,15 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); - Kokkos::fence(); + Kokkos::fence(); // wait before returning to calling function - // FIXME: Remove the following + const int kb = 62; print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), hetfrz_contact_nucleation_tend_(0, kb), hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - std::cout << " FACTNUM-0 :" << factnum_(0, 0, kb) << std::endl; - const Real ans = hetfrz_immersion_nucleation_tend_(0, kb); - if(ans < 5.65184e-06 || ans > 5.65186e-06) { - std::cout << "Somethign changed!!!! :" - << hetfrz_immersion_nucleation_tend_(0, kb) << std::endl; - exit(1); - } - std::cout << " FACTNUM-1 :" << factnum_(0, 1, kb) << std::endl; - if((factnum_(0, 0, kb) < 0.998557 || factnum_(0, 0, kb) > 0.998559) || - (factnum_(0, 1, kb) < 0.861768 || factnum_(0, 1, kb) > 0.861770)) { - std::cout << "Somethign changed FACTNUM!!!! :" << factnum_(0, 0, kb) - << std::endl; - exit(1); - } - Kokkos::fence(); // wait before returning to calling function } } // namespace scream \ No newline at end of file diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 3e778315bdfc..2c4172b0da11 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -26,14 +26,17 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file #we should get the following variables from other processes pbl_height : 1.0 - dgnum: 1e-3 + dgnum: 4.362354500358337E-008 + cldfrac_liq_prev: 0.138584624960092 + eddy_diff_heat: 0.620820118789603 + w_variance: 1.110977269289875E-002 # The parameters for I/O control Scorpio: From 21cba9ddfa8ba187ebd0369464324f046c0a3606 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 13:08:19 -0700 Subject: [PATCH 332/476] Removes unused field tags and add missing namelist vars in the intput.yaml files --- .../mam/eamxx_mam_aci_process_interface.cpp | 24 +++++++--- .../eamxx/src/share/field/field_tag.hpp | 45 +++++++------------ .../input.yaml | 3 ++ .../input.yaml | 3 ++ .../physics_only/mam/shoc_mam4_aci/input.yaml | 3 ++ 5 files changed, 41 insertions(+), 37 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b98421ca2335..d285a42748c8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,5 +1,19 @@ #include +/* NOTES: +1. w_variance assumes that we are using SE dycore. If we use EUL +dycore, we need to get its value from previous dynamic time step + +2. w_variance and eddy_diff_heat are at midpoints in EAMxx, we +derived their interface values using boundary conditions to be +the top and botton values of the mid point arrays. We assume that +this assumption should not cause any issues. + +FUTURE WORK: +1. MAM4xx submodule should point to MAM4xx main branch +2. Link hetrozenous freezing outputs to microphysics +*/ + namespace scream { namespace { @@ -949,15 +963,15 @@ void MAMAci::set_grids( // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); n_unit.set_string("1(cm^-3 s^-1)"); - // heterogeous freezing by immersion nucleation [cm^-3 s^-1] + // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); - // heterogeous freezing by contact nucleation [cm^-3 s^-1] + // heterogeneous freezing by contact nucleation [cm^-3 s^-1] add_field("hetfrz_contact_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); - // heterogeous freezing by deposition nucleation [cm^-3 s^-1] + // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); } // function set_grids ends @@ -1245,10 +1259,6 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - const int itop = - (dry_atm_.p_mid(0, 0) < dry_atm_.p_mid(0, nlev_ - 1)) ? 0 : nlev_ - 1; - std::cout << "ITOP::::::::" << itop << std::endl; - compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index 7e6b0b83412a..a2e196621629 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,15 +34,12 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - num_modes, - num_constituents, - num_aero_const, - psat, // - nrefindex_real, - nrefindex_im, - ncoef_number, - mode + MAM_NumModes, + MAM_NumRefIndexReal, + MAM_NumRefIndexImag, + MAM_NumCoefficients, + MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -59,16 +56,13 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - constexpr auto NMODES = FieldTag::num_modes; - constexpr auto MAM_NCNST = FieldTag::num_constituents; - constexpr auto MAM_AERO_NCNST = FieldTag::num_aero_const; - constexpr auto MAM_PSAT = FieldTag::psat; + constexpr auto NMODES = FieldTag::MAM_NumModes; // - constexpr auto NREFINDEX_REAL = FieldTag::nrefindex_real; - constexpr auto NREFINDEX_IM = FieldTag::nrefindex_im; + constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; + constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; - constexpr auto NCOEF_NUMBER = FieldTag::ncoef_number; - constexpr auto MODE = FieldTag::mode; + constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; + constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -99,28 +93,19 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - case FieldTag::num_modes: + case FieldTag::MAM_NumModes: name = "num_modes"; break; - case FieldTag::num_constituents: - name = "num_constituents"; - break; - case FieldTag::num_aero_const: - name = "num_aero_const"; - break; - case FieldTag::psat: - name = "psat"; - break; - case FieldTag::nrefindex_real: + case FieldTag::MAM_NumRefIndexReal: name = "refindex_real"; break; - case FieldTag::nrefindex_im: + case FieldTag::MAM_NumRefIndexImag: name = "refindex_im"; break; - case FieldTag::ncoef_number: + case FieldTag::MAM_NumCoefficients: name = "coef_number"; break; - case FieldTag::mode: + case FieldTag::MAM_NumModesInFile: name = "mode"; break; default: diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml index f3198b1308a0..20d6f142c52a 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/input.yaml @@ -16,6 +16,9 @@ atmosphere_processes: Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 p3: max_total_ni: 740.0e3 do_prescribed_ccn: false diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml index d61f1318f8f4..7ec317eda928 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/input.yaml @@ -16,6 +16,9 @@ atmosphere_processes: Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 p3: max_total_ni: 740.0e3 do_prescribed_ccn: false diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml index 1d6d4f48eba7..ae997bb3da57 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/input.yaml @@ -16,6 +16,9 @@ atmosphere_processes: Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 shoc: lambda_low: 0.001 lambda_high: 0.04 From a4be737029decba2452592c6c31bfcb50c6536b7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 13:23:55 -0700 Subject: [PATCH 333/476] Adds a CIME test --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 9 +++++++++ .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands new file mode 100644 index 000000000000..cde7694f5f35 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -0,0 +1,9 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41 +./xmlchange SCREAM_CMAKE_OPTIONS="`./xmlquery --value SCREAM_CMAKE_OPTIONS | sed 's/SCREAM_NUM_TRACERS [0-9][0-9]*/SCREAM_NUM_TRACERS 41/'`" + +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d285a42748c8..d2ddf4436ad2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -4,7 +4,7 @@ 1. w_variance assumes that we are using SE dycore. If we use EUL dycore, we need to get its value from previous dynamic time step -2. w_variance and eddy_diff_heat are at midpoints in EAMxx, we +2. w_variance and eddy_diff_heat are at midpoints in EAMxx, we derived their interface values using boundary conditions to be the top and botton values of the mid point arrays. We assume that this assumption should not cause any issues. From cd1f99a3fc6cbb21024c9bb4eaee241c33c6158a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 14:10:04 -0700 Subject: [PATCH 334/476] Removes print statements and recomputes dry/wet mmrs in mam_coupling --- .../mam/eamxx_mam_aci_process_interface.cpp | 82 +------------------ .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 ++--- 3 files changed, 15 insertions(+), 89 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index d2ddf4436ad2..f1e7b320cb68 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,6 +1,8 @@ #include -/* NOTES: +/* +----------------------------------------------------------------- +NOTES: 1. w_variance assumes that we are using SE dycore. If we use EUL dycore, we need to get its value from previous dynamic time step @@ -12,81 +14,13 @@ this assumption should not cause any issues. FUTURE WORK: 1. MAM4xx submodule should point to MAM4xx main branch 2. Link hetrozenous freezing outputs to microphysics +----------------------------------------------------------------- */ namespace scream { namespace { -void print_output(const Real w0, const Real rho, const Real tke, - const Real wsub, const Real wice, const Real wsig, - const Real naai_hom, const Real naai, const Real rpdel, - MAMAci::view_3d factnum, const Real tendnd, - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - const Real hetfrz_immersion_nucleation_tend, - const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, - const mam_coupling::AerosolState &dry_aero, const int kb) { - std::cout << "w0:" << w0 << std::endl; - std::cout << " rho: " << rho << std::endl; - std::cout << "TKE:" << tke << std::endl; - std::cout << "WSUB:" << wsub << std::endl; - std::cout << "WICE:" << wice << std::endl; - std::cout << "WSIG:" << wsig << std::endl; - std::cout << "naai_hom_:" << naai_hom << std::endl; - std::cout << "naai_:" << naai << std::endl; - std::cout << "rpdel_:" << rpdel << std::endl; - std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) - << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) - << std::endl; - std::cout << "tendnd_:" << tendnd << std::endl; - for(int ic = 9; ic < 40; ++ic) { - std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; - } - for(int ic = 0; ic < 25; ++ic) { - std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) - << std::endl; - } - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend << ":" - << hetfrz_immersion_nucleation_tend << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend << std::endl; -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -1361,14 +1295,6 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function - - const int kb = 62; - print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), - wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), - rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, - qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), - hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); } } // namespace scream \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 7538451db6ef..974907ee8c02 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -256,7 +256,7 @@ class MAMAci final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_post_; mam_coupling::DryAtmosphere dry_atm_post_; mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; - }; // MAMMicrophysics::Postprocess + }; // Postprocess private: // pre- and postprocessing scratch pads diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 25cc9d5de9de..6b5cfbd186d1 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -644,11 +644,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -669,21 +669,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);// PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } From 94a80d9664db5a69538be91011ecebf2a5bc175d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 14:56:30 -0700 Subject: [PATCH 335/476] Revives small kernels shoc function, changes shell cmd for CIME test, fix test runtimes --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 4 ++-- .../physics/shoc/disp/shoc_assign_2d_view_disp.cpp | 4 ++-- .../physics/shoc/eamxx_shoc_process_interface.cpp | 12 ++++++------ .../eamxx/src/physics/shoc/impl/shoc_main_impl.hpp | 2 +- .../mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 3 +-- .../tests/single-process/mam/aci/CMakeLists.txt | 2 +- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index cde7694f5f35..3bffab45ba68 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -1,7 +1,7 @@ #Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41 -./xmlchange SCREAM_CMAKE_OPTIONS="`./xmlquery --value SCREAM_CMAKE_OPTIONS | sed 's/SCREAM_NUM_TRACERS [0-9][0-9]*/SCREAM_NUM_TRACERS 41/'`" +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b diff --git a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp index 8f5592a38360..e6cd04054d0a 100644 --- a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp +++ b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp @@ -13,7 +13,7 @@ ::shoc_assign_2d_view_disp( const view_2d& input_view, const view_2d& output_view) { - /*using ExeSpace = typename KT::ExeSpace; + using ExeSpace = typename KT::ExeSpace; const auto nlev_packs = ekat::npack(nlev); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); @@ -22,7 +22,7 @@ ::shoc_assign_2d_view_disp( Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { output_view (i,k) = input_view(i,k); }); - });*/ + }); } } // namespace shoc diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 6059a4369ed9..25c2674c9121 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -80,12 +80,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); // Output variables add_field("pbl_height", scalar2d_layout_col, m, grid_name); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 34a0455c14ff..ca163a49ddd8 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -266,7 +266,7 @@ void Functions::shoc_main_internal( check_tke(team,nlev,tke); } - // End SHOC parameterization + // End SHOC parameterization // Use SHOC outputs to update the host model // temperature diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt index 81eaa3ae3e4c..946d68cb7501 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -13,8 +13,7 @@ CreateADUnitTest(${TEST_BASE_NAME} # Set AD configurable options set (ATM_TIME_STEP 1800) -#SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) # 1h 4h 24h +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h set (RUN_T0 2021-10-12-45000) # Determine num subcycles needed to keep shoc dt<=300s diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 95da48b7fa44..b175e6f5e7ce 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,7 +12,7 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) From 888fdb2fbc2b0cfb05e743c8b4a3976c6387ff93 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 23 Apr 2024 16:43:08 -0700 Subject: [PATCH 336/476] Adds int def after reabse --- components/eamxx/src/physics/mam/mam_coupling.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 6b5cfbd186d1..c2fcfb6a8eac 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -35,6 +35,8 @@ using uview_2d = typename ekat::template Unmanaged; +using view_int_1d = typename KT::template view_1d; + // number of constituents in gas chemistry "work arrays" KOKKOS_INLINE_FUNCTION constexpr int gas_pcnst() { From 7e33cdac4862980b4b7afbed4771cbe9d7d6c723 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Apr 2024 14:58:39 -0700 Subject: [PATCH 337/476] Fixes GNU compilation: Make views public and removed an unused variable --- .../mam/eamxx_mam_aci_process_interface.cpp | 3 +-- .../mam/eamxx_mam_aci_process_interface.hpp | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index f1e7b320cb68..016e0c5b5fe0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -473,7 +473,6 @@ void call_function_dropmixnuc( KOKKOS_LAMBDA(int klev) { // for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - Real qqcw_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) // NOTE: The order of species in state_q_at_lev_col @@ -1297,4 +1296,4 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait before returning to calling function } -} // namespace scream \ No newline at end of file +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 974907ee8c02..394ab89f21ed 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -24,19 +24,21 @@ class MAMAci final : public scream::AtmosphereProcess { static constexpr int hetro_scratch_ = 43; static constexpr int dropmix_scratch_ = 15; + // views for multi-column data + using view_2d = scream::mam_coupling::view_2d; + using const_view_2d = scream::mam_coupling::const_view_2d; + using view_3d = scream::mam_coupling::view_3d; + using const_view_3d = scream::mam_coupling::const_view_3d; + private: using KT = ekat::KokkosTypes; mam4::NucleateIce nucleate_ice_; mam4::Hetfrz hetfrz_; - // views for single- and multi-column data + // views for single-column data using view_1d = scream::mam_coupling::view_1d; - using view_2d = scream::mam_coupling::view_2d; - using view_3d = scream::mam_coupling::view_3d; using const_view_1d = scream::mam_coupling::const_view_1d; - using const_view_2d = scream::mam_coupling::const_view_2d; - using const_view_3d = scream::mam_coupling::const_view_3d; //------------------------------------------------------------------------ // ACI runtime ( or namelist) options @@ -153,19 +155,19 @@ class MAMAci final : public scream::AtmosphereProcess { MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // Process metadata: Return type of the process - AtmosphereProcessType MAMAci::type() const { + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } // Return name of the process - std::string MAMAci::name() const { return "mam4_aci"; } + std::string name() const { return "mam4_aci"; } // grid void set_grids( const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t MAMAci::requested_buffer_size_in_bytes() const { + size_t requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } From 9f54b0d5f593dd58c9995ff0d2c73a5153939adb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Apr 2024 17:00:32 -0700 Subject: [PATCH 338/476] PM-GPU: Fixes nested Kokkos loops compilation --- .../physics/mam/eamxx_mam_aci_process_interface.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 016e0c5b5fe0..3b4ce1521352 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -469,9 +469,9 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - KOKKOS_LAMBDA(int klev) { - // for(int klev = 0; klev < mam4::ndrop::pver; ++klev) { + Kokkos::TeamThreadRange(team, 0u, 72),//mam4::ndrop::pver), + [=](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -570,7 +570,7 @@ void update_interstitial_aerosols( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamThreadRange(team, nlev), [=](int kk) { int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { for(int a = 0; a < mam4::num_species_mode(m); ++a) { @@ -686,7 +686,7 @@ void call_hetfrz_compute_tendencies( // assign cloud fraction Kokkos::parallel_for( Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - KOKKOS_LAMBDA(int klev) { + [=](int klev) { diags.stratiform_cloud_fraction(klev) = haero_atm.cloud_fraction(klev); }); From 365325c70ac14ec77dd2a1dae2302ad9dcba0ce9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Apr 2024 17:17:19 -0700 Subject: [PATCH 339/476] Removes a debug change --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 3b4ce1521352..9c0c99cea066 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -469,7 +469,7 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, 72),//mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; From 727cc8ac80fc36d5256cc8c7d7cc07e30dc8ce01 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 11:55:01 -0700 Subject: [PATCH 340/476] Changes needed for single cell validation --- .../mam/eamxx_mam_aci_process_interface.cpp | 416 ++++++++++++++++-- .../mam/eamxx_mam_aci_process_interface.hpp | 4 +- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 +- .../single-process/mam/aci/CMakeLists.txt | 3 +- .../tests/single-process/mam/aci/input.yaml | 11 +- 5 files changed, 403 insertions(+), 51 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9c0c99cea066..9e3c272ef895 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -21,6 +21,217 @@ namespace scream { namespace { +void print_output(const Real w0, const Real rho, const Real tke, + const Real wsub, const Real wice, const Real wsig, + const Real naai_hom, const Real naai, const Real rpdel, + MAMAci::view_3d factnum, const Real tendnd, + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + const Real hetfrz_immersion_nucleation_tend, + const Real hetfrz_contact_nucleation_tend, + const Real hetfrz_depostion_nucleation_tend, + const mam_coupling::AerosolState &dry_aero, const int kb) { + std::cout << "w0:" << w0 << std::endl; + std::cout << " rho: " << rho << std::endl; + std::cout << "TKE:" << tke << std::endl; + std::cout << "WSUB:" << wsub << std::endl; + std::cout << "WICE:" << wice << std::endl; + std::cout << "WSIG:" << wsig << std::endl; + std::cout << "naai_hom_:" << naai_hom << std::endl; + std::cout << "naai_:" << naai << std::endl; + std::cout << "rpdel_:" << rpdel << std::endl; + std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) + << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) + << std::endl; + std::cout << "tendnd_:" << tendnd << std::endl; + for(int ic = 9; ic < 40; ++ic) { + std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; + } + for(int ic = 0; ic < 25; ++ic) { + std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) + << std::endl; + } + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend << ":" + << hetfrz_immersion_nucleation_tend << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend << std::endl; +} + +void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, + const int ncol_, const int nlev_) { + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; + } + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; + } + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; + } +} +void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, + const int nlev_) { + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } +} + KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -469,9 +680,8 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -559,33 +769,61 @@ void update_cloud_borne_aerosols( } } -// Update interstitial aerosols using tendencies +// Update interstitial aerosols using tendencies - levels +void update_interstitial_aerosols_levs( + const haero::ThreadTeam &team, const int nlev, const int icol, + const int s_idx, const Real dt, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + // output + MAMAci::view_2d aero_mr) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { + aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; + }); +} + +// Update interstitial aerosols using tendencies- cols and levs void update_interstitial_aerosols( haero::ThreadTeamPolicy team_policy, const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, const Real dt, // output mam_coupling::AerosolState &dry_aero) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); + // starting index of ptend_q array (for MAM4, pcnst=40, ncnst_tot=25 ) + int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // loop through all modes and species + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + // mass mixing ratio of species "a" of mode "m" + auto aero_mmr = dry_aero.int_aero_mmr[m][a]; + + if(aero_mmr.data()) { Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, nlev), [=](int kk) { - int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](icol, kk) += - ptend_q[s_idx](icol, kk) * dt; - s_idx++; - } - } - dry_aero.int_aero_nmr[m](icol, kk) += - ptend_q[s_idx](icol, kk) * dt; - s_idx++; - } + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_mmr); }); - }); + // update index for the next species (only if aero_mmr.data() is True) + ++s_idx; + } + } + auto aero_nmr = + dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_nmr); + }); + ++s_idx; // update index for the next species + } } void call_hetfrz_compute_tendencies( @@ -801,7 +1039,8 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + // add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -815,7 +1054,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1182,6 +1421,9 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { + // FIXME: Remove set_input and print_input + set_input(w_sec_int_, kvh_int_, ncol_, nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1190,16 +1432,17 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_); + w_sec_int_);*/ compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1218,6 +1461,8 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + // FIXME:Remove set_dgait + set_dgait(aitken_dry_dia_, ncol_, nlev_); // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is @@ -1243,9 +1488,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, - // output - kvh_int_); + // compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + // // output + // kvh_int_); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1291,9 +1536,122 @@ void MAMAci::run_impl(const double dt) { // output dry_aero_); + const int kb = 62; + print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), + wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), + rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, + qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), + hetfrz_contact_nucleation_tend_(0, kb), + hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); + // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function } } // namespace scream + +/* FORTRAN output + + AT THE END OF MICROP_AERO---- + ----- OUPUT AT time step: 6 6 + wo 3.343177351577457E-004 + rho: 1.28186433384113 + TKE: 1.666465903934813E-002 6.097664104873541E-002 + wsub: 0.160873967324407 + wsubi: 0.200000000000000 + wsig: 0.160873967324407 + dgnum_ait: 4.362354500358337E-008 + naai: 11464.5957222634 + naai_hom: 29838.6879763933 + factnum: 0.998556176544263 0.861689826773470 + 0.999999999974140 0.000000000000000E+000 + nctend_mixnuc: -839.404918218856 + O3_state: 7.467031959984916E-008 + H2O2_state: 9.349365721024434E-013 + H2SO4_state: 6.828290138182288E-015 + SO2_state: 5.331014679199532E-012 + DMS_state: 2.308803742493950E-012 + SOAG_state: 5.353788713507973E-012 + so4_a1_state: 2.725322171440633E-011 + pom_a1_state: 5.550204151691568E-012 + soa_a1_state: 4.265179209409266E-011 + bc_a1_state: 4.464761667085421E-013 + dst_a1_state: 4.822960238167224E-013 + ncl_a1_state: 1.572123611679146E-013 + mom_a1_state: 1.295791882646146E-014 + num_a1_state: 3113980.27259761 + so4_a2_state: 5.931234261210945E-014 + soa_a2_state: 5.442455780020074E-015 + ncl_a2_state: 1.536147864466575E-015 + mom_a2_state: 1.003300717028814E-016 + num_a2_state: 302994.843261936 + dst_a3_state: 1.680288936900572E-011 + ncl_a3_state: 2.766061699712112E-012 + so4_a3_state: 2.666433567899354E-013 + bc_a3_state: 2.232845976874488E-015 + pom_a3_state: 6.726406544964171E-015 + soa_a3_state: 5.446646976638719E-014 + mom_a3_state: 4.675942808467773E-016 + num_a3_state: 889.148316994625 + pom_a4_state: 2.796218924673357E-012 + bc_a4_state: 2.713331090178062E-013 + mom_a4_state: 1.155299651446152E-018 + num_a4_state: 2110535.03879661 + O3_tend: 0.000000000000000E+000 + H2O2_tend: 0.000000000000000E+000 + H2SO4_tend: 0.000000000000000E+000 + SO2_tend: 0.000000000000000E+000 + DMS_tend: 0.000000000000000E+000 + SOAG_tend: 0.000000000000000E+000 + so4_a1_tend: -1.384310943569190E-014 + pom_a1_tend: -2.950799002682271E-015 + soa_a1_tend: -2.283869371271150E-014 + bc_a1_tend: -2.202672700280516E-016 + dst_a1_tend: -2.194060414083922E-016 + ncl_a1_tend: -6.981124745267083E-017 + mom_a1_tend: -5.662690535048673E-018 + num_a1_tend: -1626.38730532537 + so4_a2_tend: -2.548707408341951E-017 + soa_a2_tend: -1.929990276139139E-018 + ncl_a2_tend: -6.855087738119723E-019 + mom_a2_tend: -4.571691604185304E-020 + num_a2_tend: -111.890330245159 + dst_a3_tend: -7.609611333715698E-015 + ncl_a3_tend: -1.256350252705383E-015 + so4_a3_tend: -1.320988655880598E-016 + bc_a3_tend: -1.099586316024402E-018 + pom_a3_tend: -3.309939677824050E-018 + soa_a3_tend: -2.767668335981830E-017 + mom_a3_tend: -2.254911604324925E-019 + num_a3_tend: -0.406788905791186 + pom_a4_tend: -2.375070578317096E-017 + bc_a4_tend: 4.082479713528180E-017 + mom_a4_tend: -4.901356204764327E-023 + num_a4_tend: -34.4232851017138 + At the end:num_c1 1322558.40616450 + At the end:so4_c1 1.233997159684093E-011 + At the end:pom_c1 2.393397505793018E-012 + At the end:soa_c1 1.872520079842212E-011 + At the end:bc_c1 1.922157644491776E-013 + At the end:dst_c1 2.530385880698310E-013 + At the end:ncl_c1 8.315014074126540E-014 + At the end:mom_c1 6.895143982427237E-015 + At the end:num_c2 142214.467855250 + At the end:so4_c2 3.181118913367564E-014 + At the end:soa_c2 2.996638282752921E-015 + At the end:ncl_c2 8.055333842809433E-016 + At the end:mom_c2 5.202632275847699E-017 + At the end:num_c3 464.524744048298 + At the end:dst_c3 8.814810290885256E-012 + At the end:ncl_c3 1.417415096039318E-012 + At the end:so4_c3 1.284022582681906E-013 + At the end:bc_c3 1.081226492056768E-015 + At the end:pom_c3 3.292708392236485E-015 + At the end:soa_c3 2.584587804061707E-014 + At the end:mom_c3 2.324887367507579E-016 + At the end:num_c4 6.934165370527951E-014 + At the end:pom_c4 2.703924752644086E-026 + At the end:bc_c4 3.684880329580097E-027 + At the end:mom_c4 1.049556651713384E-032 +*/ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 394ab89f21ed..3f6f7a093142 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -155,9 +155,7 @@ class MAMAci final : public scream::AtmosphereProcess { MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // Process metadata: Return type of the process - AtmosphereProcessType type() const { - return AtmosphereProcessType::Physics; - } + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } // Return name of the process std::string name() const { return "mam4_aci"; } diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index c2fcfb6a8eac..fc5077ead496 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -646,11 +646,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -671,21 +671,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index b175e6f5e7ce..7d77ba718fe3 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,7 +12,8 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +#SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +SetVarDependingOnTestSize(NUM_STEPS 1 1 1) set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index 2c4172b0da11..dd164585b6bf 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -26,17 +26,12 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file - - #we should get the following variables from other processes - pbl_height : 1.0 - dgnum: 4.362354500358337E-008 - cldfrac_liq_prev: 0.138584624960092 - eddy_diff_heat: 0.620820118789603 - w_variance: 1.110977269289875E-002 + pbl_height: 1.0 + dgnum: 0.1 # The parameters for I/O control Scorpio: From 8f237ffe01a46b02c7ba44cd402f43c27f177744 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 14:00:58 -0700 Subject: [PATCH 341/476] Fixes GPU runtime error about illegal memory access --- .../mam/eamxx_mam_aci_process_interface.cpp | 350 +----------------- .../eamxx/src/physics/mam/mam_coupling.hpp | 20 +- .../single-process/mam/aci/CMakeLists.txt | 5 +- .../tests/single-process/mam/aci/input.yaml | 13 +- 4 files changed, 31 insertions(+), 357 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 9e3c272ef895..64e0e3420e80 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -21,217 +21,6 @@ namespace scream { namespace { -void print_output(const Real w0, const Real rho, const Real tke, - const Real wsub, const Real wice, const Real wsig, - const Real naai_hom, const Real naai, const Real rpdel, - MAMAci::view_3d factnum, const Real tendnd, - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - const Real hetfrz_immersion_nucleation_tend, - const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, - const mam_coupling::AerosolState &dry_aero, const int kb) { - std::cout << "w0:" << w0 << std::endl; - std::cout << " rho: " << rho << std::endl; - std::cout << "TKE:" << tke << std::endl; - std::cout << "WSUB:" << wsub << std::endl; - std::cout << "WICE:" << wice << std::endl; - std::cout << "WSIG:" << wsig << std::endl; - std::cout << "naai_hom_:" << naai_hom << std::endl; - std::cout << "naai_:" << naai << std::endl; - std::cout << "rpdel_:" << rpdel << std::endl; - std::cout << "factnum_:" << factnum(0, 0, kb) << " : " << factnum(0, 1, kb) - << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) - << std::endl; - std::cout << "tendnd_:" << tendnd << std::endl; - for(int ic = 9; ic < 40; ++ic) { - std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; - } - for(int ic = 0; ic < 25; ++ic) { - std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) - << std::endl; - } - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend << ":" - << hetfrz_immersion_nucleation_tend << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend << std::endl; -} - -void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, - const int ncol_, const int nlev_) { - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } -} -void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, - const int nlev_) { - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -680,8 +469,9 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -1039,8 +829,7 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - // add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); - add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -1054,7 +843,7 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); @@ -1421,9 +1210,6 @@ void MAMAci::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMAci::run_impl(const double dt) { - // FIXME: Remove set_input and print_input - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -1432,17 +1218,16 @@ void MAMAci::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_);*/ + w_sec_int_); compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1461,8 +1246,6 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. - // FIXME:Remove set_dgait - set_dgait(aitken_dry_dia_, ncol_, nlev_); // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is @@ -1488,9 +1271,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - // compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, - // // output - // kvh_int_); + compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + // output + kvh_int_); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1536,122 +1319,9 @@ void MAMAci::run_impl(const double dt) { // output dry_aero_); - const int kb = 62; - print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), - wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), - rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, - qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), - hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); - // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function } } // namespace scream - -/* FORTRAN output - - AT THE END OF MICROP_AERO---- - ----- OUPUT AT time step: 6 6 - wo 3.343177351577457E-004 - rho: 1.28186433384113 - TKE: 1.666465903934813E-002 6.097664104873541E-002 - wsub: 0.160873967324407 - wsubi: 0.200000000000000 - wsig: 0.160873967324407 - dgnum_ait: 4.362354500358337E-008 - naai: 11464.5957222634 - naai_hom: 29838.6879763933 - factnum: 0.998556176544263 0.861689826773470 - 0.999999999974140 0.000000000000000E+000 - nctend_mixnuc: -839.404918218856 - O3_state: 7.467031959984916E-008 - H2O2_state: 9.349365721024434E-013 - H2SO4_state: 6.828290138182288E-015 - SO2_state: 5.331014679199532E-012 - DMS_state: 2.308803742493950E-012 - SOAG_state: 5.353788713507973E-012 - so4_a1_state: 2.725322171440633E-011 - pom_a1_state: 5.550204151691568E-012 - soa_a1_state: 4.265179209409266E-011 - bc_a1_state: 4.464761667085421E-013 - dst_a1_state: 4.822960238167224E-013 - ncl_a1_state: 1.572123611679146E-013 - mom_a1_state: 1.295791882646146E-014 - num_a1_state: 3113980.27259761 - so4_a2_state: 5.931234261210945E-014 - soa_a2_state: 5.442455780020074E-015 - ncl_a2_state: 1.536147864466575E-015 - mom_a2_state: 1.003300717028814E-016 - num_a2_state: 302994.843261936 - dst_a3_state: 1.680288936900572E-011 - ncl_a3_state: 2.766061699712112E-012 - so4_a3_state: 2.666433567899354E-013 - bc_a3_state: 2.232845976874488E-015 - pom_a3_state: 6.726406544964171E-015 - soa_a3_state: 5.446646976638719E-014 - mom_a3_state: 4.675942808467773E-016 - num_a3_state: 889.148316994625 - pom_a4_state: 2.796218924673357E-012 - bc_a4_state: 2.713331090178062E-013 - mom_a4_state: 1.155299651446152E-018 - num_a4_state: 2110535.03879661 - O3_tend: 0.000000000000000E+000 - H2O2_tend: 0.000000000000000E+000 - H2SO4_tend: 0.000000000000000E+000 - SO2_tend: 0.000000000000000E+000 - DMS_tend: 0.000000000000000E+000 - SOAG_tend: 0.000000000000000E+000 - so4_a1_tend: -1.384310943569190E-014 - pom_a1_tend: -2.950799002682271E-015 - soa_a1_tend: -2.283869371271150E-014 - bc_a1_tend: -2.202672700280516E-016 - dst_a1_tend: -2.194060414083922E-016 - ncl_a1_tend: -6.981124745267083E-017 - mom_a1_tend: -5.662690535048673E-018 - num_a1_tend: -1626.38730532537 - so4_a2_tend: -2.548707408341951E-017 - soa_a2_tend: -1.929990276139139E-018 - ncl_a2_tend: -6.855087738119723E-019 - mom_a2_tend: -4.571691604185304E-020 - num_a2_tend: -111.890330245159 - dst_a3_tend: -7.609611333715698E-015 - ncl_a3_tend: -1.256350252705383E-015 - so4_a3_tend: -1.320988655880598E-016 - bc_a3_tend: -1.099586316024402E-018 - pom_a3_tend: -3.309939677824050E-018 - soa_a3_tend: -2.767668335981830E-017 - mom_a3_tend: -2.254911604324925E-019 - num_a3_tend: -0.406788905791186 - pom_a4_tend: -2.375070578317096E-017 - bc_a4_tend: 4.082479713528180E-017 - mom_a4_tend: -4.901356204764327E-023 - num_a4_tend: -34.4232851017138 - At the end:num_c1 1322558.40616450 - At the end:so4_c1 1.233997159684093E-011 - At the end:pom_c1 2.393397505793018E-012 - At the end:soa_c1 1.872520079842212E-011 - At the end:bc_c1 1.922157644491776E-013 - At the end:dst_c1 2.530385880698310E-013 - At the end:ncl_c1 8.315014074126540E-014 - At the end:mom_c1 6.895143982427237E-015 - At the end:num_c2 142214.467855250 - At the end:so4_c2 3.181118913367564E-014 - At the end:soa_c2 2.996638282752921E-015 - At the end:ncl_c2 8.055333842809433E-016 - At the end:mom_c2 5.202632275847699E-017 - At the end:num_c3 464.524744048298 - At the end:dst_c3 8.814810290885256E-012 - At the end:ncl_c3 1.417415096039318E-012 - At the end:so4_c3 1.284022582681906E-013 - At the end:bc_c3 1.081226492056768E-015 - At the end:pom_c3 3.292708392236485E-015 - At the end:soa_c3 2.584587804061707E-014 - At the end:mom_c3 2.324887367507579E-016 - At the end:num_c4 6.934165370527951E-014 - At the end:pom_c4 2.703924752644086E-026 - At the end:bc_c4 3.684880329580097E-027 - At the end:mom_c4 1.049556651713384E-032 -*/ \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index fc5077ead496..c2fcfb6a8eac 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -646,11 +646,11 @@ void compute_dry_mixing_ratios(const Team& team, int i = column_index; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); - dry_atm.qv(i,k) = wet_atm.qv(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); - dry_atm.qc(i,k) = wet_atm.qc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); - dry_atm.nc(i,k) = wet_atm.nc(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); - dry_atm.qi(i,k) = wet_atm.qi(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); - dry_atm.ni(i,k) = wet_atm.ni(i,k);//PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); }); } @@ -671,21 +671,21 @@ void compute_dry_mixing_ratios(const Team& team, Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { const auto qv_ik = wet_atm.qv(i,k); for (int m = 0; m < num_aero_modes(); ++m) { - dry_aero.int_aero_nmr[m](i,k) = wet_aero.int_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); if (dry_aero.cld_aero_nmr[m].data()) { - dry_aero.cld_aero_nmr[m](i,k) = wet_aero.cld_aero_nmr[m](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); } for (int a = 0; a < num_aero_species(); ++a) { if (dry_aero.int_aero_mmr[m][a].data()) { - dry_aero.int_aero_mmr[m][a](i,k) = wet_aero.int_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); } if (dry_aero.cld_aero_mmr[m][a].data()) { - dry_aero.cld_aero_mmr[m][a](i,k) = wet_aero.cld_aero_mmr[m][a](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); } } } for (int g = 0; g < num_aero_gases(); ++g) { - dry_aero.gas_mmr[g](i,k) = wet_aero.gas_mmr[g](i,k);//PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); } }); } diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 7d77ba718fe3..8c5028828bb1 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,8 +12,7 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -#SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -SetVarDependingOnTestSize(NUM_STEPS 1 1 1) +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) set (ATM_TIME_STEP 600) set (RUN_T0 2021-10-12-45000) @@ -43,4 +42,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # Note: one is enough, since we already check that np1 is BFB with npX set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) -endif() \ No newline at end of file +endif() diff --git a/components/eamxx/tests/single-process/mam/aci/input.yaml b/components/eamxx/tests/single-process/mam/aci/input.yaml index dd164585b6bf..fe26d828d41d 100644 --- a/components/eamxx/tests/single-process/mam/aci/input.yaml +++ b/components/eamxx/tests/single-process/mam/aci/input.yaml @@ -26,14 +26,19 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 #These should come from the input file - pbl_height: 1.0 - dgnum: 0.1 + + #we should get the following variables from other processes + pbl_height : 1.0 + dgnum: 4.362354500358337E-008 + cldfrac_liq_prev: 0.138584624960092 + eddy_diff_heat: 0.620820118789603 + w_variance: 1.110977269289875E-002 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] -... \ No newline at end of file +... From 6c77406b20370449de6f082cf661c98288605f0f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 14:09:49 -0700 Subject: [PATCH 342/476] Clang format --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 64e0e3420e80..b9c8ae9e8f85 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -469,9 +469,8 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), [=](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) From c047515ad32f6da84fb75ccf172239f77d6e2c57 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 30 Apr 2024 20:18:18 -0700 Subject: [PATCH 343/476] After rebase changes, adds make_layout for aci --- .../mam/eamxx_mam_aci_process_interface.cpp | 10 +++++-- .../eamxx/src/physics/mam/mam_coupling.hpp | 29 ------------------- .../eamxx/src/share/field/field_tag.hpp | 28 ------------------ 3 files changed, 8 insertions(+), 59 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b9c8ae9e8f85..33d57f2050d4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -771,6 +771,12 @@ void MAMAci::set_grids( // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + auto make_layout = [](const std::vector &extents, + const std::vector &names) { + std::vector tags(extents.size(), CMP); + return FieldLayout(tags, extents, names); + }; + using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers q_unit.set_string("kg/kg"); @@ -846,8 +852,8 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); - FieldLayout scalar4d_layout_mid{{COL, NMODES, LEV}, - {ncol_, num_aero_modes, nlev_}}; + FieldLayout scalar4d_layout_mid = + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); // dry diameter of aerosols [m] add_field("dgnum", scalar4d_layout_mid, m, grid_name); diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index c2fcfb6a8eac..ae8c84210b58 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -836,19 +836,9 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { -<<<<<<< HEAD - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); -======= int m = static_cast(mode_index); -<<<<<<< HEAD - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); @@ -879,19 +869,9 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { -<<<<<<< HEAD - if (aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); -======= int m = static_cast(mode_index); -<<<<<<< HEAD - if(aero_id != NoAero) { // constituent is an aerosol species - int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index const Real mw = mam4::aero_species(a).molecular_weight; q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); @@ -927,17 +907,8 @@ void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], int a = aerosol_index_for_mode(mode_index, aero_id); progs.q_aero_i[m][a](k) = q[i]; progs.q_aero_c[m][a](k) = qqcw[i]; -<<<<<<< HEAD -<<<<<<< HEAD - } else { // constituent is a modal number mixing ratio -======= - } else { // constituent is a modal number mixing ratio - int m = static_cast(mode_index); ->>>>>>> Modified species order to match E3SM with the folling change in mam4xx -======= } else { // constituent is a modal number mixing ratio int m = static_cast(mode_index); ->>>>>>> Reverted back mam_coupling changes for indexing but added a func to get prog index progs.n_mode_i[m](k) = q[i]; progs.n_mode_c[m](k) = qqcw[i]; } diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index a2e196621629..673208d6ae86 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -34,12 +34,6 @@ enum class FieldTag { GaussPoint, Component, TimeLevel, - // - MAM_NumModes, - MAM_NumRefIndexReal, - MAM_NumRefIndexImag, - MAM_NumCoefficients, - MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -56,13 +50,6 @@ namespace ShortFieldTagsNames { constexpr auto LEV = FieldTag::LevelMidPoint; constexpr auto ILEV = FieldTag::LevelInterface; constexpr auto CMP = FieldTag::Component; - constexpr auto NMODES = FieldTag::MAM_NumModes; - // - constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; - constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; - - constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; - constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -93,21 +80,6 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::Component: name = "dim"; break; - case FieldTag::MAM_NumModes: - name = "num_modes"; - break; - case FieldTag::MAM_NumRefIndexReal: - name = "refindex_real"; - break; - case FieldTag::MAM_NumRefIndexImag: - name = "refindex_im"; - break; - case FieldTag::MAM_NumCoefficients: - name = "coef_number"; - break; - case FieldTag::MAM_NumModesInFile: - name = "mode"; - break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } From f7d3e404a77746247192a6162dee28b429630aca Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 1 May 2024 08:53:31 -0700 Subject: [PATCH 344/476] Fixes wrong layouts and removed function to copy views --- .../eamxx/src/physics/shoc/CMakeLists.txt | 1 - .../shoc/disp/shoc_assign_2d_view_disp.cpp | 29 ------------------- .../shoc/eamxx_shoc_process_interface.cpp | 22 +++++++------- .../src/physics/shoc/impl/shoc_main_impl.hpp | 6 ++-- .../eamxx/src/physics/shoc/shoc_functions.hpp | 7 ----- 5 files changed, 14 insertions(+), 51 deletions(-) delete mode 100644 components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index f57506fe0ed0..e37379095d00 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -85,7 +85,6 @@ set(SHOC_SK_SRCS disp/shoc_diag_third_shoc_moments_disp.cpp disp/shoc_assumed_pdf_disp.cpp disp/shoc_update_host_dse_disp.cpp - disp/shoc_assign_2d_view_disp.cpp ) if (NOT SCREAM_DEBUG) diff --git a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp b/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp deleted file mode 100644 index e6cd04054d0a..000000000000 --- a/components/eamxx/src/physics/shoc/disp/shoc_assign_2d_view_disp.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "shoc_functions.hpp" - -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream { -namespace shoc { - -template<> -void Functions -::shoc_assign_2d_view_disp( - const Int& shcol, - const Int& nlev, - const view_2d& input_view, - const view_2d& output_view) -{ - using ExeSpace = typename KT::ExeSpace; - - const auto nlev_packs = ekat::npack(nlev); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(shcol, nlev_packs); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - const Int i = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const Int& k) { - output_view (i,k) = input_view(i,k); - }); - }); -} - -} // namespace shoc -} // namespace scream diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 25c2674c9121..5ceb40c738f7 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -80,19 +80,19 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("phis", scalar2d , m2/s2, grid_name, ps); // Input/Output variables - add_field("tke", scalar3d_layout_mid, m2/s2, grid_name, "tracers", ps); - add_field("horiz_winds", horiz_wind_layout, m/s, grid_name, ps); - add_field("sgs_buoy_flux", scalar3d_layout_mid, K*(m/s), grid_name, ps); - add_field("eddy_diff_mom", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_layout_mid, Qunit, grid_name, "tracers", ps); - add_field("cldfrac_liq", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("tke", scalar3d_mid, m2/s2, grid_name, "tracers", ps); + add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); + add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); + add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); + add_field("qc", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d_layout_col, m, grid_name); - add_field("inv_qc_relvar", scalar3d_layout_mid, Qunit*Qunit, grid_name, ps); - add_field("eddy_diff_heat", scalar3d_layout_mid, m2/s, grid_name, ps); - add_field("w_variance", scalar3d_layout_mid, m2/s2, grid_name, ps); - add_field("cldfrac_liq_prev", scalar3d_layout_mid, nondim, grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); + add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); + add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index ca163a49ddd8..640400f54d20 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -575,7 +575,7 @@ void Functions::shoc_main_internal( pblh); // Output // Assign tkh to the tkh output variable - shoc_assign_2d_view_disp(shcol, nlev, tkh_out,tkh); + Kokkos::deep_copy(tkh_out,tkh); } #endif @@ -654,7 +654,7 @@ Int Functions::shoc_main( const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); - const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); + const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); const auto shoc_mix_s = ekat::subview(shoc_history_output.shoc_mix, i); const auto w_sec_s = ekat::subview(shoc_history_output.w_sec, i); const auto thl_sec_s = ekat::subview(shoc_history_output.thl_sec, i); @@ -686,7 +686,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_out_s, // Output + pblh_s, shoc_ql2_s, tkh_out_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 0b6512630644..ccc941bfd055 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -1205,13 +1205,6 @@ struct Functions const view_2d& tk, const view_2d& tkh, const view_2d& isotropy); - - static void shoc_assign_2d_view_disp( - const Int& shcol, - const Int& nlev, - const view_2d& input_view, - const view_2d& output_view); - #endif }; // struct Functions From 5e90223fbadb3779b2860f696bc899ec6ff605d0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 2 May 2024 17:27:23 -0700 Subject: [PATCH 345/476] Fixes process order by adding spa, adds dgnum nominal value --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 ++ .../testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands | 1 + 2 files changed, 3 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 49244880280e..1942e339ef78 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -487,6 +487,8 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0,0.0 + + 2.6e-08 0.0 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 3bffab45ba68..6ce4e74690c7 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -4,6 +4,7 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b $CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b From 99782e4d0950e113a6a20610a8da87c3acfd1450 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 2 May 2024 21:07:22 -0700 Subject: [PATCH 346/476] Explicitly specify do_predict_nc and do_prescribed_ccn flags --- .../testmods_dirs/scream/mam4xx/aci/shell_commands | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 6ce4e74690c7..68eaf51698b3 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -3,8 +3,21 @@ #Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options ./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#modify initial condition file to get aerosol species ICs $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add spa as RRTMG needs spa $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b + +# Replace spa with mam4_aci to invoke mam4 aci scheme $CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b +#Set precribed ccn to false so that P3 uses input from ACI +$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_prescribed_ccn=false -b + +#Set predicted ccn to true so that P3 uses input from ACI +$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_predict_nc=true -b + + + From 2fcc3c319da01adb4fe7f1c14d8e4fc683530846 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 4 May 2024 12:24:46 -0700 Subject: [PATCH 347/476] Temporarily adds cloudborne aerosols in IC section of the namelist --- .../cime_config/namelist_defaults_scream.xml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 1942e339ef78..50019833d8ae 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -489,6 +489,31 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0,0.0 2.6e-08 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 0.0 From efa21196a077000eed27704abd14fc86c1215385 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 6 May 2024 12:27:15 -0600 Subject: [PATCH 348/476] Fix allocation of tkh. --- .../eamxx/src/physics/shoc/shoc_functions_f90.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 0e4acceb1a79..20fdb7b3ee8e 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2743,8 +2743,6 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, Real* qw_sec, Real* qwthl_sec, Real* wthl_sec, Real* wqw_sec, Real* wtke_sec, Real* uw_sec, Real* vw_sec, Real* w3, Real* wqls_sec, Real* brunt, Real* shoc_ql2) { - // tkh is a local variable in C++ impl - (void)tkh; using SHF = Functions; @@ -2758,7 +2756,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, // Initialize Kokkos views, sync to device static constexpr Int num_1d_arrays = 7; - static constexpr Int num_2d_arrays = 34; + static constexpr Int num_2d_arrays = 35; static constexpr Int num_3d_arrays = 1; std::vector temp_1d_d(num_1d_arrays); @@ -2768,14 +2766,14 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, std::vector dim1_2d_sizes = {shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, - shcol, shcol, shcol, shcol, + shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol, shcol}; std::vector dim2_2d_sizes = {nlev, nlevi, nlev, nlevi, nlev, nlev, nlev, num_qtracers, nlev, nlev, nlev, nlev, nlev, nlev, nlev, - nlev, nlev, nlev, nlev, + nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev, nlev, nlev}; @@ -2786,7 +2784,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, thv, w_field, wtracer_sfc, inv_exner, host_dse, tke, thetal, qw, u_wind, v_wind, wthv_sec, tk, shoc_cldfrac, shoc_ql, - shoc_ql2, shoc_mix, w_sec, thl_sec, qw_sec, + shoc_ql2, tkh, shoc_mix, w_sec, thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, vw_sec, w3, wqls_sec, brunt, isotropy}; @@ -2827,6 +2825,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, shoc_cldfrac_d(temp_2d_d[index_counter++]), shoc_ql_d (temp_2d_d[index_counter++]), shoc_ql2_d (temp_2d_d[index_counter++]), + tkh_d (temp_2d_d[index_counter++]), shoc_mix_d (temp_2d_d[index_counter++]), w_sec_d (temp_2d_d[index_counter++]), thl_sec_d (temp_2d_d[index_counter++]), @@ -2879,7 +2878,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d}; + SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, From 893df5e0dd15efd2622af679991503e1db27da66 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 6 May 2024 13:13:41 -0600 Subject: [PATCH 349/476] Seems that tkh is an array of nlev+1, not nlev. --- .../src/physics/shoc/shoc_functions_f90.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 20fdb7b3ee8e..7e3ae283418e 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2774,19 +2774,19 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, nlev, nlev, num_qtracers, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, - nlev, nlev, nlev, nlevi, nlevi, + nlevi, nlev, nlev, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev, nlev, nlev}; std::vector ptr_array_1d = {host_dx, host_dy, wthl_sfc, wqw_sfc, uw_sfc, vw_sfc, phis}; - std::vector ptr_array_2d = {zt_grid, zi_grid, pres, presi, pdel, - thv, w_field, wtracer_sfc, inv_exner, host_dse, - tke, thetal, qw, u_wind, v_wind, - wthv_sec, tk, shoc_cldfrac, shoc_ql, - shoc_ql2, tkh, shoc_mix, w_sec, thl_sec, qw_sec, - qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, - vw_sec, w3, wqls_sec, brunt, isotropy}; + std::vector ptr_array_2d = {zt_grid, zi_grid, pres, presi, pdel, + thv, w_field, wtracer_sfc, inv_exner, host_dse, + tke, thetal, qw, u_wind, v_wind, + wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, + tkh, shoc_mix, w_sec, thl_sec, qw_sec, + qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, + vw_sec, w3, wqls_sec, brunt, isotropy}; ScreamDeepCopy::copy_to_device(ptr_array_1d, shcol, temp_1d_d); ekat::host_to_device(ptr_array_2d, dim1_2d_sizes, dim2_2d_sizes, temp_2d_d, true); From 04bd1a6415774be36c8cc82213a135785fb76cdb Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Mon, 6 May 2024 15:01:00 -0600 Subject: [PATCH 350/476] Use this version if tkh is nlev in length, not nlev+1. --- components/eamxx/src/physics/shoc/shoc_functions_f90.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 7e3ae283418e..5624169ccfb8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2774,7 +2774,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, nlev, nlev, num_qtracers, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, nlev, - nlevi, nlev, nlev, nlevi, nlevi, + nlev, nlev, nlev, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlevi, nlev, nlev, nlev}; @@ -2784,7 +2784,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, thv, w_field, wtracer_sfc, inv_exner, host_dse, tke, thetal, qw, u_wind, v_wind, wthv_sec, tk, shoc_cldfrac, shoc_ql, shoc_ql2, - tkh, shoc_mix, w_sec, thl_sec, qw_sec, + tkh, shoc_mix, w_sec, thl_sec, qw_sec, qwthl_sec, wthl_sec, wqw_sec, wtke_sec, uw_sec, vw_sec, w3, wqls_sec, brunt, isotropy}; @@ -2909,7 +2909,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, tabs ("shoc_tabs", shcol, nlev_packs), dz_zt ("dz_zt", shcol, nlevi_packs), dz_zi ("dz_zi", shcol, nlevi_packs), - tkhv ("tkh", shcol, nlevi_packs); + tkhv ("tkh", shcol, nlev_packs); SHF::SHOCTemporaries shoc_temporaries{ se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, From cec5469f9f611a3699af4e75b515d6f6106dfa95 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 10 May 2024 08:51:55 -0700 Subject: [PATCH 351/476] Init all computed vars in aci to zero; remove tkh_out from SHOC --- .../mam/eamxx_mam_aci_process_interface.cpp | 36 ++++++++++++++++++- .../shoc/eamxx_shoc_process_interface.cpp | 3 +- .../src/physics/shoc/impl/shoc_main_impl.hpp | 34 +++++++----------- .../eamxx/src/physics/shoc/shoc_functions.hpp | 7 ++-- .../src/physics/shoc/shoc_functions_f90.cpp | 5 ++- 5 files changed, 53 insertions(+), 32 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 33d57f2050d4..51c4b9601d84 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -189,6 +189,14 @@ void compute_nucleate_ice_tendencies( // from ice nucleation //------------------------------------------------------------- + //Initialize (zero out) all the output + Kokkos::deep_copy(naai, 0.0); + Kokkos::deep_copy(nihf, 0.0); + Kokkos::deep_copy(niim, 0.0); + Kokkos::deep_copy(nidep, 0.0); + Kokkos::deep_copy(nimey, 0.0); + Kokkos::deep_copy(naai_hom, 0.0); + Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -326,10 +334,11 @@ void call_function_dropmixnuc( MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // ## outputs to be used by other processes ## + // ## input-outputs to be used by other processes ## // qqcw_fld_work should be directly assigned to the cloud borne aerosols MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + // ## outputs to be used by other processes ## // ptend_q are the tendencies to the interstitial aerosols MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], @@ -344,6 +353,26 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { + + + // initialize the purely output variables to zero + for(int i = 0; i < mam4::aero_model::pcnst; ++i)Kokkos::deep_copy(ptend_q[i], 0.0); + Kokkos::deep_copy(factnum, 0.0); + Kokkos::deep_copy(tendnd, 0.0); + + //initialize all diagnostic ouputs to zero: + for(int i = 0; i < mam4::aero_model::pcnst; ++i){ + Kokkos::deep_copy(coltend[i], 0.0); + Kokkos::deep_copy(coltend_cw[i], 0.0); + } + Kokkos::deep_copy(qcld, 0.0); + Kokkos::deep_copy(ndropcol, 0.0); + Kokkos::deep_copy(ndropmix, 0.0); + Kokkos::deep_copy(nsource, 0.0); + Kokkos::deep_copy(wtke, 0.0); + Kokkos::deep_copy(ccn, 0.0); + + // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -1072,10 +1101,15 @@ void MAMAci::initialize_impl(const RunType run_type) { // hetrozenous freezing outputs hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); + Kokkos::deep_copy(hetfrz_immersion_nucleation_tend_, 0.0); //init to zero + hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); + Kokkos::deep_copy(hetfrz_contact_nucleation_tend_, 0.0); //init to zero + hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + Kokkos::deep_copy(hetfrz_depostion_nucleation_tend_, 0.0); //init to zero //--------------------------------------------------------------------------------- // Allocate memory for the class members diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 5ceb40c738f7..904d7f9672ec 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -186,7 +186,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.inv_exner, &m_buffer.thlm, &m_buffer.qw, &m_buffer.dse, &m_buffer.tke_copy, &m_buffer.qc_copy, &m_buffer.shoc_ql2, &m_buffer.shoc_mix, &m_buffer.isotropy, &m_buffer.w_sec, &m_buffer.wqls_sec, &m_buffer.brunt #ifdef SCREAM_SMALL_KERNELS - , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt, &m_buffer.tkh + , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt #endif }; @@ -374,7 +374,6 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.tabs = m_buffer.tabs; temporaries.dz_zt = m_buffer.dz_zt; temporaries.dz_zi = m_buffer.dz_zi; - temporaries.tkh = m_buffer.tkh; #endif shoc_postprocess.set_variables(m_num_cols,m_num_levs,m_num_tracers, diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 640400f54d20..120ed505f261 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -119,7 +119,7 @@ void Functions::shoc_main_internal( // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, - const uview_1d& tkh_out, + const uview_1d& tkh, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -138,10 +138,10 @@ void Functions::shoc_main_internal( { // Define temporary variables - uview_1d rho_zt, shoc_qv, shoc_tabs, dz_zt, dz_zi, tkh; - workspace.template take_many_and_reset<6>( - {"rho_zt", "shoc_qv", "shoc_tabs", "dz_zt", "dz_zi", "tkh"}, - {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); + uview_1d rho_zt, shoc_qv, shoc_tabs, dz_zt, dz_zi; + workspace.template take_many_and_reset<5>( + {"rho_zt", "shoc_qv", "shoc_tabs", "dz_zt", "dz_zi"}, + {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, @@ -311,15 +311,9 @@ void Functions::shoc_main_internal( workspace, // Workspace pblh); // Output - // Assign tkh to the tkh output variable - const Int nlev_pack = ekat::npack(nlev); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_pack), [&] (const Int& k) { - tkh_out(k) = tkh(k); - }); - // Release temporary variables from the workspace - workspace.template release_many_contiguous<6>( - {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi, &tkh}); + workspace.template release_many_contiguous<5>( + {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); } #else template @@ -378,7 +372,7 @@ void Functions::shoc_main_internal( // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, - const view_2d& tkh_out, + const view_2d& tkh, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, @@ -412,8 +406,7 @@ void Functions::shoc_main_internal( const view_2d& shoc_qv, const view_2d& shoc_tabs, const view_2d& dz_zt, - const view_2d& dz_zi, - const view_2d& tkh) + const view_2d& dz_zi) { // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -573,9 +566,6 @@ void Functions::shoc_main_internal( kbfs,shoc_cldfrac, // Input workspace_mgr, // Workspace mgr pblh); // Output - - // Assign tkh to the tkh output variable - Kokkos::deep_copy(tkh_out,tkh); } #endif @@ -654,7 +644,7 @@ Int Functions::shoc_main( const auto shoc_cldfrac_s = ekat::subview(shoc_input_output.shoc_cldfrac, i); const auto shoc_ql_s = ekat::subview(shoc_input_output.shoc_ql, i); const auto shoc_ql2_s = ekat::subview(shoc_output.shoc_ql2, i); - const auto tkh_out_s = ekat::subview(shoc_output.tkh, i); + const auto tkh_s = ekat::subview(shoc_output.tkh, i); const auto shoc_mix_s = ekat::subview(shoc_history_output.shoc_mix, i); const auto w_sec_s = ekat::subview(shoc_history_output.w_sec, i); const auto thl_sec_s = ekat::subview(shoc_history_output.thl_sec, i); @@ -686,7 +676,7 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_out_s, // Output + pblh_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables @@ -719,7 +709,7 @@ Int Functions::shoc_main( shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, - shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi, shoc_temporaries.tkh); + shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif auto finish = std::chrono::steady_clock::now(); diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index ccc941bfd055..74602c688c23 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -890,7 +890,7 @@ struct Functions // Output Variables Scalar& pblh, const uview_1d& shoc_ql2, - const uview_1d& tkh_out, + const uview_1d& tkh, // Diagnostic Output Variables const uview_1d& shoc_mix, const uview_1d& w_sec, @@ -962,7 +962,7 @@ struct Functions // Output Variables const view_1d& pblh, const view_2d& shoc_ql2, - const view_2d& tkh_out, + const view_2d& tkh, // Diagnostic Output Variables const view_2d& shoc_mix, const view_2d& w_sec, @@ -996,8 +996,7 @@ struct Functions const view_2d& shoc_qv, const view_2d& tabs, const view_2d& dz_zt, - const view_2d& dz_zi, - const view_2d& tkh); + const view_2d& dz_zi); #endif // Return microseconds elapsed diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index 5624169ccfb8..dc45e2456aa8 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2908,12 +2908,11 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, shoc_qv ("shoc_qv", shcol, nlevi_packs), tabs ("shoc_tabs", shcol, nlev_packs), dz_zt ("dz_zt", shcol, nlevi_packs), - dz_zi ("dz_zi", shcol, nlevi_packs), - tkhv ("tkh", shcol, nlev_packs); + dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, - rho_zt, shoc_qv, tabs, dz_zt, dz_zi, tkhv}; + rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif // Create local workspace From c162ee4b31dd9a880e8fb42042a79de2f3e5871c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 10 May 2024 11:10:24 -0700 Subject: [PATCH 352/476] Removes some unneeded deep copies --- .../eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 51c4b9601d84..ca7133d8f3a5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1101,15 +1101,12 @@ void MAMAci::initialize_impl(const RunType run_type) { // hetrozenous freezing outputs hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - Kokkos::deep_copy(hetfrz_immersion_nucleation_tend_, 0.0); //init to zero hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - Kokkos::deep_copy(hetfrz_contact_nucleation_tend_, 0.0); //init to zero hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); - Kokkos::deep_copy(hetfrz_depostion_nucleation_tend_, 0.0); //init to zero //--------------------------------------------------------------------------------- // Allocate memory for the class members From 6349d140cb2eeb50ea26a3b05f65b164971a6204 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 12 May 2024 23:40:24 -0700 Subject: [PATCH 353/476] Adds ndropmix and nsource as output variables --- .../mam/eamxx_mam_aci_process_interface.cpp | 50 ++++++------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index ca7133d8f3a5..5016c8bee825 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -189,14 +189,6 @@ void compute_nucleate_ice_tendencies( // from ice nucleation //------------------------------------------------------------- - //Initialize (zero out) all the output - Kokkos::deep_copy(naai, 0.0); - Kokkos::deep_copy(nihf, 0.0); - Kokkos::deep_copy(niim, 0.0); - Kokkos::deep_copy(nidep, 0.0); - Kokkos::deep_copy(nimey, 0.0); - Kokkos::deep_copy(naai_hom, 0.0); - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -334,11 +326,10 @@ void call_function_dropmixnuc( MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, MAMAci::view_2d wtke, MAMAci::view_3d ccn, - // ## input-outputs to be used by other processes ## + // ## outputs to be used by other processes ## // qqcw_fld_work should be directly assigned to the cloud borne aerosols MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - // ## outputs to be used by other processes ## // ptend_q are the tendencies to the interstitial aerosols MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], @@ -353,26 +344,6 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { - - - // initialize the purely output variables to zero - for(int i = 0; i < mam4::aero_model::pcnst; ++i)Kokkos::deep_copy(ptend_q[i], 0.0); - Kokkos::deep_copy(factnum, 0.0); - Kokkos::deep_copy(tendnd, 0.0); - - //initialize all diagnostic ouputs to zero: - for(int i = 0; i < mam4::aero_model::pcnst; ++i){ - Kokkos::deep_copy(coltend[i], 0.0); - Kokkos::deep_copy(coltend_cw[i], 0.0); - } - Kokkos::deep_copy(qcld, 0.0); - Kokkos::deep_copy(ndropcol, 0.0); - Kokkos::deep_copy(ndropmix, 0.0); - Kokkos::deep_copy(nsource, 0.0); - Kokkos::deep_copy(wtke, 0.0); - Kokkos::deep_copy(ccn, 0.0); - - // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -949,6 +920,14 @@ void MAMAci::set_grids( add_field("nc_nuceat_tend", scalar3d_layout_mid, n_unit / s, grid_name); + // FIXME: [TEMPORARY]droplet number mixing ratio source tendency [#/kg/s] + add_field("nsource", scalar3d_layout_mid, n_unit / s, + grid_name); + + // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing [#/kg/s] + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, + grid_name); + // ------------------------------------------------------------------------ // Output from hetrozenous freezing // ------------------------------------------------------------------------ @@ -1101,10 +1080,8 @@ void MAMAci::initialize_impl(const RunType run_type) { // hetrozenous freezing outputs hetfrz_immersion_nucleation_tend_ = get_field_out("hetfrz_immersion_nucleation_tend").get_view(); - hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = get_field_out("hetfrz_depostion_nucleation_tend").get_view(); @@ -1165,10 +1142,15 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(ndropcol_, ncol_, nlev_); // droplet number mixing ratio tendency due to mixing [#/kg/s] - Kokkos::resize(ndropmix_, ncol_, nlev_); + //Kokkos::resize(ndropmix_, ncol_, nlev_); + //Temporarily output ndropmix_ + ndropmix_ = get_field_out("ndropmix").get_view(); // droplet number mixing ratio source tendency [#/kg/s] - Kokkos::resize(nsource_, ncol_, nlev_); + //Kokkos::resize(nsource_, ncol_, nlev_); + //Temporarily output nsource_ + nsource_ = get_field_out("nsource").get_view(); + // subgrid vertical velocity [m/s] Kokkos::resize(wtke_, ncol_, nlev_); From 3ce0a91a63ecc5cbb14a490e74f39bbceb612a20 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 14 May 2024 12:32:38 -0700 Subject: [PATCH 354/476] Adds a temporary varaible to capture nc before aci --- .../mam/eamxx_mam_aci_process_interface.cpp | 33 ++++++++++++++----- .../mam/eamxx_mam_aci_process_interface.hpp | 1 + 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 5016c8bee825..2cf2d70fc8d1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -921,11 +921,14 @@ void MAMAci::set_grids( grid_name); // FIXME: [TEMPORARY]droplet number mixing ratio source tendency [#/kg/s] - add_field("nsource", scalar3d_layout_mid, n_unit / s, - grid_name); + add_field("nsource", scalar3d_layout_mid, n_unit / s, grid_name); + + // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing + // [#/kg/s] + add_field("ndropmix", scalar3d_layout_mid, n_unit / s, grid_name); - // FIXME: [TEMPORARY]droplet number mixing ratio tendency due to mixing [#/kg/s] - add_field("ndropmix", scalar3d_layout_mid, n_unit / s, + // FIXME: [TEMPORARY]droplet number as seen by ACI [#/kg] + add_field("nc_inp_to_aci", scalar3d_layout_mid, n_unit / s, grid_name); // ------------------------------------------------------------------------ @@ -1142,15 +1145,17 @@ void MAMAci::initialize_impl(const RunType run_type) { Kokkos::resize(ndropcol_, ncol_, nlev_); // droplet number mixing ratio tendency due to mixing [#/kg/s] - //Kokkos::resize(ndropmix_, ncol_, nlev_); - //Temporarily output ndropmix_ + // Kokkos::resize(ndropmix_, ncol_, nlev_); + // Temporarily output ndropmix_ ndropmix_ = get_field_out("ndropmix").get_view(); // droplet number mixing ratio source tendency [#/kg/s] - //Kokkos::resize(nsource_, ncol_, nlev_); - //Temporarily output nsource_ + // Kokkos::resize(nsource_, ncol_, nlev_); + // Temporarily output nsource_ nsource_ = get_field_out("nsource").get_view(); - + + // Temporarily output nc_inp_to_aci_ + nc_inp_to_aci_ = get_field_out("nc_inp_to_aci").get_view(); // subgrid vertical velocity [m/s] Kokkos::resize(wtke_, ncol_, nlev_); @@ -1238,6 +1243,16 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // FIXME: Temporary assignment of nc + // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [=](int kk) { + nc_inp_to_aci_(icol, kk) = wet_atm_.nc(icol, kk); + }); + }); + compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output w0_, rho_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 3f6f7a093142..f1a1f21d33f5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -98,6 +98,7 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d ndropcol_; view_2d ndropmix_; view_2d nsource_; + view_2d nc_inp_to_aci_; // FIXME: TEMPORARY output view_2d wtke_; view_3d ccn_; view_2d coltend_[mam4::ndrop::ncnst_tot]; From 4f295d63c4cd48426fcfb36843f97bafcfeedf53 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 15 May 2024 17:18:44 -0700 Subject: [PATCH 355/476] Temporarily adds ccn diagnostics --- .../mam/eamxx_mam_aci_process_interface.cpp | 30 +++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 6 ++++ 2 files changed, 36 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2cf2d70fc8d1..036e4e903c2d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -930,6 +930,15 @@ void MAMAci::set_grids( // FIXME: [TEMPORARY]droplet number as seen by ACI [#/kg] add_field("nc_inp_to_aci", scalar3d_layout_mid, n_unit / s, grid_name); + const auto cm_tmp = m / 100; // FIXME: [TEMPORARY] remove this + const auto cm3 = cm_tmp * cm_tmp * cm_tmp; // FIXME: [TEMPORARY] remove this + // FIXME: [TEMPORARY] remove the following ccn outputs + add_field("ccn_0p02", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p05", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p1", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p2", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_0p5", scalar3d_layout_mid, cm3, grid_name); + add_field("ccn_1p0", scalar3d_layout_mid, cm3, grid_name); // ------------------------------------------------------------------------ // Output from hetrozenous freezing @@ -1157,6 +1166,14 @@ void MAMAci::initialize_impl(const RunType run_type) { // Temporarily output nc_inp_to_aci_ nc_inp_to_aci_ = get_field_out("nc_inp_to_aci").get_view(); + // FIXME: [TEMPORARY] remove the following ccn outputs + ccn_0p02_ = get_field_out("ccn_0p02").get_view(); + ccn_0p05_ = get_field_out("ccn_0p05").get_view(); + ccn_0p1_ = get_field_out("ccn_0p1").get_view(); + ccn_0p2_ = get_field_out("ccn_0p2").get_view(); + ccn_0p5_ = get_field_out("ccn_0p5").get_view(); + ccn_1p0_ = get_field_out("ccn_1p0").get_view(); + // subgrid vertical velocity [m/s] Kokkos::resize(wtke_, ncol_, nlev_); @@ -1323,6 +1340,19 @@ void MAMAci::run_impl(const double dt) { dropmixnuc_scratch_mem_); Kokkos::fence(); // wait for ptend_q_ to be computed. + Kokkos::deep_copy(ccn_0p02_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 0)); + Kokkos::deep_copy(ccn_0p05_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 1)); + Kokkos::deep_copy(ccn_0p1_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 2)); + Kokkos::deep_copy(ccn_0p2_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 3)); + Kokkos::deep_copy(ccn_0p5_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 4)); + Kokkos::deep_copy(ccn_1p0_, + Kokkos::subview(ccn_, Kokkos::ALL(), Kokkos::ALL(), 5)); + //--------------------------------------------------------------------------- // NOTE: DO NOT UPDATE cloud borne aerosols using the qqcw_fld_work_ array // at this point as heterozenous freezing needs to use cloud borne aerosols diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index f1a1f21d33f5..093dd571dd90 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -99,6 +99,12 @@ class MAMAci final : public scream::AtmosphereProcess { view_2d ndropmix_; view_2d nsource_; view_2d nc_inp_to_aci_; // FIXME: TEMPORARY output + view_2d ccn_0p02_; // FIXME: TEMPORARY output + view_2d ccn_0p05_; // FIXME: TEMPORARY output + view_2d ccn_0p1_; // FIXME: TEMPORARY output + view_2d ccn_0p2_; // FIXME: TEMPORARY output + view_2d ccn_0p5_; // FIXME: TEMPORARY output + view_2d ccn_1p0_; // FIXME: TEMPORARY output view_2d wtke_; view_3d ccn_; view_2d coltend_[mam4::ndrop::ncnst_tot]; From 380ad10e5c8eea1bb1a77b87592ab0bcbebc12bf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 18 May 2024 21:28:33 -0700 Subject: [PATCH 356/476] Fixes optics test, this commit has ACI single grid cell printouts --- .../mam/eamxx_mam_aci_process_interface.cpp | 259 +++++++++++++++++- .../eamxx/src/physics/mam/mam_coupling.hpp | 16 +- .../single-process/mam/aci/CMakeLists.txt | 3 +- .../tests/single-process/mam/aci/input.yaml | 6 +- externals/mam4xx | 2 +- 5 files changed, 263 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 036e4e903c2d..2916ee737259 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,5 @@ #include +#include /* ----------------------------------------------------------------- @@ -21,6 +22,222 @@ namespace scream { namespace { +void print_output(const Real w0, const Real rho, const Real tke, + const Real wsub, const Real wice, const Real wsig, + const Real naai_hom, const Real naai, const Real rpdel, + MAMAci::view_3d factnum, const Real tendnd, + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + const Real hetfrz_immersion_nucleation_tend, + const Real hetfrz_contact_nucleation_tend, + const Real hetfrz_depostion_nucleation_tend, + const mam_coupling::AerosolState &dry_aero, const int kb) { + + std::cout << "w0:" << std::setprecision (15) << w0 << std::endl; + std::cout << "rho: " << rho << std::endl; + std::cout << "TKE:" << tke << std::endl; + std::cout << "wsub:" << wsub << std::endl; + std::cout << "wsubi:" << wice << std::endl; + std::cout << "wsig:" << wsig << std::endl; + std::cout << "dgnum_ait:"<< std::endl; + std::cout << "naai_:" << naai << std::endl; + std::cout << "naai_hom:" << naai_hom << std::endl; + //std::cout << "rpdel_:" << rpdel << std::endl; + std::cout << "factnum:" << factnum(0, 0, kb) << " " << factnum(0, 1, kb) + << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) + << std::endl; + std::cout << "nctend_mixnuc:" << tendnd << std::endl; + + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.int_aero_mmr[m][a].data()) { + std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + for(int ic = 9; ic < 40; ++ic) { + std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; + } + /*for(int ic = 0; ic < 25; ++ic) { + std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) + << std::endl; + }*/ + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + } + + + /*for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) + << std::endl; + } + } + }*/ + + std::cout << "hetfrz_immersion_nucleation_tend_:" + << hetfrz_immersion_nucleation_tend << ":" + << hetfrz_immersion_nucleation_tend << std::endl; + std::cout << "hetfrz_contact_nucleation_tend_:" + << hetfrz_contact_nucleation_tend << std::endl; + std::cout << "hetfrz_depostion_nucleation_tend_:" + << hetfrz_depostion_nucleation_tend << std::endl; +} + +void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, + const int ncol_, const int nlev_) { + const Real w_sec_e3sm[73] = { + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.40000000000000002E-003, + 0.40000000000000002E-003, 0.44201892319518146E-003, + 0.77315620137962326E-003, 0.24806301482800117E-002, + 0.11109772692898754E-001, 0.40651094032490273E-001, + 0.82156694426095800E-001, 0.12207124453993526E+000, + 0.15516728994634110E+000, 0.17775318086169636E+000, + 0.18549817250146838E+000, 0.17184548286554119E+000, + 0.12741230682196053E+000, 0.65495229516041628E-001, + 0.26909155217660592E-001}; + + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + w_sec_int_(icol, kk) = w_sec_e3sm[kk]; + // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); + } + // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); + w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; + } + const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, + 0.75991761081225006E-002, 0.12291092068185365E-001, + 0.11484807652762415E-001, 0.10856880396302943E-001, + 0.10500384508819637E-001, 0.10361486171738229E-001, + 0.10333325067964508E-001, 0.10258838031435397E-001, + 0.10027325248446619E-001, 0.97784259072973521E-002, + 0.96611845055866539E-002, 0.96249746122327937E-002, + 0.95773431515696512E-002, 0.95180614513688099E-002, + 0.94713233348487150E-002, 0.94503864489758338E-002, + 0.94536294366578833E-002, 0.94575972194308883E-002, + 0.94403767489615684E-002, 0.93975694769176284E-002, + 0.93322843554751022E-002, 0.92777070192527501E-002, + 0.92456776697171228E-002, 0.92266924824142716E-002, + 0.92123025773060436E-002, 0.91888715633294191E-002, + 0.91516797753615851E-002, 0.90958299606649744E-002, + 0.89988037524983237E-002, 0.88220984587642423E-002, + 0.85231270833157156E-002, 0.81397522619395188E-002, + 0.79160421807845088E-002, 0.81206851117902653E-002, + 0.86526891616674779E-002, 0.91682975412125615E-002, + 0.96043394254592580E-002, 0.10033689085881327E-001, + 0.10428656694074272E-001, 0.10715913043864789E-001, + 0.10919631245454951E-001, 0.11250937075285789E-001, + 0.11829292157343831E-001, 0.12413311776454055E-001, + 0.12851317662157077E-001, 0.13175523677700330E-001, + 0.13224182907540188E-001, 0.13085937680733115E-001, + 0.12615055546741534E-001, 0.11995423733019836E-001, + 0.12346556881757400E-001, 0.13433752971524651E-001, + 0.13904308240950175E-001, 0.13539811748121957E-001, + 0.12555099320041433E-001, 0.11519643673351362E-001, + 0.11414071302852231E-001, 0.13409756835238139E-001, + 0.24071962815959351E-001, 0.75489419450816414E-001, + 0.62082011878960308E+000, 0.63952862312816796E+001, + 0.16226857944175123E+002, 0.21882852534279891E+002, + 0.24966173574402408E+002, 0.25710753126453692E+002, + 0.24069881024271943E+002, 0.19743922403487922E+002, + 0.98667814246712027E+001, 0.25633359450143991E+001, + 0.14682471685037493E+001}; + // compute eddy diffusivity of heat at the interfaces + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + // kvh_int_(icol, kk) = kvh_mid_(icol, kk); + kvh_int_(icol, kk) = kvh_e3sm[kk]; + } + // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); + kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; + } +} + +void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, + const int nlev_) { + const Real dgnum_ait_e3sm[72] = { + 0.20877713336487552E-007, 0.21782230353342090E-007, + 0.21688324003865861E-007, 0.21112855042342451E-007, + 0.19162058462939536E-007, 0.18102979880838476E-007, + 0.17906980715477606E-007, 0.20271254074583327E-007, + 0.22698983422181942E-007, 0.24134835117044986E-007, + 0.25498156808001372E-007, 0.29796738799905547E-007, + 0.35822987394021908E-007, 0.41170963764365215E-007, + 0.44892726528330642E-007, 0.47217231342203107E-007, + 0.48928661807108766E-007, 0.50170939816128735E-007, + 0.51078750853732200E-007, 0.52247333465736065E-007, + 0.53190758580174931E-007, 0.53576491941850044E-007, + 0.53915614473890715E-007, 0.54510964775236826E-007, + 0.55643231691556703E-007, 0.57057811112589899E-007, + 0.58177383586181116E-007, 0.58209849180850108E-007, + 0.57976751598840998E-007, 0.52000000000000002E-007, + 0.50728746567226150E-007, 0.49119902704480870E-007, + 0.48212162162050883E-007, 0.49227715213506454E-007, + 0.46876827233752246E-007, 0.45360603896257791E-007, + 0.49986783979004747E-007, 0.51186879246229022E-007, + 0.50009353247048599E-007, 0.48250264542204811E-007, + 0.47560278748093609E-007, 0.48298089720730957E-007, + 0.49095935613468768E-007, 0.49493024126912931E-007, + 0.50250797590476007E-007, 0.51949267668322422E-007, + 0.53778727208416418E-007, 0.53563593301099588E-007, + 0.51218136771199298E-007, 0.43171429694325200E-007, + 0.39019610039033895E-007, 0.36175109143257051E-007, + 0.42731638777892750E-007, 0.38060728507221777E-007, + 0.44046323901481340E-007, 0.39216732751330010E-007, + 0.34842233953609988E-007, 0.34068804733226066E-007, + 0.30636043694263528E-007, 0.28302341686131413E-007, + 0.33023014309036320E-007, 0.34745748365385196E-007, + 0.43623545003583371E-007, 0.48206451795644064E-007, + 0.49854490325455530E-007, 0.50346335647724146E-007, + 0.50661560988561763E-007, 0.50986261962838767E-007, + 0.51256955985111086E-007, 0.51482578449096488E-007, + 0.51684364851091471E-007, 0.51849719162939729E-007}; + for(int icol = 0; icol < ncol_; ++icol) { + for(int kk = 0; kk < nlev_; ++kk) { + aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; + } + } +} + KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -490,7 +707,7 @@ void call_function_dropmixnuc( // get qqcw at a grid cell (col,lev) // NOTE: The layout for qqcw array is based on mam_idx in - // dropmixnuc To mimic that, we are using the following for-loops + // dropmixnuc. To mimic that, we are using the following for-loops int ind_qqcw = 0; for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { qqcw_view[ind_qqcw](klev) = @@ -834,7 +1051,8 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + //add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -848,8 +1066,8 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); - + //add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = @@ -1054,12 +1272,14 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + //std::cout<<"index:"<(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + //std::cout<<"index:"< 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + //std::cout<<"indices:"< 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); + //std::cout<<"indices:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // FIXME: Remove set_input and print_input + set_input(w_sec_int_, kvh_int_, ncol_, nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); // FIXME: Temporary assignment of nc // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); @@ -1275,9 +1504,9 @@ void MAMAci::run_impl(const double dt) { w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_); + w_sec_int_);*/ compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1294,6 +1523,8 @@ void MAMAci::run_impl(const double dt) { compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); + // FIXME:Remove set_dgait + set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1321,9 +1552,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + /*compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, // output - kvh_int_); + kvh_int_);*/ // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1385,6 +1616,14 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function + + const int kb = 62; + print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), + wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), + rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, + qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), + hetfrz_contact_nucleation_tend_(0, kb), + hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); } } // namespace scream diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index ae8c84210b58..ab7b18c83c78 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -548,10 +548,12 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + //std::cout<<"progs.n_mode_i index:"< Date: Sat, 18 May 2024 21:40:56 -0700 Subject: [PATCH 357/476] Fixes optics tests --- .../mam/eamxx_mam_aci_process_interface.cpp | 257 +----------------- .../eamxx/src/physics/mam/mam_coupling.hpp | 18 +- .../single-process/mam/aci/CMakeLists.txt | 3 +- .../tests/single-process/mam/aci/input.yaml | 6 +- 4 files changed, 21 insertions(+), 263 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 2916ee737259..52da3ed04984 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,5 +1,4 @@ #include -#include /* ----------------------------------------------------------------- @@ -22,222 +21,6 @@ namespace scream { namespace { -void print_output(const Real w0, const Real rho, const Real tke, - const Real wsub, const Real wice, const Real wsig, - const Real naai_hom, const Real naai, const Real rpdel, - MAMAci::view_3d factnum, const Real tendnd, - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - const Real hetfrz_immersion_nucleation_tend, - const Real hetfrz_contact_nucleation_tend, - const Real hetfrz_depostion_nucleation_tend, - const mam_coupling::AerosolState &dry_aero, const int kb) { - - std::cout << "w0:" << std::setprecision (15) << w0 << std::endl; - std::cout << "rho: " << rho << std::endl; - std::cout << "TKE:" << tke << std::endl; - std::cout << "wsub:" << wsub << std::endl; - std::cout << "wsubi:" << wice << std::endl; - std::cout << "wsig:" << wsig << std::endl; - std::cout << "dgnum_ait:"<< std::endl; - std::cout << "naai_:" << naai << std::endl; - std::cout << "naai_hom:" << naai_hom << std::endl; - //std::cout << "rpdel_:" << rpdel << std::endl; - std::cout << "factnum:" << factnum(0, 0, kb) << " " << factnum(0, 1, kb) - << " : " << factnum(0, 2, kb) << " : " << factnum(0, 3, kb) - << std::endl; - std::cout << "nctend_mixnuc:" << tendnd << std::endl; - - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "inter_num:" << dry_aero.int_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.int_aero_mmr[m][a].data()) { - std::cout << "inter-mmr:" << dry_aero.int_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - for(int ic = 9; ic < 40; ++ic) { - std::cout << "ptend_q_:" << ic << ": " << ptend_q[ic](0, kb) << std::endl; - } - /*for(int ic = 0; ic < 25; ++ic) { - std::cout << "qqcw_:" << ic << ": " << qqcw_fld_work[ic](0, kb) - << std::endl; - }*/ - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cldbrn_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cldbrn-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - } - - - /*for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - std::cout << "cld_num:" << dry_aero.cld_aero_nmr[m](0, kb) << std::endl; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - std::cout << "cld-mmr:" << dry_aero.cld_aero_mmr[m][a](0, kb) - << std::endl; - } - } - }*/ - - std::cout << "hetfrz_immersion_nucleation_tend_:" - << hetfrz_immersion_nucleation_tend << ":" - << hetfrz_immersion_nucleation_tend << std::endl; - std::cout << "hetfrz_contact_nucleation_tend_:" - << hetfrz_contact_nucleation_tend << std::endl; - std::cout << "hetfrz_depostion_nucleation_tend_:" - << hetfrz_depostion_nucleation_tend << std::endl; -} - -void set_input(MAMAci::view_2d w_sec_int_, MAMAci::view_2d kvh_int_, - const int ncol_, const int nlev_) { - const Real w_sec_e3sm[73] = { - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.40000000000000002E-003, - 0.40000000000000002E-003, 0.44201892319518146E-003, - 0.77315620137962326E-003, 0.24806301482800117E-002, - 0.11109772692898754E-001, 0.40651094032490273E-001, - 0.82156694426095800E-001, 0.12207124453993526E+000, - 0.15516728994634110E+000, 0.17775318086169636E+000, - 0.18549817250146838E+000, 0.17184548286554119E+000, - 0.12741230682196053E+000, 0.65495229516041628E-001, - 0.26909155217660592E-001}; - - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - w_sec_int_(icol, kk) = w_sec_e3sm[kk]; - // w_sec_int_(icol, kk) = w_sec_mid_(icol, kk); - } - // w_sec_int_(icol, nlev_ + 1) = w_sec_mid_(icol, nlev_); - w_sec_int_(icol, nlev_ + 1) = w_sec_e3sm[nlev_ + 1]; - } - const Real kvh_e3sm[73] = {0.25020913575496480E-002, 0.25021052914616470E-002, - 0.75991761081225006E-002, 0.12291092068185365E-001, - 0.11484807652762415E-001, 0.10856880396302943E-001, - 0.10500384508819637E-001, 0.10361486171738229E-001, - 0.10333325067964508E-001, 0.10258838031435397E-001, - 0.10027325248446619E-001, 0.97784259072973521E-002, - 0.96611845055866539E-002, 0.96249746122327937E-002, - 0.95773431515696512E-002, 0.95180614513688099E-002, - 0.94713233348487150E-002, 0.94503864489758338E-002, - 0.94536294366578833E-002, 0.94575972194308883E-002, - 0.94403767489615684E-002, 0.93975694769176284E-002, - 0.93322843554751022E-002, 0.92777070192527501E-002, - 0.92456776697171228E-002, 0.92266924824142716E-002, - 0.92123025773060436E-002, 0.91888715633294191E-002, - 0.91516797753615851E-002, 0.90958299606649744E-002, - 0.89988037524983237E-002, 0.88220984587642423E-002, - 0.85231270833157156E-002, 0.81397522619395188E-002, - 0.79160421807845088E-002, 0.81206851117902653E-002, - 0.86526891616674779E-002, 0.91682975412125615E-002, - 0.96043394254592580E-002, 0.10033689085881327E-001, - 0.10428656694074272E-001, 0.10715913043864789E-001, - 0.10919631245454951E-001, 0.11250937075285789E-001, - 0.11829292157343831E-001, 0.12413311776454055E-001, - 0.12851317662157077E-001, 0.13175523677700330E-001, - 0.13224182907540188E-001, 0.13085937680733115E-001, - 0.12615055546741534E-001, 0.11995423733019836E-001, - 0.12346556881757400E-001, 0.13433752971524651E-001, - 0.13904308240950175E-001, 0.13539811748121957E-001, - 0.12555099320041433E-001, 0.11519643673351362E-001, - 0.11414071302852231E-001, 0.13409756835238139E-001, - 0.24071962815959351E-001, 0.75489419450816414E-001, - 0.62082011878960308E+000, 0.63952862312816796E+001, - 0.16226857944175123E+002, 0.21882852534279891E+002, - 0.24966173574402408E+002, 0.25710753126453692E+002, - 0.24069881024271943E+002, 0.19743922403487922E+002, - 0.98667814246712027E+001, 0.25633359450143991E+001, - 0.14682471685037493E+001}; - // compute eddy diffusivity of heat at the interfaces - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - // kvh_int_(icol, kk) = kvh_mid_(icol, kk); - kvh_int_(icol, kk) = kvh_e3sm[kk]; - } - // kvh_int_(icol, nlev_ + 1) = kvh_mid_(icol, nlev_); - kvh_int_(icol, nlev_ + 1) = kvh_e3sm[nlev_ + 1]; - } -} - -void set_dgait(MAMAci::view_2d aitken_dry_dia_, const int ncol_, - const int nlev_) { - const Real dgnum_ait_e3sm[72] = { - 0.20877713336487552E-007, 0.21782230353342090E-007, - 0.21688324003865861E-007, 0.21112855042342451E-007, - 0.19162058462939536E-007, 0.18102979880838476E-007, - 0.17906980715477606E-007, 0.20271254074583327E-007, - 0.22698983422181942E-007, 0.24134835117044986E-007, - 0.25498156808001372E-007, 0.29796738799905547E-007, - 0.35822987394021908E-007, 0.41170963764365215E-007, - 0.44892726528330642E-007, 0.47217231342203107E-007, - 0.48928661807108766E-007, 0.50170939816128735E-007, - 0.51078750853732200E-007, 0.52247333465736065E-007, - 0.53190758580174931E-007, 0.53576491941850044E-007, - 0.53915614473890715E-007, 0.54510964775236826E-007, - 0.55643231691556703E-007, 0.57057811112589899E-007, - 0.58177383586181116E-007, 0.58209849180850108E-007, - 0.57976751598840998E-007, 0.52000000000000002E-007, - 0.50728746567226150E-007, 0.49119902704480870E-007, - 0.48212162162050883E-007, 0.49227715213506454E-007, - 0.46876827233752246E-007, 0.45360603896257791E-007, - 0.49986783979004747E-007, 0.51186879246229022E-007, - 0.50009353247048599E-007, 0.48250264542204811E-007, - 0.47560278748093609E-007, 0.48298089720730957E-007, - 0.49095935613468768E-007, 0.49493024126912931E-007, - 0.50250797590476007E-007, 0.51949267668322422E-007, - 0.53778727208416418E-007, 0.53563593301099588E-007, - 0.51218136771199298E-007, 0.43171429694325200E-007, - 0.39019610039033895E-007, 0.36175109143257051E-007, - 0.42731638777892750E-007, 0.38060728507221777E-007, - 0.44046323901481340E-007, 0.39216732751330010E-007, - 0.34842233953609988E-007, 0.34068804733226066E-007, - 0.30636043694263528E-007, 0.28302341686131413E-007, - 0.33023014309036320E-007, 0.34745748365385196E-007, - 0.43623545003583371E-007, 0.48206451795644064E-007, - 0.49854490325455530E-007, 0.50346335647724146E-007, - 0.50661560988561763E-007, 0.50986261962838767E-007, - 0.51256955985111086E-007, 0.51482578449096488E-007, - 0.51684364851091471E-007, 0.51849719162939729E-007}; - for(int icol = 0; icol < ncol_; ++icol) { - for(int kk = 0; kk < nlev_; ++kk) { - aitken_dry_dia_(icol, kk) = dgnum_ait_e3sm[kk]; - } - } -} - KOKKOS_INLINE_FUNCTION void compute_w0_and_rho(const haero::ThreadTeam &team, const MAMAci::const_view_2d omega, @@ -1051,8 +834,7 @@ void MAMAci::set_grids( // and we might need to revisit this. // Vertical velocity variance at midpoints - //add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); - add_field("w_variance", scalar3d_layout_int, m2 / s2, grid_name); + add_field("w_variance", scalar3d_layout_mid, m2 / s2, grid_name); // NOTE: "cldfrac_liq" is updated in SHOC. "cldfrac_liq" in C++ code is // equivalent to "alst" in the shoc_intr.F90. In the C++ code, it is used as @@ -1066,8 +848,8 @@ void MAMAci::set_grids( grid_name); // Eddy diffusivity for heat - //add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); - add_field("eddy_diff_heat", scalar3d_layout_int, m2 / s, grid_name); + add_field("eddy_diff_heat", scalar3d_layout_mid, m2 / s, grid_name); + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = @@ -1272,14 +1054,12 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - //std::cout<<"index:"<(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - //std::cout<<"index:"< 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - //std::cout<<"indices:"< 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); - //std::cout<<"indices:"<::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - - // FIXME: Remove set_input and print_input - set_input(w_sec_int_, kvh_int_, ncol_, nlev_); - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of local derivied // quantities Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); // FIXME: Temporary assignment of nc // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); @@ -1504,9 +1275,9 @@ void MAMAci::run_impl(const double dt) { w0_, rho_); // Get w_sec_int_ from w_sec_mid_ - /*compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, + compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output - w_sec_int_);*/ + w_sec_int_); compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output @@ -1523,8 +1294,6 @@ void MAMAci::run_impl(const double dt) { compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, // output aitken_dry_dia_); - // FIXME:Remove set_dgait - set_dgait(aitken_dry_dia_, ncol_, nlev_); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -1552,9 +1321,9 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. // Get kvh_int_ from kvh_mid_ - /*compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, + compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, // output - kvh_int_);*/ + kvh_int_); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial @@ -1616,14 +1385,6 @@ void MAMAci::run_impl(const double dt) { // call post processing to convert dry mixing ratios to wet mixing ratios Kokkos::parallel_for("postprocess", scan_policy, postprocess_); Kokkos::fence(); // wait before returning to calling function - - const int kb = 62; - print_output(w0_(0, kb), rho_(0, kb), tke_(0, kb), wsub_(0, kb), - wsubice_(0, kb), wsig_(0, kb), naai_hom_(0, kb), naai_(0, kb), - rpdel_(0, kb), factnum_, tendnd_(0, kb), ptend_q_, - qqcw_fld_work_, hetfrz_immersion_nucleation_tend_(0, kb), - hetfrz_contact_nucleation_tend_(0, kb), - hetfrz_depostion_nucleation_tend_(0, kb), dry_aero_, kb); } } // namespace scream diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index ab7b18c83c78..2d89b573a54d 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -548,12 +548,9 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), "int_aero_nmr not defined for dry aerosol state!"); progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); - //std::cout<<"progs.n_mode_i index:"< Date: Sat, 18 May 2024 21:48:21 -0700 Subject: [PATCH 358/476] Rebases mam4xx submodule --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 72219c8db1b2..a8aa56ba915c 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 72219c8db1b27ccacc3e906738b68dd639196961 +Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e From 423add534520db83abdbb807f01c0a1937438a65 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 24 May 2024 09:36:30 -0700 Subject: [PATCH 359/476] Moves aci functions to a separate hpp file --- .../physics/mam/eamxx_mam_aci_functions.hpp | 726 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.cpp | 717 +---------------- 2 files changed, 727 insertions(+), 716 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp new file mode 100644 index 000000000000..5174eb652369 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -0,0 +1,726 @@ +#ifndef EAMXX_MAM_ACI_FUNCTION_HPP +#define EAMXX_MAM_ACI_FUNCTION_HPP + +#include +#include +#include + +namespace scream { + +namespace { + +KOKKOS_INLINE_FUNCTION +void compute_w0_and_rho(const haero::ThreadTeam &team, + const MAMAci::const_view_2d omega, + const MAMAci::const_view_2d T_mid, + const MAMAci::const_view_2d p_mid, const int icol, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { + // Get physical constants + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + // Gas constant for dry air [J/(kg*K) or J/Kg/K] + static constexpr auto rair = C::Rair; + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + w0(icol, kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); + w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); + }); +} +void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const int top_lev, const int nlev, + // output + MAMAci::view_2d w0, MAMAci::view_2d rho) { + MAMAci::const_view_2d omega = dry_atmosphere.omega; + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, + // output + w0, rho); + }); +} + +void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_2d var_mid, + const MAMAci::view_2d dz, const int nlev_, + // output + MAMAci::view_2d var_int) { + using CO = scream::ColumnOps; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + const auto var_mid_col = ekat::subview(var_mid, icol); + const auto var_int_col = ekat::subview(var_int, icol); + const auto dz_col = ekat::subview(dz, icol); + + const Real bc_top = var_mid_col(0); + const Real bc_bot = var_mid_col(nlev_ - 1); + + CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, + bc_top, bc_bot, var_int_col); + }); +} + +KOKKOS_INLINE_FUNCTION +void compute_tke_using_w_sec(const haero::ThreadTeam &team, + const MAMAci::const_view_2d w_sec, const int icol, + const int nlev, + // output + MAMAci::view_2d tke) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev + 1), + KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); +} +void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_2d w_sec, const int nlev, + // output + MAMAci::view_2d tke) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_tke_using_w_sec(team, w_sec, icol, nlev, + // output + tke); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_subgrid_scale_velocities( + const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, + const Real wsubmin, const int icol, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = wsubmin; + wsubice(icol, kk) = 0.001; + wsig(icol, kk) = 0.001; + }); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * + (2.0 / 3.0)); + wsig(icol, kk) = + mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); + wsubice(icol, kk) = + mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); + wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); + }); +} +void compute_subgrid_scale_velocities( + haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, + const Real wsubmin, const int top_lev, const int nlev, + // output + MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, + nlev, + // output + wsub, wsubice, wsig); + }); +} + +KOKKOS_INLINE_FUNCTION +void compute_aitken_dry_diameter(const haero::ThreadTeam &team, + const MAMAci::const_view_3d dgnum, + const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); + }); +} +void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, + const MAMAci::const_view_3d dgnum, + const int top_lev, const int nlev, + // output + MAMAci::view_2d aitken_dry_dia) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, + // output + aitken_dry_dia); + }); +} + +void compute_nucleate_ice_tendencies( + const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const mam_coupling::AerosolState &dry_aero, const MAMAci::view_2d wsubice, + const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, + // output + MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, + MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, + // ## output used by other processes ## + MAMAci::view_2d naai) { + //------------------------------------------------------------- + // Get number of activated aerosol for ice nucleation (naai) + // from ice nucleation + //------------------------------------------------------------- + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + //--------------------------------------------------------------------- + // Set up surface, pronostics atmosphere, diagnostics, and tendencies + // classes. + //--------------------------------------------------------------------- + + // For compute_tendecies interface only, this structure is empty + haero::Surface surf{}; + + // Store interstitial and cld borne aerosols in "progrs" struture + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + + // Update the updraft velocity needed by nucleation to be "wsubice" + // in the haero_atm object + haero_atm.updraft_vel_ice_nucleation = ekat::subview(wsubice, icol); + + // All the output from this process is diagnotics; creates "diags" with + // nlev column length + mam4::Diagnostics diags(nlev); + + // Aitken mode index + const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); + diags.dry_geometric_mean_diameter_i[aitken_idx] = + ekat::subview(aitken_dry_dia, icol); + + // These are the fields that are updated. Taking subviews means that + // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated + // in nucleate_ice.compute_tendencies. + diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); + diags.icenuc_num_immfrz = ekat::subview(niim, icol); + diags.icenuc_num_depnuc = ekat::subview(nidep, icol); + diags.icenuc_num_meydep = ekat::subview(nimey, icol); + + // naai and naai_hom are the outputs needed for nucleate_ice and these + // are not tendencies. + diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); + diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); + + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); // not used + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, + surf, progs, diags, tends); + }); +} +KOKKOS_INLINE_FUNCTION +void store_liquid_cloud_fraction( + const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, + const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, + const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + //------------------------------------------------------------- + // cut-off for cloud amount (ice or liquid) + static constexpr auto qsmall = 1e-18; + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + if((qc(icol, kk) + qi(icol, kk)) > qsmall) { + cloud_frac(icol, kk) = liqcldf(icol, kk); + cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); + } else { + cloud_frac(icol, kk) = 0; + cloud_frac_prev(icol, kk) = 0; + } + }); +} +void store_liquid_cloud_fraction( + haero::ThreadTeamPolicy team_policy, + const mam_coupling::DryAtmosphere &dry_atmosphere, + const MAMAci::const_view_2d liqcldf, + const MAMAci::const_view_2d liqcldf_prev, const int top_lev, const int nlev, + // output + MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { + MAMAci::const_view_2d qc = dry_atmosphere.qc; + MAMAci::const_view_2d qi = dry_atmosphere.qi; + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, + top_lev, nlev, + // output + cloud_frac, cloud_frac_prev); + }); +} +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density(const haero::ThreadTeam &team, + const MAMAci::const_view_2d pdel, + const int icol, const int nlev, + // output + MAMAci::view_2d rpdel) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); +} +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + MAMAci::const_view_2d pdel, + const int nlev, + // output + MAMAci::view_2d rpdel) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + compute_recipical_pseudo_density(team, pdel, icol, nlev, + // output + rpdel); + }); +} + +void call_function_dropmixnuc( + haero::ThreadTeamPolicy team_policy, const Real dt, + mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, + const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, + const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, + const mam_coupling::AerosolState &dry_aero, const int nlev, + + // Following outputs are all diagnostics + MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], + MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, + MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, + MAMAci::view_2d wtke, MAMAci::view_3d ccn, + + // ## outputs to be used by other processes ## + // qqcw_fld_work should be directly assigned to the cloud borne aerosols + MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], + + // ptend_q are the tendencies to the interstitial aerosols + MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + + // factnum is used by the hetrozenous freezing + MAMAci::view_3d factnum, + + // tendnd is used by microphysics scheme (e.g. P3) + MAMAci::view_2d tendnd, + + // ## work arrays ## + MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], + MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, + MAMAci::view_3d nact, MAMAci::view_3d mact, + MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { + // Extract atmosphere variables + MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; + MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; + MAMAci::const_view_2d zm = dry_atmosphere.z_mid; + MAMAci::const_view_2d pdel = dry_atmosphere.p_del; + MAMAci::const_view_2d p_int = dry_atmosphere.p_int; + MAMAci::const_view_2d nc = dry_atmosphere.nc; + + //---------------------------------------------------------------------- + // ## Declare local variables for class variables + //(FIXME: GPU hack, revisit this) + //---------------------------------------------------------------------- + MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; + MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; + MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d loc_ptend_q[mam4::aero_model::pcnst]; + MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; + MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; + + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + loc_raercol_cw[i][j] = raercol_cw[i][j]; + loc_raercol[i][j] = raercol[i][j]; + } + } + + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + loc_coltend[i] = coltend[i]; + loc_coltend_cw[i] = coltend_cw[i]; + } + + for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; + + MAMAci::view_2d qqcw_fld_work_loc[25]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) + qqcw_fld_work_loc[i] = qqcw_fld_work[i]; + + MAMAci::view_3d state_q_work_loc = state_q_work; + + //---------------------------------------------------------------------- + // ## Assign scratch memory for work variables + //---------------------------------------------------------------------- + + MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; + MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; + MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; + MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; + MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; + MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; + MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; + MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; + MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; + MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; + MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; + MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; + MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; + MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; + MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; + + //--------------------------------------------------------------------------- + // ## Initialize the ndrop class. + //--------------------------------------------------------------------------- + const int ntot_amode = mam_coupling::num_aero_modes(); + const int maxd_aspectype = mam4::ndrop::maxd_aspectype; + const int nspec_max = mam4::ndrop::nspec_max; + int nspec_amode[ntot_amode] = {}; + int lspectype_amode[maxd_aspectype][ntot_amode] = {}; + int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; + int numptr_amode[ntot_amode] = {}; + int mam_idx[ntot_amode][nspec_max] = {}; + int mam_cnst_idx[ntot_amode][nspec_max] = {}; + + Real specdens_amode[maxd_aspectype] = {}; + Real spechygro[maxd_aspectype] = {}; + Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, + num2vol_ratio_min_nmodes[ntot_amode] = {}, + num2vol_ratio_max_nmodes[ntot_amode] = {}; + Real aten = 0; + mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, + numptr_amode, specdens_amode, spechygro, + mam_idx, mam_cnst_idx); + mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes); + //--------------------------------------------------------------------------- + //--------------------------------------------------------------------------- + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // for (int icol=0; icol<5; ++icol){ + mam4::ndrop::View1D raercol_cw_view[mam4::ndrop::pver][2]; + mam4::ndrop::View1D raercol_view[mam4::ndrop::pver][2]; + for(int i = 0; i < mam4::ndrop::pver; ++i) { + for(int j = 0; j < 2; ++j) { + raercol_cw_view[i][j] = ekat::subview(loc_raercol_cw[i][j], icol); + raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); + } + } + mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); + } + mam4::ColumnView ptend_q_view[mam4::aero_model::pcnst]; + for(int i = 0; i < mam4::aero_model::pcnst; ++i) { + ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); + } + mam4::ColumnView coltend_view[mam4::ndrop::ncnst_tot], + coltend_cw_view[mam4::ndrop::ncnst_tot]; + for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { + coltend_view[i] = ekat::subview(loc_coltend[i], icol); + coltend_cw_view[i] = ekat::subview(loc_coltend_cw[i], icol); + } + + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + + // get prognostics + mam4::Prognostics progs_at_col = aerosols_for_column(dry_aero, icol); + + // get atmospheric quantities + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + + // Construct state_q (interstitial) and qqcw (cloud borne) arrays + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + [=](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } + + // get qqcw at a grid cell (col,lev) + // NOTE: The layout for qqcw array is based on mam_idx in + // dropmixnuc. To mimic that, we are using the following for-loops + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_nmr[m](icol, klev); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + qqcw_view[ind_qqcw](klev) = + dry_aero.cld_aero_mmr[m][a](icol, klev); + ++ind_qqcw; + } + } + } + }); + + mam4::ndrop::dropmixnuc( + team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), + ekat::subview(p_int, icol), ekat::subview(pdel, icol), + ekat::subview(rpdel, icol), + // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] + ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), + ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] + ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, + spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, + num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, + alogsig, aten, mam_idx, mam_cnst_idx, + ekat::subview(qcld, icol), // out + ekat::subview(wsub, icol), // in + ekat::subview(cloud_frac_prev, icol), // in + qqcw_view, // inout + ptend_q_view, ekat::subview(tendnd, icol), + ekat::subview(factnum, icol), ekat::subview(ndropcol, icol), + ekat::subview(ndropmix, icol), ekat::subview(nsource, icol), + ekat::subview(wtke, icol), ekat::subview(ccn, icol), coltend_view, + coltend_cw_view, + // work arrays + raercol_cw_view, raercol_view, ekat::subview(nact, icol), + ekat::subview(mact, icol), ekat::subview(eddy_diff, icol), + ekat::subview(zn, icol), ekat::subview(csbot, icol), + ekat::subview(zs, icol), ekat::subview(overlapp, icol), + ekat::subview(overlapm, icol), ekat::subview(eddy_diff_kp, icol), + ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), + ekat::subview(srcn, icol), ekat::subview(source, icol), + ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), + ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); + }); +} + +// Update cloud borne aerosols +void update_cloud_borne_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, + // output + mam_coupling::AerosolState &dry_aero) { + int ind_qqcw = 0; + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], qqcw_fld_work[ind_qqcw]); + ++ind_qqcw; + } + } + } +} + +// Update interstitial aerosols using tendencies - levels +void update_interstitial_aerosols_levs( + const haero::ThreadTeam &team, const int nlev, const int icol, + const int s_idx, const Real dt, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + // output + MAMAci::view_2d aero_mr) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { + aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; + }); +} + +// Update interstitial aerosols using tendencies- cols and levs +void update_interstitial_aerosols( + haero::ThreadTeamPolicy team_policy, + const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, + const Real dt, + // output + mam_coupling::AerosolState &dry_aero) { + // starting index of ptend_q array (for MAM4, pcnst=40, ncnst_tot=25 ) + int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // loop through all modes and species + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + // mass mixing ratio of species "a" of mode "m" + auto aero_mmr = dry_aero.int_aero_mmr[m][a]; + + if(aero_mmr.data()) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_mmr); + }); + // update index for the next species (only if aero_mmr.data() is True) + ++s_idx; + } + } + auto aero_nmr = + dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // update values for all levs at this column + update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, + ptend_q, + // output + aero_nmr); + }); + ++s_idx; // update index for the next species + } +} + +void call_hetfrz_compute_tendencies( + haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, + mam_coupling::DryAtmosphere &dry_atm_, + mam_coupling::AerosolState &dry_aero_, MAMAci::view_3d factnum, + const double dt, const int nlev, + // output + MAMAci::view_2d hetfrz_immersion_nucleation_tend, + MAMAci::view_2d hetfrz_contact_nucleation_tend, + MAMAci::view_2d hetfrz_depostion_nucleation_tend, + MAMAci::view_2d diagnostic_scratch_[]) { + mam4::Hetfrz hetfrz = hetfrz_; + mam_coupling::AerosolState dry_aero = dry_aero_; + mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; + + MAMAci::view_2d diagnostic_scratch[MAMAci::hetro_scratch_]; + for(int i = 0; i < MAMAci::hetro_scratch_; ++i) + diagnostic_scratch[i] = diagnostic_scratch_[i]; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + // Set up an atmosphere, surface, diagnostics, pronostics and + // tendencies class. + + haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atmosphere, icol); + haero::Surface surf{}; + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); + const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); + + mam4::Diagnostics diags(nlev); + + diags.activation_fraction[accum_idx] = + ekat::subview(factnum, icol, accum_idx); + + diags.activation_fraction[coarse_idx] = + ekat::subview(factnum, icol, coarse_idx); + + // These are the output tendencies from heterogeneous freezing that need + // to be added correctly to the cloud-micorphysics scheme. + diags.hetfrz_immersion_nucleation_tend = + ekat::subview(hetfrz_immersion_nucleation_tend, icol); + diags.hetfrz_contact_nucleation_tend = + ekat::subview(hetfrz_contact_nucleation_tend, icol); + diags.hetfrz_depostion_nucleation_tend = + ekat::subview(hetfrz_depostion_nucleation_tend, icol); + + diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); + diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); + diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); + diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); + diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); + diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); + diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); + diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); + diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); + diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); + diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); + diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); + diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); + diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); + diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); + diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); + diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); + diags.na500 = ekat::subview(diagnostic_scratch[18], icol); + diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); + diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); + diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); + diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); + diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); + diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); + diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); + diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); + diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); + diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); + diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); + diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); + diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); + diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); + diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); + diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); + diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); + diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); + diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); + diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); + diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); + diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); + diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); + diags.stratiform_cloud_fraction = + ekat::subview(diagnostic_scratch[42], icol); + + // assign cloud fraction + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), + [=](int klev) { + diags.stratiform_cloud_fraction(klev) = + haero_atm.cloud_fraction(klev); + }); + //------------------------------------------------------------- + // Heterogeneous freezing + // frzimm, frzcnt, frzdep are the outputs of + // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) + //------------------------------------------------------------- + // + // grab views from the buffer to store tendencies, not used as all + // values are store in diags above. + const mam4::Tendencies tends(nlev); + const mam4::AeroConfig aero_config; + const Real t = 0; // not used + hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, + progs, diags, tends); + }); +} +} // namespace +} //namespace scream + +#endif // ifdef EAMXX_MAM_ACI_FUNCTION_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 52da3ed04984..67791fe09af7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,4 +1,5 @@ #include +#include /* ----------------------------------------------------------------- @@ -18,722 +19,6 @@ FUTURE WORK: */ namespace scream { - -namespace { - -KOKKOS_INLINE_FUNCTION -void compute_w0_and_rho(const haero::ThreadTeam &team, - const MAMAci::const_view_2d omega, - const MAMAci::const_view_2d T_mid, - const MAMAci::const_view_2d p_mid, const int icol, - const int top_lev, const int nlev, - // output - MAMAci::view_2d w0, MAMAci::view_2d rho) { - // Get physical constants - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - // Gas constant for dry air [J/(kg*K) or J/Kg/K] - static constexpr auto rair = C::Rair; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol, kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); - w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); - }); -} -void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, - const mam_coupling::DryAtmosphere &dry_atmosphere, - const int top_lev, const int nlev, - // output - MAMAci::view_2d w0, MAMAci::view_2d rho) { - MAMAci::const_view_2d omega = dry_atmosphere.omega; - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, - // output - w0, rho); - }); -} - -void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_2d var_mid, - const MAMAci::view_2d dz, const int nlev_, - // output - MAMAci::view_2d var_int) { - using CO = scream::ColumnOps; - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - const auto var_mid_col = ekat::subview(var_mid, icol); - const auto var_int_col = ekat::subview(var_int, icol); - const auto dz_col = ekat::subview(dz, icol); - - const Real bc_top = var_mid_col(0); - const Real bc_bot = var_mid_col(nlev_ - 1); - - CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, - bc_top, bc_bot, var_int_col); - }); -} - -KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec(const haero::ThreadTeam &team, - const MAMAci::const_view_2d w_sec, const int icol, - const int nlev, - // output - MAMAci::view_2d tke) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev + 1), - KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); -} -void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_2d w_sec, const int nlev, - // output - MAMAci::view_2d tke) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_tke_using_w_sec(team, w_sec, icol, nlev, - // output - tke); - }); -} -KOKKOS_INLINE_FUNCTION -void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, - const Real wsubmin, const int icol, const int top_lev, const int nlev, - // output - MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { - // More refined computation of sub-grid vertical velocity - // Set to be zero at the surface by initialization. - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol, kk) = wsubmin; - wsubice(icol, kk) = 0.001; - wsig(icol, kk) = 0.001; - }); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * - (2.0 / 3.0)); - wsig(icol, kk) = - mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); - wsubice(icol, kk) = - mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); - wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); - }); -} -void compute_subgrid_scale_velocities( - haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, - const Real wsubmin, const int top_lev, const int nlev, - // output - MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, - nlev, - // output - wsub, wsubice, wsig); - }); -} - -KOKKOS_INLINE_FUNCTION -void compute_aitken_dry_diameter(const haero::ThreadTeam &team, - const MAMAci::const_view_3d dgnum, - const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); - }); -} -void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_3d dgnum, - const int top_lev, const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, - // output - aitken_dry_dia); - }); -} - -void compute_nucleate_ice_tendencies( - const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, - const mam_coupling::DryAtmosphere &dry_atmosphere, - const mam_coupling::AerosolState &dry_aero, const MAMAci::view_2d wsubice, - const MAMAci::view_2d aitken_dry_dia, const int nlev, const double dt, - // output - MAMAci::view_2d nihf, MAMAci::view_2d niim, MAMAci::view_2d nidep, - MAMAci::view_2d nimey, MAMAci::view_2d naai_hom, - // ## output used by other processes ## - MAMAci::view_2d naai) { - //------------------------------------------------------------- - // Get number of activated aerosol for ice nucleation (naai) - // from ice nucleation - //------------------------------------------------------------- - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - //--------------------------------------------------------------------- - // Set up surface, pronostics atmosphere, diagnostics, and tendencies - // classes. - //--------------------------------------------------------------------- - - // For compute_tendecies interface only, this structure is empty - haero::Surface surf{}; - - // Store interstitial and cld borne aerosols in "progrs" struture - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aero, icol); - - // Store atmopsheric vars (Tmid, Pmid, cloud fraction, qv, wsubmin) - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - - // Update the updraft velocity needed by nucleation to be "wsubice" - // in the haero_atm object - haero_atm.updraft_vel_ice_nucleation = ekat::subview(wsubice, icol); - - // All the output from this process is diagnotics; creates "diags" with - // nlev column length - mam4::Diagnostics diags(nlev); - - // Aitken mode index - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - diags.dry_geometric_mean_diameter_i[aitken_idx] = - ekat::subview(aitken_dry_dia, icol); - - // These are the fields that are updated. Taking subviews means that - // the nihf, niim, nidep, nimey, naai_hom, and naai fields are updated - // in nucleate_ice.compute_tendencies. - diags.icenuc_num_hetfrz = ekat::subview(nihf, icol); - diags.icenuc_num_immfrz = ekat::subview(niim, icol); - diags.icenuc_num_depnuc = ekat::subview(nidep, icol); - diags.icenuc_num_meydep = ekat::subview(nimey, icol); - - // naai and naai_hom are the outputs needed for nucleate_ice and these - // are not tendencies. - diags.num_act_aerosol_ice_nucle_hom = ekat::subview(naai_hom, icol); - diags.num_act_aerosol_ice_nucle = ekat::subview(naai, icol); - - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); // not used - const mam4::AeroConfig aero_config; - const Real t = 0; // not used - nucleate_ice.compute_tendencies(aero_config, team, t, dt, haero_atm, - surf, progs, diags, tends); - }); -} -KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, - const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, - const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { - //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud - // is above qsmall threshold value - //------------------------------------------------------------- - // cut-off for cloud amount (ice or liquid) - static constexpr auto qsmall = 1e-18; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - if((qc(icol, kk) + qi(icol, kk)) > qsmall) { - cloud_frac(icol, kk) = liqcldf(icol, kk); - cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); - } else { - cloud_frac(icol, kk) = 0; - cloud_frac_prev(icol, kk) = 0; - } - }); -} -void store_liquid_cloud_fraction( - haero::ThreadTeamPolicy team_policy, - const mam_coupling::DryAtmosphere &dry_atmosphere, - const MAMAci::const_view_2d liqcldf, - const MAMAci::const_view_2d liqcldf_prev, const int top_lev, const int nlev, - // output - MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { - MAMAci::const_view_2d qc = dry_atmosphere.qc; - MAMAci::const_view_2d qi = dry_atmosphere.qi; - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, - top_lev, nlev, - // output - cloud_frac, cloud_frac_prev); - }); -} -KOKKOS_INLINE_FUNCTION -void compute_recipical_pseudo_density(const haero::ThreadTeam &team, - const MAMAci::const_view_2d pdel, - const int icol, const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); - }); -} -void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::const_view_2d pdel, - const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, pdel, icol, nlev, - // output - rpdel); - }); -} - -void call_function_dropmixnuc( - haero::ThreadTeamPolicy team_policy, const Real dt, - mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, - const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, - const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, - const mam_coupling::AerosolState &dry_aero, const int nlev, - - // Following outputs are all diagnostics - MAMAci::view_2d coltend[mam4::ndrop::ncnst_tot], - MAMAci::view_2d coltend_cw[mam4::ndrop::ncnst_tot], MAMAci::view_2d qcld, - MAMAci::view_2d ndropcol, MAMAci::view_2d ndropmix, MAMAci::view_2d nsource, - MAMAci::view_2d wtke, MAMAci::view_3d ccn, - - // ## outputs to be used by other processes ## - // qqcw_fld_work should be directly assigned to the cloud borne aerosols - MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], - - // ptend_q are the tendencies to the interstitial aerosols - MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - - // factnum is used by the hetrozenous freezing - MAMAci::view_3d factnum, - - // tendnd is used by microphysics scheme (e.g. P3) - MAMAci::view_2d tendnd, - - // ## work arrays ## - MAMAci::view_2d raercol_cw[mam4::ndrop::pver][2], - MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, - MAMAci::view_3d nact, MAMAci::view_3d mact, - MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { - // Extract atmosphere variables - MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; - MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; - MAMAci::const_view_2d zm = dry_atmosphere.z_mid; - MAMAci::const_view_2d pdel = dry_atmosphere.p_del; - MAMAci::const_view_2d p_int = dry_atmosphere.p_int; - MAMAci::const_view_2d nc = dry_atmosphere.nc; - - //---------------------------------------------------------------------- - // ## Declare local variables for class variables - //(FIXME: GPU hack, revisit this) - //---------------------------------------------------------------------- - MAMAci::view_2d loc_raercol_cw[mam4::ndrop::pver][2]; - MAMAci::view_2d loc_raercol[mam4::ndrop::pver][2]; - MAMAci::view_2d loc_qqcw[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d loc_ptend_q[mam4::aero_model::pcnst]; - MAMAci::view_2d loc_coltend[mam4::ndrop::ncnst_tot]; - MAMAci::view_2d loc_coltend_cw[mam4::ndrop::ncnst_tot]; - - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - loc_raercol_cw[i][j] = raercol_cw[i][j]; - loc_raercol[i][j] = raercol[i][j]; - } - } - - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - loc_coltend[i] = coltend[i]; - loc_coltend_cw[i] = coltend_cw[i]; - } - - for(int i = 0; i < mam4::aero_model::pcnst; ++i) loc_ptend_q[i] = ptend_q[i]; - - MAMAci::view_2d qqcw_fld_work_loc[25]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) - qqcw_fld_work_loc[i] = qqcw_fld_work[i]; - - MAMAci::view_3d state_q_work_loc = state_q_work; - - //---------------------------------------------------------------------- - // ## Assign scratch memory for work variables - //---------------------------------------------------------------------- - - MAMAci::view_2d eddy_diff = dropmixnuc_scratch_mem[0]; - MAMAci::view_2d zn = dropmixnuc_scratch_mem[1]; - MAMAci::view_2d csbot = dropmixnuc_scratch_mem[2]; - MAMAci::view_2d zs = dropmixnuc_scratch_mem[3]; - MAMAci::view_2d overlapp = dropmixnuc_scratch_mem[4]; - MAMAci::view_2d overlapm = dropmixnuc_scratch_mem[5]; - MAMAci::view_2d eddy_diff_kp = dropmixnuc_scratch_mem[6]; - MAMAci::view_2d eddy_diff_km = dropmixnuc_scratch_mem[7]; - MAMAci::view_2d qncld = dropmixnuc_scratch_mem[8]; - MAMAci::view_2d srcn = dropmixnuc_scratch_mem[9]; - MAMAci::view_2d source = dropmixnuc_scratch_mem[10]; - MAMAci::view_2d dz = dropmixnuc_scratch_mem[11]; - MAMAci::view_2d csbot_cscen = dropmixnuc_scratch_mem[12]; - MAMAci::view_2d raertend = dropmixnuc_scratch_mem[13]; - MAMAci::view_2d qqcwtend = dropmixnuc_scratch_mem[14]; - - //--------------------------------------------------------------------------- - // ## Initialize the ndrop class. - //--------------------------------------------------------------------------- - const int ntot_amode = mam_coupling::num_aero_modes(); - const int maxd_aspectype = mam4::ndrop::maxd_aspectype; - const int nspec_max = mam4::ndrop::nspec_max; - int nspec_amode[ntot_amode] = {}; - int lspectype_amode[maxd_aspectype][ntot_amode] = {}; - int lmassptr_amode[maxd_aspectype][ntot_amode] = {}; - int numptr_amode[ntot_amode] = {}; - int mam_idx[ntot_amode][nspec_max] = {}; - int mam_cnst_idx[ntot_amode][nspec_max] = {}; - - Real specdens_amode[maxd_aspectype] = {}; - Real spechygro[maxd_aspectype] = {}; - Real exp45logsig[ntot_amode] = {}, alogsig[ntot_amode] = {}, - num2vol_ratio_min_nmodes[ntot_amode] = {}, - num2vol_ratio_max_nmodes[ntot_amode] = {}; - Real aten = 0; - mam4::ndrop::get_e3sm_parameters(nspec_amode, lspectype_amode, lmassptr_amode, - numptr_amode, specdens_amode, spechygro, - mam_idx, mam_cnst_idx); - mam4::ndrop::ndrop_init(exp45logsig, alogsig, aten, num2vol_ratio_min_nmodes, - num2vol_ratio_max_nmodes); - //--------------------------------------------------------------------------- - //--------------------------------------------------------------------------- - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // for (int icol=0; icol<5; ++icol){ - mam4::ndrop::View1D raercol_cw_view[mam4::ndrop::pver][2]; - mam4::ndrop::View1D raercol_view[mam4::ndrop::pver][2]; - for(int i = 0; i < mam4::ndrop::pver; ++i) { - for(int j = 0; j < 2; ++j) { - raercol_cw_view[i][j] = ekat::subview(loc_raercol_cw[i][j], icol); - raercol_view[i][j] = ekat::subview(loc_raercol[i][j], icol); - } - } - mam4::ColumnView qqcw_view[mam4::ndrop::ncnst_tot]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - qqcw_view[i] = ekat::subview(qqcw_fld_work_loc[i], icol); - } - mam4::ColumnView ptend_q_view[mam4::aero_model::pcnst]; - for(int i = 0; i < mam4::aero_model::pcnst; ++i) { - ptend_q_view[i] = ekat::subview(loc_ptend_q[i], icol); - } - mam4::ColumnView coltend_view[mam4::ndrop::ncnst_tot], - coltend_cw_view[mam4::ndrop::ncnst_tot]; - for(int i = 0; i < mam4::ndrop::ncnst_tot; ++i) { - coltend_view[i] = ekat::subview(loc_coltend[i], icol); - coltend_cw_view[i] = ekat::subview(loc_coltend_cw[i], icol); - } - - // To construct state_q, we need fields from Prognostics (aerosols) - // and Atmosphere (water species such as qv, qc etc.) - - // get prognostics - mam4::Prognostics progs_at_col = aerosols_for_column(dry_aero, icol); - - // get atmospheric quantities - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - - // Construct state_q (interstitial) and qqcw (cloud borne) arrays - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; - ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } - - // get qqcw at a grid cell (col,lev) - // NOTE: The layout for qqcw array is based on mam_idx in - // dropmixnuc. To mimic that, we are using the following for-loops - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - qqcw_view[ind_qqcw](klev) = - dry_aero.cld_aero_nmr[m](icol, klev); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - qqcw_view[ind_qqcw](klev) = - dry_aero.cld_aero_mmr[m][a](icol, klev); - ++ind_qqcw; - } - } - } - }); - - mam4::ndrop::dropmixnuc( - team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), - ekat::subview(p_int, icol), ekat::subview(pdel, icol), - ekat::subview(rpdel, icol), - // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] - ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), - ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] - ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, - spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, - num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, - alogsig, aten, mam_idx, mam_cnst_idx, - ekat::subview(qcld, icol), // out - ekat::subview(wsub, icol), // in - ekat::subview(cloud_frac_prev, icol), // in - qqcw_view, // inout - ptend_q_view, ekat::subview(tendnd, icol), - ekat::subview(factnum, icol), ekat::subview(ndropcol, icol), - ekat::subview(ndropmix, icol), ekat::subview(nsource, icol), - ekat::subview(wtke, icol), ekat::subview(ccn, icol), coltend_view, - coltend_cw_view, - // work arrays - raercol_cw_view, raercol_view, ekat::subview(nact, icol), - ekat::subview(mact, icol), ekat::subview(eddy_diff, icol), - ekat::subview(zn, icol), ekat::subview(csbot, icol), - ekat::subview(zs, icol), ekat::subview(overlapp, icol), - ekat::subview(overlapm, icol), ekat::subview(eddy_diff_kp, icol), - ekat::subview(eddy_diff_km, icol), ekat::subview(qncld, icol), - ekat::subview(srcn, icol), ekat::subview(source, icol), - ekat::subview(dz, icol), ekat::subview(csbot_cscen, icol), - ekat::subview(raertend, icol), ekat::subview(qqcwtend, icol)); - }); -} - -// Update cloud borne aerosols -void update_cloud_borne_aerosols( - haero::ThreadTeamPolicy team_policy, - const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, - // output - mam_coupling::AerosolState &dry_aero) { - int ind_qqcw = 0; - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], qqcw_fld_work[ind_qqcw]); - ++ind_qqcw; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - if(dry_aero.cld_aero_mmr[m][a].data()) { - Kokkos::deep_copy(dry_aero.cld_aero_mmr[m][a], qqcw_fld_work[ind_qqcw]); - ++ind_qqcw; - } - } - } -} - -// Update interstitial aerosols using tendencies - levels -void update_interstitial_aerosols_levs( - const haero::ThreadTeam &team, const int nlev, const int icol, - const int s_idx, const Real dt, - const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], - // output - MAMAci::view_2d aero_mr) { - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { - aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; - }); -} - -// Update interstitial aerosols using tendencies- cols and levs -void update_interstitial_aerosols( - haero::ThreadTeamPolicy team_policy, - const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], const int nlev, - const Real dt, - // output - mam_coupling::AerosolState &dry_aero) { - // starting index of ptend_q array (for MAM4, pcnst=40, ncnst_tot=25 ) - int s_idx = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // loop through all modes and species - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - for(int a = 0; a < mam4::num_species_mode(m); ++a) { - // mass mixing ratio of species "a" of mode "m" - auto aero_mmr = dry_aero.int_aero_mmr[m][a]; - - if(aero_mmr.data()) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, - // output - aero_mmr); - }); - // update index for the next species (only if aero_mmr.data() is True) - ++s_idx; - } - } - auto aero_nmr = - dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, - // output - aero_nmr); - }); - ++s_idx; // update index for the next species - } -} - -void call_hetfrz_compute_tendencies( - haero::ThreadTeamPolicy team_policy, mam4::Hetfrz &hetfrz_, - mam_coupling::DryAtmosphere &dry_atm_, - mam_coupling::AerosolState &dry_aero_, MAMAci::view_3d factnum, - const double dt, const int nlev, - // output - MAMAci::view_2d hetfrz_immersion_nucleation_tend, - MAMAci::view_2d hetfrz_contact_nucleation_tend, - MAMAci::view_2d hetfrz_depostion_nucleation_tend, - MAMAci::view_2d diagnostic_scratch_[]) { - mam4::Hetfrz hetfrz = hetfrz_; - mam_coupling::AerosolState dry_aero = dry_aero_; - mam_coupling::DryAtmosphere dry_atmosphere = dry_atm_; - - MAMAci::view_2d diagnostic_scratch[MAMAci::hetro_scratch_]; - for(int i = 0; i < MAMAci::hetro_scratch_; ++i) - diagnostic_scratch[i] = diagnostic_scratch_[i]; - - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - // Set up an atmosphere, surface, diagnostics, pronostics and - // tendencies class. - - haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atmosphere, icol); - haero::Surface surf{}; - mam4::Prognostics progs = - mam_coupling::aerosols_for_column(dry_aero, icol); - - const int accum_idx = static_cast(mam4::ModeIndex::Accumulation); - const int coarse_idx = static_cast(mam4::ModeIndex::Coarse); - - mam4::Diagnostics diags(nlev); - - diags.activation_fraction[accum_idx] = - ekat::subview(factnum, icol, accum_idx); - - diags.activation_fraction[coarse_idx] = - ekat::subview(factnum, icol, coarse_idx); - - // These are the output tendencies from heterogeneous freezing that need - // to be added correctly to the cloud-micorphysics scheme. - diags.hetfrz_immersion_nucleation_tend = - ekat::subview(hetfrz_immersion_nucleation_tend, icol); - diags.hetfrz_contact_nucleation_tend = - ekat::subview(hetfrz_contact_nucleation_tend, icol); - diags.hetfrz_depostion_nucleation_tend = - ekat::subview(hetfrz_depostion_nucleation_tend, icol); - - diags.bc_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst1_num = ekat::subview(diagnostic_scratch[1], icol); - diags.dst3_num = ekat::subview(diagnostic_scratch[2], icol); - diags.bcc_num = ekat::subview(diagnostic_scratch[3], icol); - diags.dst1c_num = ekat::subview(diagnostic_scratch[4], icol); - diags.dst3c_num = ekat::subview(diagnostic_scratch[5], icol); - diags.bcuc_num = ekat::subview(diagnostic_scratch[6], icol); - diags.dst1uc_num = ekat::subview(diagnostic_scratch[7], icol); - diags.dst3uc_num = ekat::subview(diagnostic_scratch[8], icol); - diags.bc_a1_num = ekat::subview(diagnostic_scratch[0], icol); - diags.dst_a1_num = ekat::subview(diagnostic_scratch[10], icol); - diags.dst_a3_num = ekat::subview(diagnostic_scratch[11], icol); - diags.bc_c1_num = ekat::subview(diagnostic_scratch[12], icol); - diags.dst_c1_num = ekat::subview(diagnostic_scratch[13], icol); - diags.dst_c3_num = ekat::subview(diagnostic_scratch[14], icol); - diags.fn_bc_c1_num = ekat::subview(diagnostic_scratch[15], icol); - diags.fn_dst_c1_num = ekat::subview(diagnostic_scratch[16], icol); - diags.fn_dst_c3_num = ekat::subview(diagnostic_scratch[17], icol); - diags.na500 = ekat::subview(diagnostic_scratch[18], icol); - diags.totna500 = ekat::subview(diagnostic_scratch[19], icol); - diags.freqimm = ekat::subview(diagnostic_scratch[20], icol); - diags.freqcnt = ekat::subview(diagnostic_scratch[21], icol); - diags.freqdep = ekat::subview(diagnostic_scratch[22], icol); - diags.freqmix = ekat::subview(diagnostic_scratch[23], icol); - diags.dstfrezimm = ekat::subview(diagnostic_scratch[24], icol); - diags.dstfrezcnt = ekat::subview(diagnostic_scratch[25], icol); - diags.dstfrezdep = ekat::subview(diagnostic_scratch[26], icol); - diags.bcfrezimm = ekat::subview(diagnostic_scratch[27], icol); - diags.bcfrezcnt = ekat::subview(diagnostic_scratch[28], icol); - diags.bcfrezdep = ekat::subview(diagnostic_scratch[19], icol); - diags.nimix_imm = ekat::subview(diagnostic_scratch[30], icol); - diags.nimix_cnt = ekat::subview(diagnostic_scratch[31], icol); - diags.nimix_dep = ekat::subview(diagnostic_scratch[32], icol); - diags.dstnidep = ekat::subview(diagnostic_scratch[33], icol); - diags.dstnicnt = ekat::subview(diagnostic_scratch[34], icol); - diags.dstniimm = ekat::subview(diagnostic_scratch[35], icol); - diags.bcnidep = ekat::subview(diagnostic_scratch[36], icol); - diags.bcnicnt = ekat::subview(diagnostic_scratch[37], icol); - diags.bcniimm = ekat::subview(diagnostic_scratch[38], icol); - diags.numice10s = ekat::subview(diagnostic_scratch[39], icol); - diags.numimm10sdst = ekat::subview(diagnostic_scratch[40], icol); - diags.numimm10sbc = ekat::subview(diagnostic_scratch[41], icol); - diags.stratiform_cloud_fraction = - ekat::subview(diagnostic_scratch[42], icol); - - // assign cloud fraction - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { - diags.stratiform_cloud_fraction(klev) = - haero_atm.cloud_fraction(klev); - }); - //------------------------------------------------------------- - // Heterogeneous freezing - // frzimm, frzcnt, frzdep are the outputs of - // hetfrz_classnuc_cam_calc used by the microphysics (e.g. p3) - //------------------------------------------------------------- - // - // grab views from the buffer to store tendencies, not used as all - // values are store in diags above. - const mam4::Tendencies tends(nlev); - const mam4::AeroConfig aero_config; - const Real t = 0; // not used - hetfrz.compute_tendencies(aero_config, team, t, dt, haero_atm, surf, - progs, diags, tends); - }); -} -} // namespace - MAMAci::MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { // Asserts for the runtime or namelist options From 570e91004f037fd83852a95d57c3e5d421957bd7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 24 May 2024 23:00:35 -0700 Subject: [PATCH 360/476] Fixes GPU access inaccessible memory space error with deep_copy --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 67791fe09af7..b2bcb7e8f77a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -546,14 +546,7 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); // FIXME: Temporary assignment of nc - // Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [=](int kk) { - nc_inp_to_aci_(icol, kk) = wet_atm_.nc(icol, kk); - }); - }); + Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output From 483db9945f8a219ec36f0cc9aa618ddd7bc4e308 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 24 May 2024 23:24:23 -0700 Subject: [PATCH 361/476] Moves an include from hpp to cpp --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 5 +++++ .../src/physics/mam/eamxx_mam_aci_process_interface.hpp | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index b2bcb7e8f77a..fc0324823a06 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -1,6 +1,11 @@ #include + +// ACI functions are stored in the following hpp file #include +// For EKAT units package +#include "ekat/util/ekat_units.hpp" + /* ----------------------------------------------------------------- NOTES: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 093dd571dd90..97bc90d3f982 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -7,9 +7,6 @@ // For declaring ACI class derived from atm process class #include -// For EKAT units package -#include "ekat/util/ekat_units.hpp" - // For aerosol configuration #include "mam4xx/aero_config.hpp" From 1db5be888da0a658052319e720b1ba53bda70baf Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 27 May 2024 23:02:21 -0700 Subject: [PATCH 362/476] Adds a new function to copy views --- .../src/physics/mam/eamxx_mam_aci_functions.hpp | 6 +++--- .../mam/eamxx_mam_aci_process_interface.cpp | 5 ++++- .../eamxx/src/physics/mam/mam_coupling.hpp | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 5174eb652369..87018c929527 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -1,8 +1,8 @@ #ifndef EAMXX_MAM_ACI_FUNCTION_HPP #define EAMXX_MAM_ACI_FUNCTION_HPP -#include #include +#include #include namespace scream { @@ -721,6 +721,6 @@ void call_hetfrz_compute_tendencies( }); } } // namespace -} //namespace scream +} // namespace scream -#endif // ifdef EAMXX_MAM_ACI_FUNCTION_HPP \ No newline at end of file +#endif // ifdef EAMXX_MAM_ACI_FUNCTION_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index fc0324823a06..6fd056f2f2d9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -20,6 +20,8 @@ this assumption should not cause any issues. FUTURE WORK: 1. MAM4xx submodule should point to MAM4xx main branch 2. Link hetrozenous freezing outputs to microphysics +3. Add postcondition and invariant checks +4. Add variables in output.yaml files of the standalone tests ----------------------------------------------------------------- */ @@ -551,7 +553,8 @@ void MAMAci::run_impl(const double dt) { haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); // FIXME: Temporary assignment of nc - Kokkos::deep_copy(nc_inp_to_aci_, wet_atm_.nc); + mam_coupling::copy_view_lev_slice(team_policy, wet_atm_.nc, nlev_, // inputs + nc_inp_to_aci_); // output compute_w0_and_rho(team_policy, dry_atm_, top_lev_, nlev_, // output diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 2d89b573a54d..a80b41f6c08b 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -724,6 +724,23 @@ void compute_wet_mixing_ratios(const Team& team, }); } +// Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses +// "packs". Following function copies a 2d view till model levels +KOKKOS_INLINE_FUNCTION +void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs + const_view_2d &inp_view, //input view to copy + const int dim, //dimension till view should be copied + view_2d &out_view) { //output view + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, dim), [=](int kk) { + out_view(icol, kk) = inp_view(icol, kk); + }); + }); + } + // Because CUDA C++ doesn't allow us to declare and use constants outside of // KOKKOS_INLINE_FUNCTIONS, we define this macro that allows us to (re)define // these constants where needed within two such functions so we don't define From b00adf27b7a8809a436bdd4c0e1a1f2257b832d8 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 28 May 2024 00:43:22 -0700 Subject: [PATCH 363/476] Updates CMAKE for the aci tests to mimic P3 CMAKE to enable np1 vs npX tests --- .../single-process/mam/aci/CMakeLists.txt | 32 +++++++++++++------ .../tests/single-process/mam/aci/output.yaml | 1 - 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 8c5028828bb1..25c83e362edc 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -12,34 +12,46 @@ CreateADUnitTest(${TEST_BASE_NAME} ) # Set AD configurable options -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (ATM_TIME_STEP 600) +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h set (RUN_T0 2021-10-12-45000) - -# Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) -GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) - +## Copy (and configure) yaml files needed by tests configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} - FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} LABELS mam4_aci physics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) +# Check tendency calculation +foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) + set (script ${SCREAM_BASE_DIR}/scripts/check-tendencies) + set (fname mam4_aci_standalone_output.INSTANT.nsteps_x1.np${NRANKS}.${RUN_T0}.nc) + set (tname mam4_aci_tend_check_np${NRANKS}) + add_test (NAME ${tname} + COMMAND ${script} -f ${fname} -v qc T_mid -t mam_aci_qc_tend mam_aci_T_mid_tend + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties (${tname} PROPERTIES + LABELS "mam4_aci;physics" + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) +endforeach() + if (SCREAM_ENABLE_BASELINE_TESTS) # Compare one of the output files with the baselines. # Note: one is enough, since we already check that np1 is BFB with npX - set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) - CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) endif() diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml index 670667576c39..1591d5fe3313 100644 --- a/components/eamxx/tests/single-process/mam/aci/output.yaml +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -11,5 +11,4 @@ Fields: output_control: Frequency: 2 frequency_units: nsteps - MPI Ranks in Filename: true ... From be1417e3825119083107c24f02f16eb39c37f035 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 28 May 2024 01:53:33 -0700 Subject: [PATCH 364/476] Changes after rebase --- .../mam/eamxx_mam_aci_process_interface.cpp | 10 +++---- .../shoc/eamxx_shoc_process_interface.cpp | 27 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 6fd056f2f2d9..4dcc169daa81 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -71,10 +71,10 @@ void MAMAci::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers - q_unit.set_string("kg/kg"); + //q_unit.set_string("kg/kg"); auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + //n_unit.set_string("#/kg"); auto nondim = ekat::units::Units::nondimensional(); @@ -116,9 +116,9 @@ void MAMAci::set_grids( add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); auto m2 = m * m; - m2.set_string("m^2"); + //m2.set_string("m^2"); auto s2 = s * s; - s2.set_string("s^2"); + //s2.set_string("s^2"); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to @@ -240,7 +240,7 @@ void MAMAci::set_grids( // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); - n_unit.set_string("1(cm^-3 s^-1)"); + //n_unit.set_string("1(cm^-3 s^-1)"); // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 904d7f9672ec..15d43c82f7c6 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -23,12 +23,6 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto Qunit = kg/kg; - Qunit.set_string("kg/kg"); - auto nondim = Units::nondimensional(); - m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -56,21 +50,22 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids constexpr int ps = Spack::n; - const auto m2 = m*m; - const auto s2 = s*s; + const auto nondim = Units::nondimensional(); + const auto m2 = pow(m,2); + const auto s2 = pow(s,2); // These variables are needed by the interface, but not actually passed to shoc_main. add_field("omega", scalar3d_mid, Pa/s, grid_name, ps); add_field("surf_sens_flux", scalar2d , W/m2, grid_name); add_field("surf_mom_flux", vector2d , N/m2, grid_name); - add_field("surf_evap", scalar2d , kg/m2/s, grid_name); - add_field ("T_mid", scalar3d_mid, K, grid_name, ps); - add_field ("qv", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("surf_evap", scalar2d , kg/(m2*s), grid_name); + add_field ("T_mid", scalar3d_mid, K, grid_name, ps); + add_field ("qv", scalar3d_mid, kg/kg, grid_name, "tracers", ps); // If TMS is a process, add surface drag coefficient to required fields if (m_params.get("apply_tms", false)) { - add_field("surf_drag_coeff_tms", scalar2d, kg/s/m2, grid_name); + add_field("surf_drag_coeff_tms", scalar2d, kg/(m2*s), grid_name); } // Input variables @@ -84,12 +79,12 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("horiz_winds", vector3d_mid, m/s, grid_name, ps); add_field("sgs_buoy_flux", scalar3d_mid, K*(m/s), grid_name, ps); add_field("eddy_diff_mom", scalar3d_mid, m2/s, grid_name, ps); - add_field("qc", scalar3d_mid, Qunit, grid_name, "tracers", ps); + add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers", ps); add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name, ps); // Output variables - add_field("pbl_height", scalar2d , m, grid_name); - add_field("inv_qc_relvar", scalar3d_mid, Qunit*Qunit, grid_name, ps); + add_field("pbl_height", scalar2d , m, grid_name); + add_field("inv_qc_relvar", scalar3d_mid, pow(kg/kg,2), grid_name, ps); add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); @@ -99,7 +94,7 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids // Boundary flux fields for energy and mass conservation checks if (has_column_conservation_check()) { - add_field("vapor_flux", scalar2d, kg/m2/s, grid_name); + add_field("vapor_flux", scalar2d, kg/(m2*s), grid_name); add_field("water_flux", scalar2d, m/s, grid_name); add_field("ice_flux", scalar2d, m/s, grid_name); add_field("heat_flux", scalar2d, W/m2, grid_name); From d0e870838358f74acc6f99a0c3ce417d5db2ad25 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Tue, 28 May 2024 17:05:05 -0600 Subject: [PATCH 365/476] fix Luca PR comments --- components/eamxx/src/share/field/field.cpp | 16 +++++++------- components/eamxx/src/share/field/field.hpp | 4 ++-- .../src/share/field/field_alloc_prop.cpp | 21 ++++++++----------- .../src/share/field/field_utils_impl.hpp | 8 +++++-- .../share/property_checks/field_nan_check.cpp | 2 ++ 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index e8a3ab4a0581..320e10a60273 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -115,7 +115,9 @@ subfield (const int idim, const int index, const bool dynamic) const { return subfield(m_header->get_identifier().name(),idim,index,dynamic); } -// slice at index idim, entries \in [index_beg, index_end] +// slice at index idim, extracting the N = (index_end - index_beg) entries +// written in math notation: [index_beg, index_end) +// or equivalently, subF = F(index_beg, ... , index_beg + N) Field Field::subfield(const std::string& sf_name, const ekat::units::Units& sf_units, const int idim, const int index_beg, const int index_end) const { @@ -172,7 +174,7 @@ get_component (const int i, const bool dynamic) { return subfield (fname + "_" + std::to_string(i),idim,i,dynamic); } -Field Field::get_components(const int i1, const int i2) { +Field Field::get_components(const int beg, const int end) { const auto& layout = get_header().get_identifier().get_layout(); const auto& fname = get_header().get_identifier().name(); EKAT_REQUIRE_MSG(layout.is_vector_layout(), @@ -181,15 +183,15 @@ Field Field::get_components(const int i1, const int i2) { fname + "': " + e2str(layout.type()) + "\n"); const int idim = layout.get_vector_component_idx(); - EKAT_REQUIRE_MSG(i1 >= 0 && i2 < layout.dim(idim), + EKAT_REQUIRE_MSG(beg >= 0 && end < layout.dim(idim), "Error! Component index range out of bounds [0," + std::to_string(layout.dim(idim)) + ").\n"); - EKAT_REQUIRE_MSG(i1 < i2, "Error! Invalid component indices (i1 >= i2).\n"); + EKAT_REQUIRE_MSG(beg < end, "Error! Invalid component indices (beg >= end).\n"); - // Add _$i1-$i2 to the field name, to avoid issues if the subfield is stored + // Add _$beg-$end to the field name, to avoid issues if the subfield is stored // in some structure that requires unique names (e.g., a remapper) - return subfield(fname + "_" + std::to_string(i1) + "-" + std::to_string(i2), - idim, i1, i2); + return subfield(fname + "_" + std::to_string(beg) + "-" + std::to_string(end), + idim, beg, end); } bool Field::equivalent(const Field& rhs) const diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 858c542fd4ab..d0966daef928 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -248,13 +248,13 @@ class Field { const int idim, const int index_beg, const int index_end) const; Field subfield (const std::string& sf_name, const int idim, const int index_beg, const int index_end) const; - Field subfield (const int idim, const int k_beg, const int k_end) const; + Field subfield (const int idim, const int index_beg, const int index_end) const; // If this field is a vector field, get a subfield for the ith component. // If dynamic = true, it is possible to "reset" the component index at runtime. // Note: throws if this is not a vector field. Field get_component (const int i, const bool dynamic = false); // version for slicing across multiple, contiguous indices - Field get_components (const int i1, const int i2); + Field get_components (const int beg, const int end); // Checks whether the underlying view has been already allocated. bool is_allocated () const { return m_data.d_view.data()!=nullptr; } diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index 48796a826bd9..266fcb88779f 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -78,17 +78,17 @@ subview (const int idim, const int k, const bool dynamic) const { } FieldAllocProp FieldAllocProp::subview(const int idim, - const int k_beg, - const int k_end) const { + const int index_beg, + const int index_end) const { EKAT_REQUIRE_MSG( is_committed(), "Error! Subview requires alloc properties to be committed.\n"); EKAT_REQUIRE_MSG(idim < m_layout.rank(), "Error! Dimension index out of bounds.\n"); - EKAT_REQUIRE_MSG(k_beg < k_end, + EKAT_REQUIRE_MSG(index_beg < index_end, "Error! Slice indices are invalid (non-increasing).\n"); EKAT_REQUIRE_MSG( - k_beg >= 0 && k_end < m_layout.dim(idim), + index_beg >= 0 && index_end < m_layout.dim(idim), "Error! Slice index range along the idim dimension is out of bounds.\n"); // Set new layout basic stuff @@ -96,23 +96,20 @@ FieldAllocProp FieldAllocProp::subview(const int idim, props.m_committed = true; props.m_scalar_type_size = m_scalar_type_size; props.m_layout = m_layout.clone(); - props.m_layout.reset_dim(idim, k_end - k_beg); + props.m_layout.reset_dim(idim, index_end - index_beg); - // Output is contiguous if either - // - this->m_contiguous=true AND idim==0 - // - m_layout.dim(i)==1 for all i; + // below, we can't be sure the field we consider has a continuous allocation, + // so we use get_strided_view() switch (layout.rank()) { case 1: { From d3000b207d02e36c04c9b8b2e98836a8b64cf2f7 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Wed, 15 May 2024 16:15:24 -0700 Subject: [PATCH 366/476] Enable modis_ctptau calculation and output --- .../eamxx/src/physics/cosp/cosp_c2f.F90 | 25 +++++++++++-------- .../eamxx/src/physics/cosp/cosp_functions.hpp | 8 +++--- .../eamxx/src/physics/cosp/eamxx_cosp.cpp | 11 +++++--- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/cosp/cosp_c2f.F90 b/components/eamxx/src/physics/cosp/cosp_c2f.F90 index 957c546c4aa7..febef82c6b04 100644 --- a/components/eamxx/src/physics/cosp/cosp_c2f.F90 +++ b/components/eamxx/src/physics/cosp/cosp_c2f.F90 @@ -43,7 +43,7 @@ module cosp_c2f lsingle = .false., & ! True if using MMF_v3_single_moment CLOUDSAT microphysical scheme (default) ldouble = .true., & ! True if using MMF_v3.5_two_moment CLOUDSAT microphysical scheme lisccp = .true. , & ! Local on/off switch for simulators (used by initialization) - lmodis = .false., & ! + lmodis = .true., & ! lmisr = .false., & ! lcalipso = .false., & ! lgrLidar532 = .false., & ! @@ -82,7 +82,7 @@ module cosp_c2f Lpctmodis = .false., & ! MODIS cloud top pressure Llwpmodis = .false., & ! MODIS cloud liquid water path Liwpmodis = .false., & ! MODIS cloud ice water path - Lclmodis = .false., & ! MODIS cloud area fraction + Lclmodis = .true. , & ! MODIS cloud area fraction Latb532 = .false., & ! CALIPSO attenuated total backscatter (532nm) Latb532gr = .false., & ! GROUND LIDAR @ 532NM attenuated total backscatter (532nm) Latb355 = .false., & ! ATLID attenuated total backscatter (355nm) @@ -239,11 +239,9 @@ module cosp_c2f subroutine cosp_c2f_init(npoints, ncolumns, nlevels) bind(c, name='cosp_c2f_init') integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels - ! Initialize/allocate COSP input and output derived types + + ! Number of vertical levels for Cloudsat/CALIPSO nlvgrid = 40 - call construct_cospIN(npoints,ncolumns,nlevels,cospIN) - call construct_cospstatein(npoints,nlevels,rttov_nchannels,cospstateIN) - call construct_cosp_outputs(npoints, ncolumns, nlevels, nlvgrid, rttov_nchannels, cospOUT) ! Initialize quickbeam_optics, also if two-moment radar microphysics scheme is wanted... if (cloudsat_micro_scheme == 'MMF_v3.5_two_moment') then @@ -264,11 +262,17 @@ subroutine cosp_c2f_init(npoints, ncolumns, nlevels) bind(c, name='cosp_c2f_init cloudsat_radar_freq, cloudsat_k2, cloudsat_use_gas_abs, & cloudsat_do_ray, isccp_topheight, isccp_topheight_direction, surface_radar, & rcfg_cloudsat, use_vgrid, csat_vgrid, Nlvgrid, Nlevels, cloudsat_micro_scheme) + + ! Initialize/allocate COSP input and output derived types + call construct_cospIN(npoints,ncolumns,nlevels,cospIN) + call construct_cospstatein(npoints,nlevels,rttov_nchannels,cospstateIN) + call construct_cosp_outputs(npoints, ncolumns, nlevels, nlvgrid, rttov_nchannels, cospOUT) + end subroutine cosp_c2f_init subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, qv, & - cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau & + cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau, modis_ctptau & ) bind(C, name='cosp_c2f_run') integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels, ntau, nctp real(kind=c_double), value, intent(in) :: emsfc_lw @@ -276,7 +280,7 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & real(kind=c_double), intent(in), dimension(npoints,nlevels) :: T_mid, p_mid, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105 real(kind=c_double), intent(in), dimension(npoints,nlevels+1) :: p_int real(kind=c_double), intent(inout), dimension(npoints) :: isccp_cldtot - real(kind=c_double), intent(inout), dimension(npoints,ntau,nctp) :: isccp_ctptau + real(kind=c_double), intent(inout), dimension(npoints,ntau,nctp) :: isccp_ctptau, modis_ctptau ! Takes normal arrays as input and populates COSP derived types character(len=256),dimension(100) :: cosp_status integer :: nptsperit @@ -351,6 +355,9 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & isccp_cldtot(:npoints) = cospOUT%isccp_totalcldarea(:npoints) isccp_ctptau(:npoints,:,:) = cospOUT%isccp_fq(:npoints,:,:) + ! Modis + modis_ctptau(:npoints,:,:) = cospOUT%modis_Optical_Thickness_vs_Cloud_Top_Pressure(:npoints,:,:) + end subroutine cosp_c2f_run subroutine cosp_c2f_final() bind(C, name='cosp_c2f_final') @@ -455,8 +462,6 @@ end subroutine construct_cospstateIN ! ! This subroutine allocates output fields based on input logical flag switches. ! ###################################################################################### - ! TODO: This is WAY too many dummy arguments! These can just be defined at module scope I think - ! and then initialized once at init subroutine construct_cosp_outputs(Npoints,Ncolumns,Nlevels,Nlvgrid,Nchan,x) ! Inputs integer,intent(in) :: & diff --git a/components/eamxx/src/physics/cosp/cosp_functions.hpp b/components/eamxx/src/physics/cosp/cosp_functions.hpp index f969cdcfed1b..ef6e5493d464 100644 --- a/components/eamxx/src/physics/cosp/cosp_functions.hpp +++ b/components/eamxx/src/physics/cosp/cosp_functions.hpp @@ -8,7 +8,7 @@ extern "C" void cosp_c2f_run(const int ncol, const int nsubcol, const int nlay, const Real emsfc_lw, const Real* sunlit, const Real* skt, const Real* T_mid, const Real* p_mid, const Real* p_int, const Real* qv, const Real* cldfrac, const Real* reff_qc, const Real* reff_qi, const Real* dtau067, const Real* dtau105, - Real* isccp_cldtot, Real* isccp_ctptau); + Real* isccp_cldtot, Real* isccp_ctptau, Real* modis_ctptau); namespace scream { @@ -36,7 +36,7 @@ namespace scream { view_2d& qv , view_2d& cldfrac, view_2d& reff_qc, view_2d& reff_qi, view_2d& dtau067, view_2d& dtau105, - view_1d& isccp_cldtot , view_3d& isccp_ctptau) { + view_1d& isccp_cldtot , view_3d& isccp_ctptau, view_3d& modis_ctptau) { // Make host copies and permute data as needed lview_host_2d @@ -45,6 +45,7 @@ namespace scream { reff_qc_h("reff_qc_h", ncol, nlay), reff_qi_h("reff_qi_h", ncol, nlay), dtau067_h("dtau_067_h", ncol, nlay), dtau105_h("dtau105_h", ncol, nlay); lview_host_3d isccp_ctptau_h("isccp_ctptau_h", ncol, ntau, nctp); + lview_host_3d modis_ctptau_h("modis_ctptau_h", ncol, ntau, nctp); // Copy to layoutLeft host views for (int i = 0; i < ncol; i++) { @@ -72,13 +73,14 @@ namespace scream { emsfc_lw, sunlit.data(), skt.data(), T_mid_h.data(), p_mid_h.data(), p_int_h.data(), qv_h.data(), cldfrac_h.data(), reff_qc_h.data(), reff_qi_h.data(), dtau067_h.data(), dtau105_h.data(), - isccp_cldtot.data(), isccp_ctptau_h.data()); + isccp_cldtot.data(), isccp_ctptau_h.data(), modis_ctptau_h.data()); // Copy outputs back to layoutRight views for (int i = 0; i < ncol; i++) { for (int j = 0; j < ntau; j++) { for (int k = 0; k < nctp; k++) { isccp_ctptau(i,j,k) = isccp_ctptau_h(i,j,k); + modis_ctptau(i,j,k) = modis_ctptau_h(i,j,k); } } } diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 0f4b9f1559e5..30de0e75c4fc 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -83,6 +83,7 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) // Set of fields used strictly as output add_field("isccp_cldtot", scalar2d, percent, grid_name); add_field("isccp_ctptau", scalar4d_ctptau, percent, grid_name, 1); + add_field("modis_ctptau", scalar4d_ctptau, percent, grid_name, 1); add_field("isccp_mask" , scalar2d, nondim, grid_name); } @@ -96,8 +97,8 @@ void Cosp::initialize_impl (const RunType /* run_type */) // Add note to output files about processing ISCCP fields that are only valid during // daytime. This can go away once I/O can handle masked time averages. using stratts_t = std::map; - std::list vnames = {"isccp_cldtot", "isccp_ctptau"}; - for (const auto field_name : {"isccp_cldtot", "isccp_ctptau"}) { + std::list vnames = {"isccp_cldtot", "isccp_ctptau", "modis_ctptau"}; + for (const auto field_name : {"isccp_cldtot", "isccp_ctptau", "modis_ctptau"}) { auto& f = get_field_out(field_name); auto& atts = f.get_header().get_extra_data("io: string attributes"); atts["note"] = "Night values are zero; divide by isccp_mask to get daytime mean"; @@ -160,6 +161,7 @@ void Cosp::run_impl (const double dt) auto dtau105 = get_field_in("dtau105").get_view(); auto isccp_cldtot = get_field_out("isccp_cldtot").get_view(); auto isccp_ctptau = get_field_out("isccp_ctptau").get_view(); + auto modis_ctptau = get_field_out("modis_ctptau").get_view(); auto isccp_mask = get_field_out("isccp_mask" ).get_view(); // Copy of sunlit flag with COSP frequency for proper averaging // Call COSP wrapper routines @@ -170,7 +172,7 @@ void Cosp::run_impl (const double dt) m_num_cols, m_num_subcols, m_num_levs, m_num_isccptau, m_num_isccpctp, emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105, - isccp_cldtot, isccp_ctptau + isccp_cldtot, isccp_ctptau, modis_ctptau ); // Remask night values to ZERO since our I/O does not know how to handle masked/missing values // in temporal averages; this is all host data, so we can just use host loops like its the 1980s @@ -180,6 +182,7 @@ void Cosp::run_impl (const double dt) for (int j = 0; j < m_num_isccptau; j++) { for (int k = 0; k < m_num_isccpctp; k++) { isccp_ctptau(i,j,k) = 0; + modis_ctptau(i,j,k) = 0; } } } @@ -196,10 +199,12 @@ void Cosp::run_impl (const double dt) // TODO: mask this when/if the AD ever supports masked averages Kokkos::deep_copy(isccp_cldtot, 0.0); Kokkos::deep_copy(isccp_ctptau, 0.0); + Kokkos::deep_copy(modis_ctptau, 0.0); Kokkos::deep_copy(isccp_mask , 0.0); } get_field_out("isccp_cldtot").sync_to_dev(); get_field_out("isccp_ctptau").sync_to_dev(); + get_field_out("modis_ctptau").sync_to_dev(); get_field_out("isccp_mask" ).sync_to_dev(); } From d957b7f7f9af2cb508a937c876e9e5d5d631c70b Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Tue, 21 May 2024 12:19:25 -0700 Subject: [PATCH 367/476] Enable MISR simulator and update cosp dimension names --- .../eamxx/src/physics/cosp/cosp_c2f.F90 | 26 ++++---- .../eamxx/src/physics/cosp/cosp_functions.hpp | 25 +++++--- .../eamxx/src/physics/cosp/eamxx_cosp.cpp | 64 ++++++++++++++++--- .../eamxx/src/physics/cosp/eamxx_cosp.hpp | 14 +++- 4 files changed, 97 insertions(+), 32 deletions(-) diff --git a/components/eamxx/src/physics/cosp/cosp_c2f.F90 b/components/eamxx/src/physics/cosp/cosp_c2f.F90 index febef82c6b04..77db4858ccd3 100644 --- a/components/eamxx/src/physics/cosp/cosp_c2f.F90 +++ b/components/eamxx/src/physics/cosp/cosp_c2f.F90 @@ -40,11 +40,11 @@ module cosp_c2f ! Local variables; control what runs and what does not logical :: & - lsingle = .false., & ! True if using MMF_v3_single_moment CLOUDSAT microphysical scheme (default) - ldouble = .true., & ! True if using MMF_v3.5_two_moment CLOUDSAT microphysical scheme + lsingle = .false., & ! True if using MMF_v3_single_moment CLOUDSAT microphysical scheme (default) + ldouble = .true. , & ! True if using MMF_v3.5_two_moment CLOUDSAT microphysical scheme lisccp = .true. , & ! Local on/off switch for simulators (used by initialization) - lmodis = .true., & ! - lmisr = .false., & ! + lmodis = .true. , & ! + lmisr = .true. , & ! lcalipso = .false., & ! lgrLidar532 = .false., & ! latlid = .false., & ! @@ -64,7 +64,7 @@ module cosp_c2f Lmeantbisccp = .false., & ! ISCCP mean all-sky 10.5micron brightness temperature Lmeantbclrisccp = .false., & ! ISCCP mean clear-sky 10.5micron brightness temperature Lalbisccp = .false., & ! ISCCP mean cloud albedo - LclMISR = .false., & ! MISR cloud fraction + LclMISR = .true. , & ! MISR cloud fraction Lcltmodis = .false., & ! MODIS total cloud fraction Lclwmodis = .false., & ! MODIS liquid cloud fraction Lclimodis = .false., & ! MODIS ice cloud fraction @@ -270,17 +270,18 @@ subroutine cosp_c2f_init(npoints, ncolumns, nlevels) bind(c, name='cosp_c2f_init end subroutine cosp_c2f_init - subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & - emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, qv, & - cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau, modis_ctptau & + subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, ncth, & + emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, z_mid, qv, & + cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau, modis_ctptau, misr_cthtau & ) bind(C, name='cosp_c2f_run') - integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels, ntau, nctp + integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels, ntau, nctp, ncth real(kind=c_double), value, intent(in) :: emsfc_lw real(kind=c_double), intent(in), dimension(npoints) :: sunlit, skt - real(kind=c_double), intent(in), dimension(npoints,nlevels) :: T_mid, p_mid, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105 + real(kind=c_double), intent(in), dimension(npoints,nlevels) :: T_mid, p_mid, z_mid, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105 real(kind=c_double), intent(in), dimension(npoints,nlevels+1) :: p_int real(kind=c_double), intent(inout), dimension(npoints) :: isccp_cldtot real(kind=c_double), intent(inout), dimension(npoints,ntau,nctp) :: isccp_ctptau, modis_ctptau + real(kind=c_double), intent(inout), dimension(npoints,ntau,ncth) :: misr_cthtau ! Takes normal arrays as input and populates COSP derived types character(len=256),dimension(100) :: cosp_status integer :: nptsperit @@ -326,7 +327,7 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & ! Translate arrays to derived types cospIN%emsfc_lw = emsfc_lw cospIN%rcfg_cloudsat = rcfg_cloudsat -! cospstateIN%hgt_matrix = zlev(start_idx:end_idx,Nlevels:1:-1) ! km + cospstateIN%hgt_matrix = z_mid(start_idx:end_idx,1:Nlevels) ! m cospstateIN%sunlit = sunlit(start_idx:end_idx) ! 0-1 cospstateIN%skt = skt(start_idx:end_idx) ! K ! cospstateIN%surfelev = surfelev(start_idx:end_idx) ! m @@ -358,6 +359,9 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & ! Modis modis_ctptau(:npoints,:,:) = cospOUT%modis_Optical_Thickness_vs_Cloud_Top_Pressure(:npoints,:,:) + ! MISR + misr_cthtau(:npoints,:,:) = cospOUT%misr_fq(:npoints,:,:) + end subroutine cosp_c2f_run subroutine cosp_c2f_final() bind(C, name='cosp_c2f_final') diff --git a/components/eamxx/src/physics/cosp/cosp_functions.hpp b/components/eamxx/src/physics/cosp/cosp_functions.hpp index ef6e5493d464..37377ca1475e 100644 --- a/components/eamxx/src/physics/cosp/cosp_functions.hpp +++ b/components/eamxx/src/physics/cosp/cosp_functions.hpp @@ -4,11 +4,11 @@ using scream::Real; extern "C" void cosp_c2f_init(int ncol, int nsubcol, int nlay); extern "C" void cosp_c2f_final(); -extern "C" void cosp_c2f_run(const int ncol, const int nsubcol, const int nlay, const int ntau, const int nctp, +extern "C" void cosp_c2f_run(const int ncol, const int nsubcol, const int nlay, const int ntau, const int nctp, const int ncth, const Real emsfc_lw, const Real* sunlit, const Real* skt, - const Real* T_mid, const Real* p_mid, const Real* p_int, const Real* qv, + const Real* T_mid, const Real* p_mid, const Real* p_int, const Real* z_mid, const Real* qv, const Real* cldfrac, const Real* reff_qc, const Real* reff_qi, const Real* dtau067, const Real* dtau105, - Real* isccp_cldtot, Real* isccp_ctptau, Real* modis_ctptau); + Real* isccp_cldtot, Real* isccp_ctptau, Real* modis_ctptau, Real* misr_cthtau); namespace scream { @@ -30,28 +30,30 @@ namespace scream { cosp_c2f_final(); }; inline void main( - const Int ncol, const Int nsubcol, const Int nlay, const Int ntau, const Int nctp, const Real emsfc_lw, + const Int ncol, const Int nsubcol, const Int nlay, const Int ntau, const Int nctp, const Int ncth, const Real emsfc_lw, view_1d& sunlit , view_1d& skt, view_2d& T_mid , view_2d& p_mid , view_2d& p_int, - view_2d& qv , view_2d& cldfrac, + view_2d& z_mid , view_2d& qv , view_2d& cldfrac, view_2d& reff_qc, view_2d& reff_qi, view_2d& dtau067, view_2d& dtau105, - view_1d& isccp_cldtot , view_3d& isccp_ctptau, view_3d& modis_ctptau) { + view_1d& isccp_cldtot , view_3d& isccp_ctptau, view_3d& modis_ctptau, view_3d& misr_cthtau) { // Make host copies and permute data as needed lview_host_2d T_mid_h("T_mid_h", ncol, nlay), p_mid_h("p_mid_h", ncol, nlay), p_int_h("p_int_h", ncol, nlay+1), - qv_h("qv_h", ncol, nlay), cldfrac_h("cldfrac_h", ncol, nlay), + z_mid_h("z_mid_h", ncol, nlay), qv_h("qv_h", ncol, nlay), cldfrac_h("cldfrac_h", ncol, nlay), reff_qc_h("reff_qc_h", ncol, nlay), reff_qi_h("reff_qi_h", ncol, nlay), dtau067_h("dtau_067_h", ncol, nlay), dtau105_h("dtau105_h", ncol, nlay); lview_host_3d isccp_ctptau_h("isccp_ctptau_h", ncol, ntau, nctp); lview_host_3d modis_ctptau_h("modis_ctptau_h", ncol, ntau, nctp); + lview_host_3d misr_cthtau_h("misr_cthtau_h", ncol, ntau, ncth); // Copy to layoutLeft host views for (int i = 0; i < ncol; i++) { for (int j = 0; j < nlay; j++) { T_mid_h(i,j) = T_mid(i,j); p_mid_h(i,j) = p_mid(i,j); + z_mid_h(i,j) = z_mid(i,j); qv_h(i,j) = qv(i,j); cldfrac_h(i,j) = cldfrac(i,j); reff_qc_h(i,j) = reff_qc(i,j); @@ -69,11 +71,11 @@ namespace scream { // Subsample here? // Call COSP wrapper - cosp_c2f_run(ncol, nsubcol, nlay, ntau, nctp, + cosp_c2f_run(ncol, nsubcol, nlay, ntau, nctp, ncth, emsfc_lw, sunlit.data(), skt.data(), T_mid_h.data(), p_mid_h.data(), p_int_h.data(), - qv_h.data(), + z_mid_h.data(), qv_h.data(), cldfrac_h.data(), reff_qc_h.data(), reff_qi_h.data(), dtau067_h.data(), dtau105_h.data(), - isccp_cldtot.data(), isccp_ctptau_h.data(), modis_ctptau_h.data()); + isccp_cldtot.data(), isccp_ctptau_h.data(), modis_ctptau_h.data(), misr_cthtau_h.data()); // Copy outputs back to layoutRight views for (int i = 0; i < ncol; i++) { @@ -82,6 +84,9 @@ namespace scream { isccp_ctptau(i,j,k) = isccp_ctptau_h(i,j,k); modis_ctptau(i,j,k) = modis_ctptau_h(i,j,k); } + for (int k = 0; k < ncth; k++) { + misr_cthtau(i,j,k) = misr_cthtau_h(i,j,k); + } } } } diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 30de0e75c4fc..004aa754207b 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -39,6 +39,8 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) Units nondim = Units::nondimensional(); Units percent (nondim,"%"); auto micron = micro*m; + auto m2 = pow(m, 2); + auto s2 = pow(s, 2); m_grid = grids_manager->get_grid("Physics"); const auto& grid_name = m_grid->name(); @@ -52,8 +54,11 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); FieldLayout scalar4d_ctptau ( {COL,CMP,CMP}, - {m_num_cols,m_num_isccptau,m_num_isccpctp}, - {e2str(COL), "ISCCPTAU", "ISCCPPRS"}); + {m_num_cols,m_num_tau,m_num_ctp}, + {e2str(COL), "cosp_tau", "cosp_prs"}); + FieldLayout scalar4d_cthtau ( {COL,CMP,CMP}, + {m_num_cols,m_num_tau,m_num_cth}, + {e2str(COL), "cosp_tau", "cosp_cth"}); // Set of fields used strictly as input // Name in AD Layout Units Grid Group @@ -69,6 +74,8 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) add_field("qv", scalar3d_mid, kg/kg, grid_name, "tracers"); add_field("qc", scalar3d_mid, kg/kg, grid_name, "tracers"); add_field("qi", scalar3d_mid, kg/kg, grid_name, "tracers"); + add_field("phis", scalar2d , m2/s2, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); add_field("cldfrac_rad", scalar3d_mid, nondim, grid_name); // Optical properties, should be computed in radiation interface add_field("dtau067", scalar3d_mid, nondim, grid_name); // 0.67 micron optical depth @@ -84,6 +91,7 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) add_field("isccp_cldtot", scalar2d, percent, grid_name); add_field("isccp_ctptau", scalar4d_ctptau, percent, grid_name, 1); add_field("modis_ctptau", scalar4d_ctptau, percent, grid_name, 1); + add_field("misr_cthtau" , scalar4d_cthtau, percent, grid_name, 1); add_field("isccp_mask" , scalar2d, nondim, grid_name); } @@ -97,8 +105,8 @@ void Cosp::initialize_impl (const RunType /* run_type */) // Add note to output files about processing ISCCP fields that are only valid during // daytime. This can go away once I/O can handle masked time averages. using stratts_t = std::map; - std::list vnames = {"isccp_cldtot", "isccp_ctptau", "modis_ctptau"}; - for (const auto field_name : {"isccp_cldtot", "isccp_ctptau", "modis_ctptau"}) { + std::list vnames = {"isccp_cldtot", "isccp_ctptau", "modis_ctptau", "misr_cthtau"}; + for (const auto field_name : {"isccp_cldtot", "isccp_ctptau", "modis_ctptau", "misr_cthtau"}) { auto& f = get_field_out(field_name); auto& atts = f.get_header().get_extra_data("io: string attributes"); atts["note"] = "Night values are zero; divide by isccp_mask to get daytime mean"; @@ -145,6 +153,8 @@ void Cosp::run_impl (const double dt) get_field_in("eff_radius_qi").sync_to_host(); get_field_in("dtau067").sync_to_host(); get_field_in("dtau105").sync_to_host(); + get_field_in("phis").sync_to_host(); + get_field_in("pseudo_density").sync_to_host(); auto qv = get_field_in("qv").get_view(); auto qc = get_field_in("qc").get_view(); @@ -154,6 +164,8 @@ void Cosp::run_impl (const double dt) auto T_mid = get_field_in("T_mid").get_view(); auto p_mid = get_field_in("p_mid").get_view(); auto p_int = get_field_in("p_int").get_view(); + auto phis = get_field_in("phis").get_view(); + auto pseudo_density = get_field_in("pseudo_density").get_view(); auto cldfrac = get_field_in("cldfrac_rad").get_view(); auto reff_qc = get_field_in("eff_radius_qc").get_view(); auto reff_qi = get_field_in("eff_radius_qi").get_view(); @@ -162,28 +174,60 @@ void Cosp::run_impl (const double dt) auto isccp_cldtot = get_field_out("isccp_cldtot").get_view(); auto isccp_ctptau = get_field_out("isccp_ctptau").get_view(); auto modis_ctptau = get_field_out("modis_ctptau").get_view(); + auto misr_cthtau = get_field_out("misr_cthtau" ).get_view(); auto isccp_mask = get_field_out("isccp_mask" ).get_view(); // Copy of sunlit flag with COSP frequency for proper averaging + // Compute heights + const auto z_mid = CospFunc::view_2d("z_mid", m_num_cols, m_num_levs); + const auto z_int = CospFunc::view_2d("z_int", m_num_cols, m_num_levs); + const auto dz = z_mid; // reuse tmp memory for dz + const auto ncol = m_num_cols; + const auto nlev = m_num_levs; + // calculate_z_int contains a team-level parallel_scan, which requires a special policy + // TODO: do this on device? + const auto scan_policy = ekat::ExeSpaceUtils::get_thread_range_parallel_scan_team_policy(ncol, nlev); + Kokkos::parallel_for(scan_policy, KOKKOS_LAMBDA (const KTH::MemberType& team) { + const int i = team.league_rank(); + const auto dz_s = ekat::subview(dz, i); + const auto p_mid_s = ekat::subview(p_mid, i); + const auto T_mid_s = ekat::subview(T_mid, i); + const auto qv_s = ekat::subview(qv, i); + const auto z_int_s = ekat::subview(z_int, i); + const auto z_mid_s = ekat::subview(z_mid, i); + const Real z_surf = phis(i) / 9.81; + const auto pseudo_density_s = ekat::subview(pseudo_density, i); + PF::calculate_dz(team, pseudo_density_s, p_mid_s, T_mid_s, qv_s, dz_s); + team.team_barrier(); + PF::calculate_z_int(team,nlev,dz_s,z_surf,z_int_s); + team.team_barrier(); + PF::calculate_z_mid(team,nlev,z_int_s,z_mid_s); + team.team_barrier(); + }); + // Call COSP wrapper routines if (update_cosp) { Real emsfc_lw = 0.99; Kokkos::deep_copy(isccp_mask, sunlit); + CospFunc::view_2d z_mid_c = z_mid; // Need a const version of z_mid for call to CospFunc::main CospFunc::main( - m_num_cols, m_num_subcols, m_num_levs, m_num_isccptau, m_num_isccpctp, - emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, qv, + m_num_cols, m_num_subcols, m_num_levs, m_num_tau, m_num_ctp, m_num_cth, + emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, z_mid_c, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105, - isccp_cldtot, isccp_ctptau, modis_ctptau + isccp_cldtot, isccp_ctptau, modis_ctptau, misr_cthtau ); // Remask night values to ZERO since our I/O does not know how to handle masked/missing values // in temporal averages; this is all host data, so we can just use host loops like its the 1980s for (int i = 0; i < m_num_cols; i++) { if (sunlit(i) == 0) { isccp_cldtot(i) = 0; - for (int j = 0; j < m_num_isccptau; j++) { - for (int k = 0; k < m_num_isccpctp; k++) { + for (int j = 0; j < m_num_tau; j++) { + for (int k = 0; k < m_num_ctp; k++) { isccp_ctptau(i,j,k) = 0; modis_ctptau(i,j,k) = 0; } + for (int k = 0; k < m_num_cth; k++) { + misr_cthtau (i,j,k) = 0; + } } } } @@ -200,11 +244,13 @@ void Cosp::run_impl (const double dt) Kokkos::deep_copy(isccp_cldtot, 0.0); Kokkos::deep_copy(isccp_ctptau, 0.0); Kokkos::deep_copy(modis_ctptau, 0.0); + Kokkos::deep_copy(misr_cthtau , 0.0); Kokkos::deep_copy(isccp_mask , 0.0); } get_field_out("isccp_cldtot").sync_to_dev(); get_field_out("isccp_ctptau").sync_to_dev(); get_field_out("modis_ctptau").sync_to_dev(); + get_field_out("misr_cthtau" ).sync_to_dev(); get_field_out("isccp_mask" ).sync_to_dev(); } diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.hpp b/components/eamxx/src/physics/cosp/eamxx_cosp.hpp index 99f3c179a36a..f7ff41d5a2db 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.hpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.hpp @@ -2,6 +2,7 @@ #define SCREAM_COSP_HPP #include "share/atm_process/atmosphere_process.hpp" +#include "share/util/scream_common_physics_functions.hpp" #include "ekat/ekat_parameter_list.hpp" #include @@ -19,6 +20,10 @@ class Cosp : public AtmosphereProcess { public: + //using Pack = ekat::Pack; + using PF = scream::PhysicsFunctions; + using KT = KokkosTypes; + using KTH = KokkosTypes; // Constructors Cosp (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -49,7 +54,12 @@ class Cosp : public AtmosphereProcess // The three main overrides for the subcomponent void initialize_impl (const RunType run_type); +#ifdef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +public: +#endif void run_impl (const double dt); +protected: void finalize_impl (); // cosp frequency; positive is interpreted as number of steps, negative as number of hours @@ -60,8 +70,8 @@ class Cosp : public AtmosphereProcess Int m_num_cols; Int m_num_subcols; Int m_num_levs; - Int m_num_isccptau = 7; - Int m_num_isccpctp = 7; + Int m_num_tau = 7; + Int m_num_ctp = 7; Int m_num_cth = 16; std::shared_ptr m_grid; From 7c053580b83e410002d6ba0d22f38f6a7137d5f7 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Tue, 21 May 2024 12:37:17 -0700 Subject: [PATCH 368/476] Rename isccp_mask to cosp_sunlit since used for all passive simulators --- components/eamxx/src/physics/cosp/eamxx_cosp.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 004aa754207b..ea7972ff5ebc 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -92,7 +92,7 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) add_field("isccp_ctptau", scalar4d_ctptau, percent, grid_name, 1); add_field("modis_ctptau", scalar4d_ctptau, percent, grid_name, 1); add_field("misr_cthtau" , scalar4d_cthtau, percent, grid_name, 1); - add_field("isccp_mask" , scalar2d, nondim, grid_name); + add_field("cosp_sunlit" , scalar2d, nondim, grid_name); } // ========================================================================================= @@ -109,7 +109,7 @@ void Cosp::initialize_impl (const RunType /* run_type */) for (const auto field_name : {"isccp_cldtot", "isccp_ctptau", "modis_ctptau", "misr_cthtau"}) { auto& f = get_field_out(field_name); auto& atts = f.get_header().get_extra_data("io: string attributes"); - atts["note"] = "Night values are zero; divide by isccp_mask to get daytime mean"; + atts["note"] = "Night values are zero; divide by cosp_sunlit to get daytime mean"; } } @@ -175,7 +175,7 @@ void Cosp::run_impl (const double dt) auto isccp_ctptau = get_field_out("isccp_ctptau").get_view(); auto modis_ctptau = get_field_out("modis_ctptau").get_view(); auto misr_cthtau = get_field_out("misr_cthtau" ).get_view(); - auto isccp_mask = get_field_out("isccp_mask" ).get_view(); // Copy of sunlit flag with COSP frequency for proper averaging + auto cosp_sunlit = get_field_out("isccp_mask" ).get_view(); // Copy of sunlit flag with COSP frequency for proper averaging // Compute heights const auto z_mid = CospFunc::view_2d("z_mid", m_num_cols, m_num_levs); @@ -207,7 +207,7 @@ void Cosp::run_impl (const double dt) // Call COSP wrapper routines if (update_cosp) { Real emsfc_lw = 0.99; - Kokkos::deep_copy(isccp_mask, sunlit); + Kokkos::deep_copy(cosp_sunlit, sunlit); CospFunc::view_2d z_mid_c = z_mid; // Need a const version of z_mid for call to CospFunc::main CospFunc::main( m_num_cols, m_num_subcols, m_num_levs, m_num_tau, m_num_ctp, m_num_cth, @@ -245,13 +245,13 @@ void Cosp::run_impl (const double dt) Kokkos::deep_copy(isccp_ctptau, 0.0); Kokkos::deep_copy(modis_ctptau, 0.0); Kokkos::deep_copy(misr_cthtau , 0.0); - Kokkos::deep_copy(isccp_mask , 0.0); + Kokkos::deep_copy(cosp_sunlit , 0.0); } get_field_out("isccp_cldtot").sync_to_dev(); get_field_out("isccp_ctptau").sync_to_dev(); get_field_out("modis_ctptau").sync_to_dev(); get_field_out("misr_cthtau" ).sync_to_dev(); - get_field_out("isccp_mask" ).sync_to_dev(); + get_field_out("cosp_sunlit" ).sync_to_dev(); } // ========================================================================================= From 976db8490bee4c6c7730fe4feba0d815e650efc2 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Wed, 22 May 2024 14:18:27 -0700 Subject: [PATCH 369/476] Make sure effective radius and mixing ratios are populated for MODIS --- components/eamxx/src/physics/cosp/cosp_c2f.F90 | 13 ++++++++----- .../eamxx/src/physics/cosp/cosp_functions.hpp | 12 ++++++++---- components/eamxx/src/physics/cosp/eamxx_cosp.cpp | 4 ++-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/cosp/cosp_c2f.F90 b/components/eamxx/src/physics/cosp/cosp_c2f.F90 index 77db4858ccd3..25d27d8ca71c 100644 --- a/components/eamxx/src/physics/cosp/cosp_c2f.F90 +++ b/components/eamxx/src/physics/cosp/cosp_c2f.F90 @@ -271,13 +271,13 @@ subroutine cosp_c2f_init(npoints, ncolumns, nlevels) bind(c, name='cosp_c2f_init end subroutine cosp_c2f_init subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, ncth, & - emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, z_mid, qv, & + emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, z_mid, qv, qc, qi, & cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau, modis_ctptau, misr_cthtau & ) bind(C, name='cosp_c2f_run') integer(kind=c_int), value, intent(in) :: npoints, ncolumns, nlevels, ntau, nctp, ncth real(kind=c_double), value, intent(in) :: emsfc_lw real(kind=c_double), intent(in), dimension(npoints) :: sunlit, skt - real(kind=c_double), intent(in), dimension(npoints,nlevels) :: T_mid, p_mid, z_mid, qv, cldfrac, reff_qc, reff_qi, dtau067, dtau105 + real(kind=c_double), intent(in), dimension(npoints,nlevels) :: T_mid, p_mid, z_mid, qv, qc, qi, cldfrac, reff_qc, reff_qi, dtau067, dtau105 real(kind=c_double), intent(in), dimension(npoints,nlevels+1) :: p_int real(kind=c_double), intent(inout), dimension(npoints) :: isccp_cldtot real(kind=c_double), intent(inout), dimension(npoints,ntau,nctp) :: isccp_ctptau, modis_ctptau @@ -311,9 +311,9 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, ncth, & cca(:npoints,:nlevels) = 0 ! Cloud fraction of convective clouds; not present or used in our model dtau_c(:npoints,:nlevels) = 0 ! Optical depth of convective clouds; not present or used in our model dem_c (:npoints,:nlevels) = 0 ! Emissivity of convective clouds; not present or used in our model - mr_lsliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid; will be needed for radar/lidar - mr_ccliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid for convective clouds; not present or used in our model - mr_lsice(:npoints,:nlevels) = 0 ! Mixing ratio of cloud ice; will be needed for radar/lidar + mr_lsliq(:npoints,:nlevels) = qc(:npoints,:nlevels) ! Mixing ratio of cloud liquid; will be needed for radar/lidar + mr_ccliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid for convective clouds; not present or used in our model + mr_lsice(:npoints,:nlevels) = qi(:npoints,:nlevels) ! Mixing ratio of cloud ice; will be needed for radar/lidar mr_ccice(:npoints,:nlevels) = 0 ! Mixing ratio of cloud ice for convective clouds; not present or used in our model mr_lsrain(:npoints,:nlevels) = 0 ! Mixing ratio of rain; will be needed for radar/lidar mr_ccrain(:npoints,:nlevels) = 0 ! Mixing ratio of rain for convective clouds; not present or used in our model @@ -321,6 +321,8 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, ncth, & mr_ccsnow(:npoints,:nlevels) = 0 ! Mixing ratio of snow for convective clouds; will be needed for radar/lidar mr_lsgrpl(:npoints,:nlevels) = 0 ! Mixing ratio of graupel; will be needed for radar/lidar reff = 0 ! Effective radii; will be needed for MODIS + reff(:npoints,:nlevels,I_LSCLIQ) = 1e-6 * reff_qc(:npoints,:nlevels) + reff(:npoints,:nlevels,I_LSCICE) = 1e-6 * reff_qi(:npoints,:nlevels) start_idx = 1 end_idx = npoints @@ -1323,6 +1325,7 @@ subroutine subsample_and_optics(nPoints, nLevels, nColumns, nHydro, overlap, use !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ! MODIS optics + ! TODO: we can probably bypass this block if we take asym and ss_alb from radiation !%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if (Lmodis) then allocate(MODIS_cloudWater(nPoints,nColumns,nLevels), & diff --git a/components/eamxx/src/physics/cosp/cosp_functions.hpp b/components/eamxx/src/physics/cosp/cosp_functions.hpp index 37377ca1475e..1dc7e3bd007c 100644 --- a/components/eamxx/src/physics/cosp/cosp_functions.hpp +++ b/components/eamxx/src/physics/cosp/cosp_functions.hpp @@ -6,7 +6,7 @@ extern "C" void cosp_c2f_init(int ncol, int nsubcol, int nlay); extern "C" void cosp_c2f_final(); extern "C" void cosp_c2f_run(const int ncol, const int nsubcol, const int nlay, const int ntau, const int nctp, const int ncth, const Real emsfc_lw, const Real* sunlit, const Real* skt, - const Real* T_mid, const Real* p_mid, const Real* p_int, const Real* z_mid, const Real* qv, + const Real* T_mid, const Real* p_mid, const Real* p_int, const Real* z_mid, const Real* qv, const Real* qc, const Real* qi, const Real* cldfrac, const Real* reff_qc, const Real* reff_qi, const Real* dtau067, const Real* dtau105, Real* isccp_cldtot, Real* isccp_ctptau, Real* modis_ctptau, Real* misr_cthtau); @@ -33,7 +33,8 @@ namespace scream { const Int ncol, const Int nsubcol, const Int nlay, const Int ntau, const Int nctp, const Int ncth, const Real emsfc_lw, view_1d& sunlit , view_1d& skt, view_2d& T_mid , view_2d& p_mid , view_2d& p_int, - view_2d& z_mid , view_2d& qv , view_2d& cldfrac, + view_2d& z_mid , view_2d& qv , view_2d& qc , view_2d& qi, + view_2d& cldfrac, view_2d& reff_qc, view_2d& reff_qi, view_2d& dtau067, view_2d& dtau105, view_1d& isccp_cldtot , view_3d& isccp_ctptau, view_3d& modis_ctptau, view_3d& misr_cthtau) { @@ -41,7 +42,8 @@ namespace scream { // Make host copies and permute data as needed lview_host_2d T_mid_h("T_mid_h", ncol, nlay), p_mid_h("p_mid_h", ncol, nlay), p_int_h("p_int_h", ncol, nlay+1), - z_mid_h("z_mid_h", ncol, nlay), qv_h("qv_h", ncol, nlay), cldfrac_h("cldfrac_h", ncol, nlay), + z_mid_h("z_mid_h", ncol, nlay), qv_h("qv_h", ncol, nlay), qc_h("qc_h", ncol, nlay), qi_h("qi_h", ncol, nlay), + cldfrac_h("cldfrac_h", ncol, nlay), reff_qc_h("reff_qc_h", ncol, nlay), reff_qi_h("reff_qi_h", ncol, nlay), dtau067_h("dtau_067_h", ncol, nlay), dtau105_h("dtau105_h", ncol, nlay); lview_host_3d isccp_ctptau_h("isccp_ctptau_h", ncol, ntau, nctp); @@ -55,6 +57,8 @@ namespace scream { p_mid_h(i,j) = p_mid(i,j); z_mid_h(i,j) = z_mid(i,j); qv_h(i,j) = qv(i,j); + qc_h(i,j) = qc(i,j); + qi_h(i,j) = qi(i,j); cldfrac_h(i,j) = cldfrac(i,j); reff_qc_h(i,j) = reff_qc(i,j); reff_qi_h(i,j) = reff_qi(i,j); @@ -73,7 +77,7 @@ namespace scream { // Call COSP wrapper cosp_c2f_run(ncol, nsubcol, nlay, ntau, nctp, ncth, emsfc_lw, sunlit.data(), skt.data(), T_mid_h.data(), p_mid_h.data(), p_int_h.data(), - z_mid_h.data(), qv_h.data(), + z_mid_h.data(), qv_h.data(), qc_h.data(), qi_h.data(), cldfrac_h.data(), reff_qc_h.data(), reff_qi_h.data(), dtau067_h.data(), dtau105_h.data(), isccp_cldtot.data(), isccp_ctptau_h.data(), modis_ctptau_h.data(), misr_cthtau_h.data()); diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index ea7972ff5ebc..0997754b4829 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -175,7 +175,7 @@ void Cosp::run_impl (const double dt) auto isccp_ctptau = get_field_out("isccp_ctptau").get_view(); auto modis_ctptau = get_field_out("modis_ctptau").get_view(); auto misr_cthtau = get_field_out("misr_cthtau" ).get_view(); - auto cosp_sunlit = get_field_out("isccp_mask" ).get_view(); // Copy of sunlit flag with COSP frequency for proper averaging + auto cosp_sunlit = get_field_out("cosp_sunlit").get_view(); // Copy of sunlit flag with COSP frequency for proper averaging // Compute heights const auto z_mid = CospFunc::view_2d("z_mid", m_num_cols, m_num_levs); @@ -211,7 +211,7 @@ void Cosp::run_impl (const double dt) CospFunc::view_2d z_mid_c = z_mid; // Need a const version of z_mid for call to CospFunc::main CospFunc::main( m_num_cols, m_num_subcols, m_num_levs, m_num_tau, m_num_ctp, m_num_cth, - emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, z_mid_c, qv, + emsfc_lw, sunlit, skt, T_mid, p_mid, p_int, z_mid_c, qv, qc, qi, cldfrac, reff_qc, reff_qi, dtau067, dtau105, isccp_cldtot, isccp_ctptau, modis_ctptau, misr_cthtau ); From 9ea131ff9d30a85d4e1b3e169d95b6427ab49622 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Tue, 28 May 2024 08:13:17 -0700 Subject: [PATCH 370/476] Add pseudo_density to cosp test input --- components/eamxx/tests/single-process/cosp/input.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/tests/single-process/cosp/input.yaml b/components/eamxx/tests/single-process/cosp/input.yaml index 1447bcc654f1..e3a80db73f09 100644 --- a/components/eamxx/tests/single-process/cosp/input.yaml +++ b/components/eamxx/tests/single-process/cosp/input.yaml @@ -33,6 +33,7 @@ initial_conditions: eff_radius_qi: 0.0 sunlit: 1.0 surf_radiative_T: 288.0 + pseudo_density: 1.0 # The parameters for I/O control Scorpio: From a25da34ade29d4d67bebfefe084fa6847e0e2a11 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Tue, 28 May 2024 21:17:03 -0700 Subject: [PATCH 371/476] Fix whitespace weirdness --- components/eamxx/src/physics/cosp/eamxx_cosp.cpp | 14 +++++++------- components/eamxx/src/physics/cosp/eamxx_cosp.hpp | 7 +++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 0997754b4829..2395aaad3dc7 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -91,8 +91,8 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) add_field("isccp_cldtot", scalar2d, percent, grid_name); add_field("isccp_ctptau", scalar4d_ctptau, percent, grid_name, 1); add_field("modis_ctptau", scalar4d_ctptau, percent, grid_name, 1); - add_field("misr_cthtau" , scalar4d_cthtau, percent, grid_name, 1); - add_field("cosp_sunlit" , scalar2d, nondim, grid_name); + add_field("misr_cthtau", scalar4d_cthtau, percent, grid_name, 1); + add_field("cosp_sunlit", scalar2d, nondim, grid_name); } // ========================================================================================= @@ -174,7 +174,7 @@ void Cosp::run_impl (const double dt) auto isccp_cldtot = get_field_out("isccp_cldtot").get_view(); auto isccp_ctptau = get_field_out("isccp_ctptau").get_view(); auto modis_ctptau = get_field_out("modis_ctptau").get_view(); - auto misr_cthtau = get_field_out("misr_cthtau" ).get_view(); + auto misr_cthtau = get_field_out("misr_cthtau").get_view(); auto cosp_sunlit = get_field_out("cosp_sunlit").get_view(); // Copy of sunlit flag with COSP frequency for proper averaging // Compute heights @@ -244,14 +244,14 @@ void Cosp::run_impl (const double dt) Kokkos::deep_copy(isccp_cldtot, 0.0); Kokkos::deep_copy(isccp_ctptau, 0.0); Kokkos::deep_copy(modis_ctptau, 0.0); - Kokkos::deep_copy(misr_cthtau , 0.0); - Kokkos::deep_copy(cosp_sunlit , 0.0); + Kokkos::deep_copy(misr_cthtau, 0.0); + Kokkos::deep_copy(cosp_sunlit, 0.0); } get_field_out("isccp_cldtot").sync_to_dev(); get_field_out("isccp_ctptau").sync_to_dev(); get_field_out("modis_ctptau").sync_to_dev(); - get_field_out("misr_cthtau" ).sync_to_dev(); - get_field_out("cosp_sunlit" ).sync_to_dev(); + get_field_out("misr_cthtau").sync_to_dev(); + get_field_out("cosp_sunlit").sync_to_dev(); } // ========================================================================================= diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.hpp b/components/eamxx/src/physics/cosp/eamxx_cosp.hpp index f7ff41d5a2db..894c8c9f511a 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.hpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.hpp @@ -20,10 +20,9 @@ class Cosp : public AtmosphereProcess { public: - //using Pack = ekat::Pack; - using PF = scream::PhysicsFunctions; - using KT = KokkosTypes; - using KTH = KokkosTypes; + using PF = scream::PhysicsFunctions; + using KT = KokkosTypes; + using KTH = KokkosTypes; // Constructors Cosp (const ekat::Comm& comm, const ekat::ParameterList& params); From 00cb3f6991df8816517b8e968e2fa461d17bafe3 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Wed, 29 May 2024 16:15:36 -0700 Subject: [PATCH 372/476] Remove unused interface for ZM deep convection The ZM interface we currently have is out-of-date and no longer used. If we decide to start using ZM again it is likely that we need to rewrite the interface to reflect the current paradigm. As a result, this commit removes the old interface to avoid confusion. --- components/eamxx/src/physics/CMakeLists.txt | 3 - .../eamxx/src/physics/zm/CMakeLists.txt | 28 - .../physics/zm/eamxx_zm_process_interface.cpp | 249 - .../physics/zm/eamxx_zm_process_interface.hpp | 71 - .../src/physics/zm/scream_zm_interface.F90 | 198 - .../src/physics/zm/scream_zm_interface.hpp | 39 - components/eamxx/src/physics/zm/zm_conv.F90 | 4348 ----------------- 7 files changed, 4936 deletions(-) delete mode 100644 components/eamxx/src/physics/zm/CMakeLists.txt delete mode 100644 components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp delete mode 100644 components/eamxx/src/physics/zm/eamxx_zm_process_interface.hpp delete mode 100644 components/eamxx/src/physics/zm/scream_zm_interface.F90 delete mode 100644 components/eamxx/src/physics/zm/scream_zm_interface.hpp delete mode 100644 components/eamxx/src/physics/zm/zm_conv.F90 diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index 475fc588cd4a..006426d01447 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -10,9 +10,6 @@ else() message(STATUS "WARNING: RRTMGP and COSP only supported for double precision builds; skipping") endif() add_subdirectory(shoc) -if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) - add_subdirectory(zm) -endif() add_subdirectory(cld_fraction) if (SCREAM_ENABLE_ML_CORRECTION) add_subdirectory(ml_correction) diff --git a/components/eamxx/src/physics/zm/CMakeLists.txt b/components/eamxx/src/physics/zm/CMakeLists.txt deleted file mode 100644 index 9867681599fc..000000000000 --- a/components/eamxx/src/physics/zm/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -set(ZM_SRCS - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/physics_utils.F90 - ${SCREAM_BASE_DIR}/../eam/src/physics/cam/scream_abortutils.F90 - zm_conv.F90 - eamxx_zm_process_interface.cpp - scream_zm_interface.F90 -) - -set(ZM_HEADERS - zm.hpp - eamxx_zm_process_interface.hpp - scream_zm_interface.hpp -) - -# Add ETI source files if not on CUDA/HIP -if (NOT EAMXX_ENABLE_GPU) - list(APPEND ZM_SRCS - ) -endif() - -add_library(zm ${ZM_SRCS}) -set_target_properties(zm PROPERTIES - Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules -) -target_include_directories(zm PUBLIC - ${CMAKE_CURRENT_BINARY_DIR}/modules -) -target_link_libraries(zm physics_share scream_share) diff --git a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp deleted file mode 100644 index f36aac6133d9..000000000000 --- a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.cpp +++ /dev/null @@ -1,249 +0,0 @@ -#include "physics/zm/scream_zm_interface.hpp" -#include "physics/zm/eamxx_zm_process_interface.hpp" - -#include "ekat/ekat_assert.hpp" - -namespace { -// A helper struct and fcn; -struct GridOpts { - GridOpts(const std::string& n, bool out, const ekat::units::Units& u, int idx) - : name(n), isOut(out), unit(u), field_idx(idx) {} - std::string name; - bool isOut; - const ekat::units::Units& unit; - int field_idx; -}; - -void set_grid_opts(std::map& opt_map); -} - -namespace scream -{ - -ZMDeepConvection::ZMDeepConvection (const ekat::Comm& comm,const ekat::ParameterList& params ) - : AtmosphereProcess(comm, params) -{ - // Nothing to do here -} - -void ZMDeepConvection::set_grids(const std::shared_ptr grids_manager) -{ - using namespace std; - using namespace ekat::units; - - constexpr int NVL = 72; /* TODO THIS NEEDS TO BE CHANGED TO A CONFIGURABLE */ - constexpr int QSZ = 35; /* TODO THIS NEEDS TO BE CHANGED TO A CONFIGURABLE */ - auto grid = grids_manager->get_grid("Physics"); - const int num_dofs = grid->get_num_local_dofs(); - const int nc = num_dofs; - - using namespace ShortFieldTagsNames; - - FieldLayout scalar3d_layout_mid { {COL,LEV}, {nc,NVL} }; // Note that C++ and Fortran read array dimensions in reverse - FieldLayout scalar3d_layout_int { {COL,ILEV}, {nc,NVL+1} }; // Note that C++ and Fortran read array dimensions in reverse - FieldLayout vector3d_layout_mid{ {COL,CMP,LEV}, {nc,QSZ,NVL} }; - FieldLayout tracers_layout { {COL,CMP,LEV}, {nc,QSZ,NVL} }; - FieldLayout scalar2d_layout{ {COL}, {nc} }; - - std::vector layout_opts = {scalar3d_layout_mid, scalar3d_layout_int, - vector3d_layout_mid, tracers_layout, scalar2d_layout}; - - std::map opt_map; - set_grid_opts(opt_map); - - for ( auto i = opt_map.begin(); i != opt_map.end(); ++i) { - add_required_field((i->second).name, layout_opts[((i->second).field_idx)], kg/kg, grid->name()); - if ( (i->second).isOut == true ) { - add_computed_field((i->second).name, layout_opts[((i->second).field_idx)], kg/kg, grid->name()); - } - } - -} - -// ========================================================================================= -void ZMDeepConvection::initialize_impl (const RunType /* run_type */) -{ - zm_init_f90 (*m_raw_ptrs_in["limcnv_in"], m_raw_ptrs_in["no_deep_pbl_in"]); -} -// ========================================================================================= -void ZMDeepConvection::run_impl (const double dt) -{ - std::vector in; - std::vector out; - - // Copy inputs to host. Copy also outputs, cause we might "update" them, rather than overwrite them. - for (auto& it : m_zm_fields_in) { - it.second.sync_to_host(); - } - for (auto& it : m_zm_fields_out) { - it.second.sync_to_host(); - } - - Real** temp = &m_raw_ptrs_out["fracis"]; - Real*** fracis = &temp; - - zm_main_f90(*m_raw_ptrs_out["lchnk"], *m_raw_ptrs_out["ncol"], m_raw_ptrs_out["t"], - m_raw_ptrs_out["qh"], m_raw_ptrs_out["prec"], m_raw_ptrs_out["jctop"], - m_raw_ptrs_out["jcbot"], m_raw_ptrs_out["pblh"], m_raw_ptrs_out["zm"], - m_raw_ptrs_out["geos"], m_raw_ptrs_out["zi"], m_raw_ptrs_out["qtnd"], - m_raw_ptrs_out["heat"], m_raw_ptrs_out["pap"], m_raw_ptrs_out["paph"], - m_raw_ptrs_out["dpp"], *m_raw_ptrs_out["delt"], m_raw_ptrs_out["mcon"], - m_raw_ptrs_out["cme"], m_raw_ptrs_out["cape"], m_raw_ptrs_out["tpert"], - m_raw_ptrs_out["dlf"], m_raw_ptrs_out["plfx"], m_raw_ptrs_out["zdu"], - m_raw_ptrs_out["rprd"], m_raw_ptrs_out["mu"], m_raw_ptrs_out["md"], - m_raw_ptrs_out["du"], m_raw_ptrs_out["eu"], m_raw_ptrs_out["ed"], - m_raw_ptrs_out["dp"], m_raw_ptrs_out["dsubcld"], m_raw_ptrs_out["jt"], - m_raw_ptrs_out["maxg"], m_raw_ptrs_out["ideep"], *m_raw_ptrs_out["lengath"], - m_raw_ptrs_out["ql"], m_raw_ptrs_out["rliq"], m_raw_ptrs_out["landfrac"], - m_raw_ptrs_out["hu_nm1"], m_raw_ptrs_out["cnv_nm1"], m_raw_ptrs_out["tm1"], - m_raw_ptrs_out["qm1"], &m_raw_ptrs_out["t_star"], &m_raw_ptrs_out["q_star"], - m_raw_ptrs_out["dcape"], m_raw_ptrs_out["qv"], &m_raw_ptrs_out["tend_s"], - &m_raw_ptrs_out["tend_q"], &m_raw_ptrs_out["cld"], m_raw_ptrs_out["snow"], - m_raw_ptrs_out["ntprprd"], m_raw_ptrs_out["ntsnprd"], - &m_raw_ptrs_out["flxprec"], &m_raw_ptrs_out["flxsnow"], - *m_raw_ptrs_out["ztodt"], m_raw_ptrs_out["pguall"], m_raw_ptrs_out["pgdall"], - m_raw_ptrs_out["icwu"], *m_raw_ptrs_out["ncnst"], fracis); - auto ts = timestamp(); - ts += dt; - for (auto& it : m_zm_fields_out) { - it.second.get_header().get_tracking().update_time_stamp(ts); - } -} -// ========================================================================================= -void ZMDeepConvection::finalize_impl() -{ - zm_finalize_f90 (); -} -// ========================================================================================= -void ZMDeepConvection:: -register_fields (const std::map>>& field_mgrs) const { - auto& field_mgr = *field_mgrs.at("Physics"); - for (const auto& fid : get_required_fields()) { - field_mgr.register_field(fid); - } - for (const auto& fid : get_computed_fields()) { - field_mgr.register_field(fid); - } - } - -void ZMDeepConvection::set_required_field_impl (const Field& f) { - // @Meredith: Diff between add_me_as_a_customer and get_tracking().add_customer? - - // Store a copy of the field. We need this in order to do some tracking checks - // at the beginning of the run call. Other than that, there would be really - // no need to store a scream field here; we could simply set the view ptr - // in the Homme's view, and be done with it. - const auto& name = f.get_header().get_identifier().name(); - m_zm_fields_in.emplace(name,f); - m_zm_host_views_in[name] = f.get_view(); - m_raw_ptrs_in[name] = m_zm_host_views_in[name].data(); - - // Add myself as customer to the field - add_me_as_customer(f); - -} - -void ZMDeepConvection::set_computed_field_impl (const Field< Real>& f) { - // Store a copy of the field. We need this in order to do some tracking updates - // at the end of the run call. Other than that, there would be really - // no need to store a scream field here; we could simply set the view ptr - // in the Homme's view, and be done with it. - const auto& name = f.get_header().get_identifier().name(); - m_zm_fields_out.emplace(name,f); - m_zm_host_views_out[name] = f.get_view(); - m_raw_ptrs_out[name] = m_zm_host_views_out[name].data(); - - // Add myself as provider for the field - add_me_as_provider(f); -} - -} // namespace scream - -namespace { -//Initializes struct GridOpts fields -void set_grid_opts_helper(std::map& opt_map, - const std::string& n, bool out, int field_idx, - const ekat::units::Units& unit = ekat::units::Units::nondimensional()) -{ - opt_map.emplace(std::piecewise_construct, - std::forward_as_tuple(n), - std::forward_as_tuple(n,out,unit,field_idx)); -} - -void set_grid_opts(std::map& opt_map){ - //Layout options are set as an int - //to be passed into the GridOpts struct - const int SCALAR_3D_MID = 0; - const int SCALAR_3D_INT = 1; - const int VECTOR_3D_MID = 2; - const int LINEAR = 4; - - const auto Pa = ekat::units::Pa; - - // Sets the value of the grid opt struct categories - set_grid_opts_helper(opt_map, "limcnv_in", true, 0); - set_grid_opts_helper(opt_map, "no_deep_pbl_in", true, VECTOR_3D_MID); - set_grid_opts_helper(opt_map, "lchnk", true, SCALAR_3D_MID, Pa); //temperature(K) - set_grid_opts_helper(opt_map, "ncol", true, SCALAR_3D_MID, Pa); //temperature(K) - set_grid_opts_helper(opt_map, "t", true, SCALAR_3D_MID, Pa); //temperature(K) - set_grid_opts_helper(opt_map, "qh", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "prec", true, LINEAR); - set_grid_opts_helper(opt_map, "jctop", true, LINEAR); - set_grid_opts_helper(opt_map, "jcbot", true, LINEAR); - set_grid_opts_helper(opt_map, "pblh", true, LINEAR); - set_grid_opts_helper(opt_map, "zm", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "geos", true, LINEAR); - set_grid_opts_helper(opt_map, "zi", true, SCALAR_3D_INT); - set_grid_opts_helper(opt_map, "qtnd", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "heat", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "pap", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "paph", true, SCALAR_3D_INT); - set_grid_opts_helper(opt_map, "dpp", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "delt", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "mcon", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "cme", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "cape", true, LINEAR); - set_grid_opts_helper(opt_map, "tpert", true, LINEAR); - set_grid_opts_helper(opt_map, "dlf", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "pflx", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "zdu", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "rprd", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "mu", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "md", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "du", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "eu", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "ed", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "dp", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "dsubcld", true, LINEAR); - set_grid_opts_helper(opt_map, "jt", true, LINEAR); - set_grid_opts_helper(opt_map, "maxg", true, LINEAR); - set_grid_opts_helper(opt_map, "ideep", true, LINEAR); - set_grid_opts_helper(opt_map, "lengath", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "ql", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "rliq", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "landfrac", true, LINEAR); - set_grid_opts_helper(opt_map, "hu_nm1", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "cnv_nm1", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "tm1", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "qm1", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "t_star", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "q_star", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "dcape", true, LINEAR); - set_grid_opts_helper(opt_map, "qv", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "tend_s", true, LINEAR); - set_grid_opts_helper(opt_map, "tend_q", true, LINEAR); - set_grid_opts_helper(opt_map, "cld", true, LINEAR); - set_grid_opts_helper(opt_map, "snow", true, LINEAR); - set_grid_opts_helper(opt_map, "ntprprd", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "ntsnprd", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "flxprec", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "flxsnow", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "ztodt", true, SCALAR_3D_MID); - set_grid_opts_helper(opt_map, "pguall", true, VECTOR_3D_MID); - set_grid_opts_helper(opt_map, "pgdall", true, VECTOR_3D_MID); - set_grid_opts_helper(opt_map, "icwu", true, VECTOR_3D_MID); - set_grid_opts_helper(opt_map, "ncnst", true, VECTOR_3D_MID); - set_grid_opts_helper(opt_map, "fracis", true, VECTOR_3D_MID); -} -} // anonymous namespace - diff --git a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.hpp b/components/eamxx/src/physics/zm/eamxx_zm_process_interface.hpp deleted file mode 100644 index 8b25a4b09ecd..000000000000 --- a/components/eamxx/src/physics/zm/eamxx_zm_process_interface.hpp +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef SCREAM_ZM_DEEPCONVECTION_HPP -#define SCREAM_ZM_DEEPCONVECTION_HPP - -#include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" - -#include - -namespace scream -{ - -/* - * The class responsible to handle the atmosphere deep convection - * - * The AD should store exactly ONE instance of this class stored - * in its list of subcomponents (the AD should make sure of this). - * -*/ - -class ZMDeepConvection : public AtmosphereProcess -{ -public: - using field_type = Field< Real>; - using const_field_type = Field; - - // Constructors - ZMDeepConvection (const ekat::Comm& comm, const ekat::ParameterList& params); - - // The type of subcomponent - AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } - - // The name of the subcomponent - std::string name () const { return "DeepConvection"; } - - // Set the grid - void set_grids (const std::shared_ptr grids_manager); - - // The three main interfaces for the subcomponent - void initialize_impl (const RunType run_type); - void run_impl (const double dt); - void finalize_impl (); - - // Register all fields in the proper field manager(s). - // Note: field_mgrs[grid_name] is the FM on grid $grid_name - void register_fields (const std::map>>& field_mgrs) const; - -protected: - - std::map m_zm_fields_in; - std::map m_zm_fields_out; - - template - using view_type = field_type::view_type; - - template - using host_view_type = field_type::get_view_type,Host>; - - using host_view_in_type = host_view_type; - using host_view_out_type = host_view_type< field_type::RT>; - - std::map m_zm_host_views_in; - std::map m_zm_host_views_out; - - std::map m_raw_ptrs_in; - std::map m_raw_ptrs_out; - -}; // class ZMDeepConvection - -} // namespace scream - -#endif // SCREAM_ZM_DEEPCONVECTION_HPP diff --git a/components/eamxx/src/physics/zm/scream_zm_interface.F90 b/components/eamxx/src/physics/zm/scream_zm_interface.F90 deleted file mode 100644 index 61840a8e7eca..000000000000 --- a/components/eamxx/src/physics/zm/scream_zm_interface.F90 +++ /dev/null @@ -1,198 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -module scream_zm_interface_mod - - use iso_c_binding, only: c_ptr, c_f_pointer, c_int, c_double, c_bool,C_NULL_CHAR, c_float - use physics_utils, only: r8 => rtype, rtype8, itype, btype - use zm_conv, only: zm_convr, zm_conv_evap, zm_convi, convtran, momtran,zmconv_readnl - - implicit none -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double - integer,parameter,public :: rtype = c_double ! 8 byte real, compatible with c type double -#else -# define c_real c_float - integer,parameter,public :: rtype = c_float ! 4 byte real, compatible with c type #endif -#endif - - - - public :: zm_init_f90 - public :: zm_main_f90 - public :: zm_finalize_f90 - - real :: test - - integer(kind=c_int) :: pcols = 32 - integer(kind=c_int) :: pver = 73 - integer(kind=c_int) :: pverp = 75 - - ! real(kind=c_real) :: pcols = 32 - ! real(kind=c_real) :: pver = 75 - real(kind=c_real) :: cpair != 1004.64000000000 - real(kind=c_real) :: gravit != 9.80616000000000 - real(kind=c_real) :: latvap != 2501000.00000000 - real(kind=c_real) :: plevp = 0.0 - -! real(kind=c_real) :: pverp = 0 - -contains - - !====================================================================! - - subroutine zm_init_f90 (limcnv_in, no_deep_pbl_in) bind(c) - integer, intent(in) :: limcnv_in - logical, intent(in), optional :: no_deep_pbl_in - call zm_convi(limcnv_in, no_deep_pbl_in) - call zmconv_readnl() - - end subroutine zm_init_f90 - !====================================================================! -subroutine zm_main_f90(lchnk ,ncol , & - t ,qh ,prec ,jctop ,jcbot , & - pblh ,zm ,geos ,zi ,qtnd , & - heat ,pap ,paph ,dpp , & - delt ,mcon ,cme ,cape , & - tpert ,dlf ,pflx ,zdu ,rprd, & - mu ,md ,du ,eu ,ed , & - dp ,dsubcld ,jt ,maxg ,ideep , & - lengath ,ql ,rliq ,landfrac,hu_nm1 , & - cnv_nm1 ,tm1 ,qm1 ,t_star ,q_star , & - dcape ,q ,tend_s ,tend_q ,cld , & - snow ,ntprprd ,ntsnprd ,flxprec ,flxsnow, & - ztodt , pguall , pgdall , icwu , ncnst, fracis) bind(c) - integer :: lchnk - integer lengath - integer, intent(in) :: ncol - real(kind=c_real), intent(inout) :: t(pcols,pver) ! State array kg/kg - real(kind=c_real) :: qh(pcols,pver) - real(kind=c_real), intent(out) :: prec(pcols) - real(kind=c_real), intent(out) :: jctop(pcols) ! o row of top-of-deep-convection indices passed out. - real(kind=c_real), intent(out) :: jcbot(pcols) ! o row of base of cloud indices passed out. - integer jt(pcols) ! wg top level index of deep cumulus convection. - integer maxg(pcols) ! wg gathered values of maxi. - integer ideep(pcols) ! w holds position of gathered points vs longitude index. - integer mx(pcols) -! -! real(r8) landfracg(pcols) ! wg grid slice of landfrac - real(kind=c_real) delt ! length of model time-step in seconds. -! real(r8) :: pcont(pcols), pconb(pcols), freqzm(pcols) -! - real(kind=c_real), intent(in) :: pblh(pcols) - real(kind=c_real), intent(in) :: geos(pcols) - real(kind=c_real), intent(in) :: zi(pcols,pver+1) - real(kind=c_real), intent(in) :: zm(pcols,pver) - real(kind=c_real), intent(in) :: pap(pcols,pver) - real(kind=c_real), intent(in) :: paph(pcols,pver+1) - real(kind=c_real), intent(in) :: dpp(pcols,pver) ! local sigma half-level thickness (i.e. dshj). - real(kind=c_real), intent(in) :: tpert(pcols) - real(kind=c_real), intent(in) :: tm1(pcols,pver) ! grid slice of temperature at mid-layer. - real(kind=c_real), intent(in) :: qm1(pcols,pver) ! grid slice of specific humidity. - real(kind=c_real), intent(in) :: landfrac(pcols) ! RBN Landfrac - - - real(kind=c_real), intent(in) :: t_star(pcols,pver) ! intermediate T between n and n-1 time step - real(kind=c_real), intent(in) :: q_star(pcols,pver) ! intermediate q between n and n-1 time step - - real(kind=c_real), intent(out) :: qtnd(pcols,pver) ! specific humidity tendency (kg/kg/s) - real(kind=c_real), intent(out) :: heat(pcols,pver) ! heating rate (dry static energy tendency, W/kg) - real(kind=c_real), intent(out) :: mcon(pcols,pverp) - real(kind=c_real), intent(out) :: dlf(pcols,pver) ! scattrd version of the detraining cld h2o tend - real(kind=c_real), intent(out) :: pflx(pcols,pverp) ! scattered precip flux at each level - real(kind=c_real), intent(out) :: cme(pcols,pver) - real(kind=c_real), intent(out) :: cape(pcols) ! w convective available potential energy. - real(kind=c_real), intent(out) :: zdu(pcols,pver) - real(kind=c_real), intent(out) :: rprd(pcols,pver) ! rain production rate - real(kind=c_real), intent(out) :: mu(pcols,pver) - real(kind=c_real), intent(out) :: md(pcols,pver) - real(kind=c_real), intent(out) :: du(pcols,pver) ! detrainement rate of updraft - real(kind=c_real), intent(out) :: ed(pcols,pver) ! entrainment rate of downdraft - real(kind=c_real), intent(out) :: eu(pcols,pver) ! entrainment rate of updraft - real(kind=c_real), intent(out) :: dp(pcols,pver) ! wg layer thickness in mbs (between upper/lower interface). - real(kind=c_real), intent(out) :: dsubcld(pcols) ! wg layer thickness in mbs between lcl and maxi. - real(kind=c_real), intent(out) :: ql(pcols,pver) - real(kind=c_real), intent(out) :: rliq(pcols) ! reserved liquid (not yet in cldliq) for energy integrals - real(kind=c_real), intent(out) :: dcape(pcols) ! output dynamical CAPE -! - real(kind=c_real), intent(inout) :: hu_nm1 (pcols,pver) - real(kind=c_real), intent(inout) :: cnv_nm1 (pcols,pver) -! -! - real(kind=c_real) q(pcols,pver) -! -!! real(r8), pointer, dimension(:,:) :: rprd ! rain production rate -!!Used for convtran exclusively - real(kind=c_real), intent(in) :: fracis(pcols,pver,ncnst) - real(kind=c_real) :: fake_dpdry(pcols,pver) - real(kind=c_real) :: fake_dqdt(pcols,pver,ncnst) ! Tracer tendency array -! - integer :: il1g - integer :: nstep -!! -! - real(kind=c_real) :: tend_s_snwprd (pcols,pver) ! Heating rate of snow production - real(kind=c_real) :: tend_s_snwevmlt(pcols,pver) ! Heating rate of evap/melting of snow - real(kind=c_real),intent(inout), dimension(pcols,pver) :: tend_s ! heating rate (J/kg/s) - real(kind=c_real),intent(inout), dimension(pcols,pver) :: tend_q ! heating rate (J/kg/s) - real(kind=c_real), intent(inout), dimension(pcols,pver) :: cld - real(kind=c_real), intent(inout), dimension(pcols) :: snow ! snow from ZM convection - real(kind=c_real) :: ntprprd(pcols,pver) ! evap outfld: net precip production in layer - real(kind=c_real) :: ntsnprd(pcols,pver) ! evap outfld: net snow production in layer - real(kind=c_real), intent(out), dimension(pcols,pver) :: flxprec ! Convective-scale flux of precip at interfaces (kg/m2/s) - real(kind=c_real), intent(out), dimension(pcols,pver) :: flxsnow ! Convective-scale flux of snow at interfaces (kg/m2/s) - real(kind=c_real), intent(in) :: ztodt ! 2 delta t (model time increment) - real(kind=c_real) :: pguall(pcols, pver, 2) - real(kind=c_real) :: pgdall(pcols, pver, 2) - real(kind=c_real) :: icwu(pcols,pver, 2) - real(kind=c_real) :: icwd(pcols,pver, 2) - real(kind=c_real) :: seten(pcols, pver) - integer, intent(in) :: ncnst -! - logical :: domomtran(ncnst) - logical :: doconvtran(ncnst) - fake_dpdry(:,:) = 0._r8 - il1g = 1 - - call zm_convr(lchnk ,ncol , & - t ,qh ,prec ,jctop ,jcbot , & - pblh ,zm ,geos ,zi ,qtnd , & - heat ,pap ,paph ,dpp , & - delt ,mcon ,cme ,cape , & - tpert ,dlf ,pflx ,zdu ,rprd , & - mu ,md ,du ,eu ,ed , & - dp ,dsubcld ,jt ,maxg ,ideep , & - lengath ,ql ,rliq ,landfrac,hu_nm1 , & - cnv_nm1 ,tm1 ,qm1 ,t_star ,q_star, dcape) - - call zm_conv_evap(ncol ,lchnk, & - t ,pap ,dpp ,q , & - tend_s, tend_s_snwprd ,tend_s_snwevmlt , & - tend_q ,rprd, cld ,ztodt , & - prec, snow, ntprprd, ntsnprd, flxprec, flxsnow ) - - - - call momtran(lchnk, ncol, & - domomtran,q ,ncnst ,mu ,md , & - du ,eu ,ed ,dp ,dsubcld , & - jt ,mx ,ideep ,il1g ,lengath , & - nstep ,fake_dqdt ,pguall ,pgdall, icwu, icwd, ztodt, seten ) - - call convtran(lchnk , & - doconvtran,q ,ncnst ,mu ,md , & - du ,eu ,ed ,dp ,dsubcld , & - jt ,mx ,ideep ,il1g , lengath , & - 1 ,fracis ,fake_dqdt, fake_dpdry ) - - end subroutine zm_main_f90 - !====================================================================! - subroutine zm_finalize_f90 () bind(c) - - - end subroutine zm_finalize_f90 - !====================================================================! - -end module scream_zm_interface_mod diff --git a/components/eamxx/src/physics/zm/scream_zm_interface.hpp b/components/eamxx/src/physics/zm/scream_zm_interface.hpp deleted file mode 100644 index ffeb15707cd8..000000000000 --- a/components/eamxx/src/physics/zm/scream_zm_interface.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef SCREAM_ZM_INTERFACE_HPP -#define SCREAM_ZM_INTERFACE_HPP - -#include "ekat/ekat_assert.hpp" -//#include "ekat/util/scream_utils.hpp" -#include "ekat/util/ekat_file_utils.hpp" - -#include "share/scream_types.hpp" - -// Put everything into a scream namespace - - -namespace scream { - -extern "C" -{ - -// Fortran routines to be called from C -void zm_init_f90 (const Real& limcnv_in, const bool& no_deep_pbl_in); -void zm_main_f90(const Real& lchnk, const Real& ncol, Real* t, Real* qh, Real* prec, - Real* jctop, Real* jcbot, Real* pblh, Real *zm, Real* geos, Real* zi, - Real* qtnd, Real* heat, Real* pap, Real* paph, Real* dpp, const Real &delt, - Real* mcon, Real* cme, Real* cape, Real* tpert, Real* dlf, Real* plfx, - Real* zdu, Real* rprd, Real* mu, Real* md, Real* du, Real* eu, - Real* ed, Real* dp, Real* dsubcld, Real* jt, Real* maxg, Real* ideep, - const Real& lengath, Real* ql, Real* rliq, Real* landfrac, Real* hu_nm1, - Real* cnv_nm1, Real* tm1, Real* qm1, Real** t_star, Real** q_star, - Real* dcape, Real* q, Real** tend_s, - Real** tend_q, Real** cld, Real* snow, Real* ntprprd, Real* ntsnprd, - Real** flxprec, Real** flxsnow, const Real& ztodt, Real* pguall, Real* pgdall, - Real* icwu, const Real& ncnst, Real*** fracis - ); -void zm_finalize_f90 (); - -} // extern "C" - -} // namespace scream - -#endif // SCREAM_SHOC_INTERFACE_HPP diff --git a/components/eamxx/src/physics/zm/zm_conv.F90 b/components/eamxx/src/physics/zm/zm_conv.F90 deleted file mode 100644 index 507d6f43d959..000000000000 --- a/components/eamxx/src/physics/zm/zm_conv.F90 +++ /dev/null @@ -1,4348 +0,0 @@ - -module zm_conv - -!--------------------------------------------------------------------------------- -! Purpose: -! -! Interface from Zhang-McFarlane convection scheme, includes evaporation of convective -! precip from the ZM scheme -! -! Apr 2006: RBN: Code added to perform a dilute ascent for closure of the CM mass flux -! based on an entraining plume a la Raymond and Blythe (1992) -! -! Author: Byron Boville, from code in tphysbc -! -!--------------------------------------------------------------------------------- - use iso_c_binding, only: c_ptr, c_f_pointer, c_int, c_double, c_bool,C_NULL_CHAR, c_float - - use physics_utils, only: r8 => rtype, rtype8, itype, btype -! use shr_kind_mod, only: r8 => shr_kind_r8 -! use spmd_utils, only: masterproc - ! use ppgrid, only: pcols, pver, pverp -! use cloud_fraction, only: cldfrc_fice - ! use physconst, only: cpair, epsilo, gravit, latice, latvap, tmelt, rair, & - ! cpwv, cpliq, rh2o -! use cam_abortutils, only: endrun -! use cam_logfile, only: iulog - use scream_abortutils, only: endscreamrun - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - - save - private ! Make default type private to the module -! -! PUBLIC: interfaces -! - public zmconv_readnl ! read zmconv_nl namelist - public zm_convi ! ZM schemea - public zm_convr ! ZM schemea - public zm_conv_evap ! evaporation of precip from ZM schemea -! AaronDonahue - TODO Question: Are convtran and momtran going to be used? Do -! not appear to be used by zm_convr - public convtran ! convective transport - public momtran ! convective momentum transport - public trigmem ! true if convective memory - public trigdcape_ull ! true if to use dcape-ULL trigger - public is_first_step -! -! Private data -! - - integer(kind=c_int) :: pcols = 32 - integer(kind=c_int) :: pver = 72 - integer(kind=c_int) :: pverp = 73 - - logical :: masterproc = .false. - real(kind=c_real) :: cpair = 1004.64000000000 - real(kind=c_real) :: rh2o = 461.504639820160 - real(kind=c_real) :: gravit = 9.80616000000000 - real(kind=c_real) :: latvap = 2501000.00000000 - real(kind=c_real) :: latice = 333700.000000000 - real(kind=c_real) :: tmelt = 273.150000000000 - real(kind=c_real) :: rair = 287.042311365049 - real(kind=c_real) :: cpliq = 4188.00000000000 - real(kind=c_real) :: cpwv = 1.810e3 - real(kind=c_real) :: epsilo = 18.0160000000000/28.9660000000000 !mwh20/mwdry - - real(r8), parameter :: unset_r8 = huge(1.0_r8) - integer , parameter :: unset_int = huge(1) - real(r8) :: zmconv_alfa = unset_r8 - - real(r8) rl ! wg latent heat of vaporization. - real(r8) cpres ! specific heat at constant pressure in j/kg-degk. - real(r8), parameter :: capelmt = 70._r8 ! threshold value for cape for deep convection. -!songxl 2014-05-20------------------ - -!DCAPE-ULL - real(r8), parameter :: trigdcapelmt = 0._r8 ! threshold value of dcape for deep convection - logical :: trigdcape_ull = .false. !true to use DCAPE trigger and -ULL - integer, allocatable :: dcapemx(:) ! save maxi from 1st call for CAPE calculation and used in 2nd call when DCAPE-ULL active -! May need to change to use local variable ! as passed via dummy argument. For now, making it threadprivate as follows, -!$omp threadprivate (dcapemx) - - real(r8) :: ke ! Tunable evaporation efficiency set from namelist input zmconv_ke - real(r8) :: c0_lnd ! set from namelist input zmconv_c0_lnd - real(r8) :: c0_ocn ! set from namelist input zmconv_c0_ocn - real(r8) :: dmpdz = unset_r8 ! Parcel fractional mass entrainment rate (/m) - real(r8) :: alfa_scalar ! maximum downdraft mass flux fraction - real(r8) :: tiedke_add = unset_r8 - logical :: trigmem ! set from namelist input zmconv_trigmem - integer :: num_cin = unset_int !number of negative buoyancy regions that are allowed before the conv. top and CAPE calc are completed - integer :: mx_bot_lyr_adj = unset_int !bottom layer adjustment for setting "launching" level(mx) (to be at maximum moist static energy). - real(r8) tau ! convective time scale - real(r8),parameter :: c1 = 6.112_r8 - real(r8),parameter :: c2 = 17.67_r8 - real(r8),parameter :: c3 = 243.5_r8 - real(r8) :: tfreez - real(r8) :: eps1 - - - logical :: no_deep_pbl ! default = .false. - ! no_deep_pbl = .true. eliminates deep convection entirely within PBL - - -!moved from moistconvection.F90 - real(r8) :: rgrav ! reciprocal of grav - real(r8) :: rgas ! gas constant for dry air - real(r8) :: grav ! = gravit - real(r8) :: cp ! = cpres = cpair - - integer limcnv ! top interface level limit for convection - - real(r8) :: tp_fac = unset_r8 ! PMA tunes tpert - - logical :: is_first_step_local = .true. ! AaronDonahue - TODO, actually check if this is the first step given input from the SCREAM-AD - -contains - -subroutine zmconv_readnl()!nlfile) -! AaronDonahue - we do not need/have namelist capabilities yet. That being -! said, we probably still need the constants that are defined here. Switching -! to hard-coded values, no namelist. -! use namelist_utils, only: find_group_name -! use units, only: getunit, freeunit -! use mpishorthand - -! character(len=*), intent(in) :: nlfile ! filepath for file containing namelist input - - ! Local variables -! integer :: unitn, ierr -! character(len=*), parameter :: subname = 'zmconv_readnl' - -! namelist /zmconv_nl/ zmconv_c0_lnd, zmconv_c0_ocn, zmconv_ke, zmconv_tau, & -! zmconv_dmpdz, zmconv_alfa, zmconv_trigmem, zmconv_tiedke_add, & -! zmconv_cape_cin, zmconv_mx_bot_lyr_adj, zmconv_tp_fac, zmconv_trigdcape_ull -! !----------------------------------------------------------------------------- - -! zmconv_tau = 3600._r8 -! if (masterproc) then -! unitn = getunit() -! open( unitn, file=trim(nlfile), status='old' ) -! call find_group_name(unitn, 'zmconv_nl', status=ierr) -! if (ierr == 0) then -! read(unitn, zmconv_nl, iostat=ierr) -! if (ierr /= 0) then -! call endrun(subname // ':: ERROR reading namelist') -! end if -! end if -! close(unitn) -! call freeunit(unitn) - - ! set local variables - c0_lnd = 0.007_r8 ! zmconv_c0_lnd - c0_ocn = 0.007_r8 ! zmconv_c0_ocn - ke = 1.5e-6_r8 ! zmconv_ke - tau = 3600 ! zmconv_tau - trigmem = .false. ! zmconv_trigmem - trigdcape_ull = .false. ! zmconv_trigdcape_ull - tiedke_add = 0.8_r8 ! zmconv_tiedke_add - num_cin = 1 ! zmconv_cape_cin - mx_bot_lyr_adj = 2 ! zmconv_mx_bot_lyr_adj - dmpdz = -0.7e-3_r8 ! zmconv_dmpdz - tp_fac = 0.0_r8 ! zmconv_tp_fac - - if ( zmconv_alfa /= unset_r8 ) then - alfa_scalar = zmconv_alfa - else - alfa_scalar = 0.1_r8 - end if - - if (trigdcape_ull) then - write(*,*)'**** ZMCONV-DCAPE trigger with unrestricted launch level:', trigdcape_ull - endif - -! end if -! -!#ifdef SPMD -! ! Broadcast namelist variables -! call mpibcast(c0_lnd, 1, mpir8, 0, mpicom) -! call mpibcast(c0_ocn, 1, mpir8, 0, mpicom) -! call mpibcast(ke, 1, mpir8, 0, mpicom) -! call mpibcast(tau, 1, mpir8, 0, mpicom) -! call mpibcast(dmpdz, 1, mpir8, 0, mpicom) -! call mpibcast(alfa_scalar, 1, mpir8, 0, mpicom) -! call mpibcast(trigmem, 1, mpilog, 0, mpicom) -! call mpibcast(trigdcape_ull, 1, mpilog, 0, mpicom) -! call mpibcast(tiedke_add, 1, mpir8, 0, mpicom) -! call mpibcast(num_cin, 1, mpiint, 0, mpicom) -! call mpibcast(mx_bot_lyr_adj, 1, mpiint, 0, mpicom) -! call mpibcast(tp_fac, 1, mpir8, 0, mpicom) -!#endif - -end subroutine zmconv_readnl - - -subroutine zm_convi(limcnv_in, no_deep_pbl_in) - - - integer, intent(in) :: limcnv_in ! top interface level limit for convection - logical, intent(in), optional :: no_deep_pbl_in ! no_deep_pbl = .true. eliminates ZM convection entirely within PBL - - ! local variables -! character(len=32) :: hgrid ! horizontal grid specifier ! AaronDonahue - Doesn't appear to be used - - ! Initialization of ZM constants - limcnv = limcnv_in - tfreez = tmelt - eps1 = epsilo - rl = latvap - cpres = cpair - rgrav = 1.0_r8/gravit - rgas = rair - grav = gravit - cp = cpres - - if ( present(no_deep_pbl_in) ) then - no_deep_pbl = no_deep_pbl_in - else - no_deep_pbl = .false. - endif - - ! tau=4800. were used in canadian climate center. however, in echam3 t42, - ! convection is too weak, thus adjusted to 2400. - -! hgrid = get_resolution() ! AaronDonahue - Doesn't appear to be used. - if(trigmem)tau = 3600._r8 - - if ( masterproc ) then - write(*,*) 'tuning parameters zm_convi: tau',tau - write(*,*) 'tuning parameters zm_convi: c0_lnd',c0_lnd, ', c0_ocn', c0_ocn - write(*,*) 'tuning parameters zm_convi: ke',ke - write(*,*) 'tuning parameters zm_convi: dmpdz',dmpdz - write(*,*) 'tuning parameters zm_convi: alfa',alfa_scalar - write(*,*) 'tuning parameters zm_convi: no_deep_pbl',no_deep_pbl - endif - - if (masterproc) write(*,*)'**** ZM: DILUTE Buoyancy Calculation ****' - -end subroutine zm_convi - - - -subroutine zm_convr(lchnk ,ncol , & - t ,qh ,prec ,jctop ,jcbot , & - pblh ,zm ,geos ,zi ,qtnd , & - heat ,pap ,paph ,dpp , & - delt ,mcon ,cme ,cape , & - tpert ,dlf ,pflx ,zdu ,rprd , & - mu ,md ,du ,eu ,ed , & - dp ,dsubcld ,jt ,maxg ,ideep , & - lengath ,ql ,rliq ,landfrac,hu_nm1 , & - cnv_nm1 ,tm1 ,qm1 ,t_star ,q_star, dcape) -!----------------------------------------------------------------------- -! -! Purpose: -! Main driver for zhang-mcfarlane convection scheme -! -! Method: -! performs deep convective adjustment based on mass-flux closure -! algorithm. -! -! Author:guang jun zhang, m.lazare, n.mcfarlane. CAM Contact: P. Rasch -! -! This is contributed code not fully standardized by the CAM core group. -! All variables have been typed, where most are identified in comments -! The current procedure will be reimplemented in a subsequent version -! of the CAM where it will include a more straightforward formulation -! and will make use of the standard CAM nomenclature -! -!----------------------------------------------------------------------- -! use time_manager, only: is_first_step, is_first_restart_step !songxl 2014-05-20 -! -! ************************ index of variables ********************** -! -! wg * alpha array of vertical differencing used (=1. for upstream). -! w * cape convective available potential energy. -! wg * capeg gathered convective available potential energy. -! c * capelmt threshold value for cape for deep convection. -! ic * cpres specific heat at constant pressure in j/kg-degk. -! i * dpp -! ic * delt length of model time-step in seconds. -! wg * dp layer thickness in mbs (between upper/lower interface). -! wg * dqdt mixing ratio tendency at gathered points. -! wg * dsdt dry static energy ("temp") tendency at gathered points. -! wg * dudt u-wind tendency at gathered points. -! wg * dvdt v-wind tendency at gathered points. -! wg * dsubcld layer thickness in mbs between lcl and maxi. -! ic * grav acceleration due to gravity in m/sec2. -! wg * du detrainment in updraft. specified in mid-layer -! wg * ed entrainment in downdraft. -! wg * eu entrainment in updraft. -! wg * hmn moist static energy. -! wg * hsat saturated moist static energy. -! w * ideep holds position of gathered points vs longitude index. -! ic * pver number of model levels. -! wg * j0 detrainment initiation level index. -! wg * jd downdraft initiation level index. -! ic * jlatpr gaussian latitude index for printing grids (if needed). -! wg * jt top level index of deep cumulus convection. -! w * lcl base level index of deep cumulus convection. -! wg * lclg gathered values of lcl. -! w * lel index of highest theoretical convective plume. -! wg * lelg gathered values of lel. -! w * lon index of onset level for deep convection. -! w * maxi index of level with largest moist static energy. -! wg * maxg gathered values of maxi. -! wg * mb cloud base mass flux. -! wg * mc net upward (scaled by mb) cloud mass flux. -! wg * md downward cloud mass flux (positive up). -! wg * mu upward cloud mass flux (positive up). specified -! at interface -! ic * msg number of missing moisture levels at the top of model. -! w * p grid slice of ambient mid-layer pressure in mbs. -! i * pblt row of pbl top indices. -! w * pcpdh scaled surface pressure. -! w * pf grid slice of ambient interface pressure in mbs. -! wg * pg grid slice of gathered values of p. -! w * q grid slice of mixing ratio. -! wg * qd grid slice of mixing ratio in downdraft. -! wg * qg grid slice of gathered values of q. -! i/o * qh grid slice of specific humidity. -! w * qh0 grid slice of initial specific humidity. -! wg * qhat grid slice of upper interface mixing ratio. -! wg * ql grid slice of cloud liquid water. -! wg * qs grid slice of saturation mixing ratio. -! w * qstp grid slice of parcel temp. saturation mixing ratio. -! wg * qstpg grid slice of gathered values of qstp. -! wg * qu grid slice of mixing ratio in updraft. -! ic * rgas dry air gas constant. -! wg * rl latent heat of vaporization. -! w * s grid slice of scaled dry static energy (t+gz/cp). -! wg * sd grid slice of dry static energy in downdraft. -! wg * sg grid slice of gathered values of s. -! wg * shat grid slice of upper interface dry static energy. -! wg * su grid slice of dry static energy in updraft. -! i/o * t -! o * jctop row of top-of-deep-convection indices passed out. -! O * jcbot row of base of cloud indices passed out. -! wg * tg grid slice of gathered values of t. -! w * tl row of parcel temperature at lcl. -! wg * tlg grid slice of gathered values of tl. -! w * tp grid slice of parcel temperatures. -! wg * tpg grid slice of gathered values of tp. -! i/o * u grid slice of u-wind (real). -! wg * ug grid slice of gathered values of u. -! i/o * utg grid slice of u-wind tendency (real). -! i/o * v grid slice of v-wind (real). -! w * va work array re-used by called subroutines. -! wg * vg grid slice of gathered values of v. -! i/o * vtg grid slice of v-wind tendency (real). -! i * w grid slice of diagnosed large-scale vertical velocity. -! w * z grid slice of ambient mid-layer height in metres. -! w * zf grid slice of ambient interface height in metres. -! wg * zfg grid slice of gathered values of zf. -! wg * zg grid slice of gathered values of z. -! -!----------------------------------------------------------------------- -! -! multi-level i/o fields: -! i => input arrays. -! i/o => input/output arrays. -! w => work arrays. -! wg => work arrays operating only on gathered points. -! ic => input data constants. -! c => data constants pertaining to subroutine itself. -! -! input arguments -! - integer, intent(in) :: lchnk ! chunk identifier - integer, intent(in) :: ncol ! number of atmospheric columns - - real(r8), intent(in) :: t(pcols,pver) ! grid slice of temperature at mid-layer. - real(r8), intent(in) :: qh(pcols,pver) ! grid slice of specific humidity. - real(r8), intent(in) :: pap(pcols,pver) - real(r8), intent(in) :: paph(pcols,pver+1) - real(r8), intent(in) :: dpp(pcols,pver) ! local sigma half-level thickness (i.e. dshj). - real(r8), intent(in) :: zm(pcols,pver) - real(r8), intent(in) :: geos(pcols) - real(r8), intent(in) :: zi(pcols,pver+1) - real(r8), intent(in) :: pblh(pcols) - real(r8), intent(in) :: tpert(pcols) - real(r8), intent(in) :: landfrac(pcols) ! RBN Landfrac -!songxl 2014-05-20------------------ - -!DCAPE-ULL - real(r8), intent(in), dimension(pcols,pver) :: t_star ! intermediate T between n and n-1 time step - real(r8), intent(in), dimension(pcols,pver) :: q_star ! intermediate q between n and n-1 time step - - -! -! output arguments -! - real(r8), intent(out) :: qtnd(pcols,pver) ! specific humidity tendency (kg/kg/s) - real(r8), intent(out) :: heat(pcols,pver) ! heating rate (dry static energy tendency, W/kg) - real(r8), intent(out) :: mcon(pcols,pverp) - real(r8), intent(out) :: dlf(pcols,pver) ! scattrd version of the detraining cld h2o tend - real(r8), intent(out) :: pflx(pcols,pverp) ! scattered precip flux at each level - real(r8), intent(out) :: cme(pcols,pver) - real(r8), intent(out) :: cape(pcols) ! w convective available potential energy. - real(r8), intent(out) :: zdu(pcols,pver) - real(r8), intent(out) :: rprd(pcols,pver) ! rain production rate -! move these vars from local storage to output so that convective -! transports can be done in outside of conv_cam. - real(r8), intent(out) :: mu(pcols,pver) - real(r8), intent(out) :: eu(pcols,pver) - real(r8), intent(out) :: du(pcols,pver) - real(r8), intent(out) :: md(pcols,pver) - real(r8), intent(out) :: ed(pcols,pver) - real(r8), intent(out) :: dp(pcols,pver) ! wg layer thickness in mbs (between upper/lower interface). - real(r8), intent(out) :: dsubcld(pcols) ! wg layer thickness in mbs between lcl and maxi. - real(r8), intent(out) :: jctop(pcols) ! o row of top-of-deep-convection indices passed out. - real(r8), intent(out) :: jcbot(pcols) ! o row of base of cloud indices passed out. - real(r8), intent(out) :: prec(pcols) - real(r8), intent(out) :: rliq(pcols) ! reserved liquid (not yet in cldliq) for energy integrals - real(r8), intent(out) :: dcape(pcols) ! output dynamical CAPE - -!songxl 2014-05-20------------------ - - real(r8) zs(pcols) - real(r8) dlg(pcols,pver) ! gathrd version of the detraining cld h2o tend - real(r8) pflxg(pcols,pverp) ! gather precip flux at each level - real(r8) cug(pcols,pver) ! gathered condensation rate - real(r8) evpg(pcols,pver) ! gathered evap rate of rain in downdraft - real(r8) mumax(pcols) - integer jt(pcols) ! wg top level index of deep cumulus convection. - integer maxg(pcols) ! wg gathered values of maxi. - integer ideep(pcols) ! w holds position of gathered points vs longitude index. - integer lengath -! diagnostic field used by chem/wetdep codes - real(r8) ql(pcols,pver) ! wg grid slice of cloud liquid water. -! - real(r8) pblt(pcols) ! i row of pbl top indices. - - - - -! -!----------------------------------------------------------------------- -! -! general work fields (local variables): -! - real(r8) q(pcols,pver) ! w grid slice of mixing ratio. - real(r8) p(pcols,pver) ! w grid slice of ambient mid-layer pressure in mbs. - real(r8) z(pcols,pver) ! w grid slice of ambient mid-layer height in metres. - real(r8) s(pcols,pver) ! w grid slice of scaled dry static energy (t+gz/cp). - real(r8) tp(pcols,pver) ! w grid slice of parcel temperatures. - real(r8) zf(pcols,pver+1) ! w grid slice of ambient interface height in metres. - real(r8) pf(pcols,pver+1) ! w grid slice of ambient interface pressure in mbs. - real(r8) qstp(pcols,pver) ! w grid slice of parcel temp. saturation mixing ratio. - - real(r8) tl(pcols) ! w row of parcel temperature at lcl. -!songxl 2014-05-20----------------- - - logical iclosure ! switch on sequence of call to buoyan_dilute to derive DCAPE - real(r8) capelmt_wk ! work capelmt to allow diff values passed to closure with trigdcape - - integer lcl(pcols) ! w base level index of deep cumulus convection. - integer lel(pcols) ! w index of highest theoretical convective plume. - - integer lon(pcols) ! w index of onset level for deep convection. - integer maxi(pcols) ! w index of level with largest moist static energy. - integer index(pcols) -! -! gathered work fields: -! - real(r8) qg(pcols,pver) ! wg grid slice of gathered values of q. - real(r8) tg(pcols,pver) ! w grid slice of temperature at interface. - real(r8) pg(pcols,pver) ! wg grid slice of gathered values of p. - real(r8) zg(pcols,pver) ! wg grid slice of gathered values of z. - real(r8) sg(pcols,pver) ! wg grid slice of gathered values of s. - real(r8) tpg(pcols,pver) ! wg grid slice of gathered values of tp. - real(r8) zfg(pcols,pver+1) ! wg grid slice of gathered values of zf. - real(r8) qstpg(pcols,pver) ! wg grid slice of gathered values of qstp. - real(r8) ug(pcols,pver) ! wg grid slice of gathered values of u. - real(r8) vg(pcols,pver) ! wg grid slice of gathered values of v. - real(r8) cmeg(pcols,pver) - - real(r8) hu_nm1g(pcols,pver) !songxl 2014-05-20 - - real(r8) rprdg(pcols,pver) ! wg gathered rain production rate - real(r8) capeg(pcols) ! wg gathered convective available potential energy. - real(r8) tlg(pcols) ! wg grid slice of gathered values of tl. - real(r8) landfracg(pcols) ! wg grid slice of landfrac - - integer lclg(pcols) ! wg gathered values of lcl. - integer lelg(pcols) -! -! work fields arising from gathered calculations. -! - real(r8) dqdt(pcols,pver) ! wg mixing ratio tendency at gathered points. - real(r8) dsdt(pcols,pver) ! wg dry static energy ("temp") tendency at gathered points. -! real(r8) alpha(pcols,pver) ! array of vertical differencing used (=1. for upstream). - real(r8) sd(pcols,pver) ! wg grid slice of dry static energy in downdraft. - real(r8) qd(pcols,pver) ! wg grid slice of mixing ratio in downdraft. - real(r8) mc(pcols,pver) ! wg net upward (scaled by mb) cloud mass flux. - real(r8) qhat(pcols,pver) ! wg grid slice of upper interface mixing ratio. - real(r8) qu(pcols,pver) ! wg grid slice of mixing ratio in updraft. - real(r8) su(pcols,pver) ! wg grid slice of dry static energy in updraft. - real(r8) qs(pcols,pver) ! wg grid slice of saturation mixing ratio. - real(r8) shat(pcols,pver) ! wg grid slice of upper interface dry static energy. - real(r8) hmn(pcols,pver) ! wg moist static energy. - real(r8) hsat(pcols,pver) ! wg saturated moist static energy. - real(r8) qlg(pcols,pver) - real(r8) dudt(pcols,pver) ! wg u-wind tendency at gathered points. - real(r8) dvdt(pcols,pver) ! wg v-wind tendency at gathered points. -! real(r8) ud(pcols,pver) -! real(r8) vd(pcols,pver) - - real(r8) mb(pcols) ! wg cloud base mass flux. - - integer jlcl(pcols) - integer j0(pcols) ! wg detrainment initiation level index. - integer jd(pcols) ! wg downdraft initiation level index. - - real(r8) delt ! length of model time-step in seconds. - - integer i - integer ii - integer k - integer msg ! ic number of missing moisture levels at the top of model. - - real(r8) qdifr - real(r8) sdifr - - logical :: is_first_step = .false. - -! -!--------------------------Data statements------------------------------ -! -! -! Set internal variable "msg" (convection limit) to "limcnv-1" -! - msg = limcnv - 1 -! -! initialize necessary arrays. -! zero out variables not used in cam -! - qtnd(:,:) = 0.0_r8 - heat(:,:) = 0.0_r8 - !real(r8), intent(out) :: mcon(pcols,pverp) -! mcon( :, :(pver+1) ) = !Meredith - This is the line that when uncommented causes issues. - rliq(:ncol) = 0.0_r8 - -! -! initialize convective tendencies -! - prec(:ncol) = 0._r8 - do k = 1,pver - do i = 1,ncol - dqdt(i,k) = 0._r8 - dsdt(i,k) = 0._r8 - dudt(i,k) = 0._r8 - dvdt(i,k) = 0._r8 - pflx(i,k) = 0._r8 - pflxg(i,k) = 0._r8 - cme(i,k) = 0._r8 - rprd(i,k) = 0._r8 - zdu(i,k) = 0._r8 - ql(i,k) = 0._r8 - qlg(i,k) = 0._r8 - dlf(i,k) = 0._r8 - dlg(i,k) = 0._r8 - end do - end do - do i = 1,ncol - pflx(i,pverp) = 0 - pflxg(i,pverp) = 0 - mcon(i, pverp) = 0 - end do -! - do i = 1,ncol - pblt(i) = pver - dsubcld(i) = 0._r8 - - jctop(i) = pver - jcbot(i) = 1 - if(trigmem)dcape(i) = 0._r8 !songxl 2014-05-20 - end do - - -! -! calculate local pressure (mbs) and height (m) for both interface -! and mid-layer locations. -! - do i = 1,ncol - zs(i) = geos(i)*rgrav - pf(i,pver+1) = paph(i,pver+1)*0.01_r8 - zf(i,pver+1) = zi(i,pver+1) + zs(i) - end do - do k = 1,pver - do i = 1,ncol - p(i,k) = pap(i,k)*0.01_r8 - pf(i,k) = paph(i,k)*0.01_r8 - z(i,k) = zm(i,k) + zs(i) - zf(i,k) = zi(i,k) + zs(i) - end do - end do -! - do k = pver - 1,msg + 1,-1 - do i = 1,ncol - if (abs(z(i,k)-zs(i)-pblh(i)) < (zf(i,k)-zf(i,k+1))*0.5_r8) pblt(i) = k - end do - end do -! -! store incoming specific humidity field for subsequent calculation -! of precipitation (through change in storage). -! define dry static energy (normalized by cp). -! - do k = 1,pver - do i = 1,ncol - q(i,k) = qh(i,k) - s(i,k) = t(i,k) + (grav/cpres)*z(i,k) - tp(i,k)=0.0_r8 - if(trigmem)tpm1(i,k) = 0.0_r8 !songxl 2014-05-20 - shat(i,k) = s(i,k) - qhat(i,k) = q(i,k) - end do - end do - -!songxl 2014-05-20--------------------- - - do i = 1,ncol - capeg(i) = 0._r8 - lclg(i) = 1 - lelg(i) = pver - maxg(i) = 1 - tlg(i) = 400._r8 - dsubcld(i) = 0._r8 - end do - - - ! Evaluate Tparcel, qs(Tparcel), buoyancy and CAPE, - ! lcl, lel, parcel launch level at index maxi()=hmax - ! - - ! 1. First call, iclosure = .true., standard calculation, scanning for launching level up to 600 hPa - ! 2. Second call, iclosure = .faklse. pass the launch level from 1st call to determine CAPE at previous step - ! The differewnce of CAPE values from the two calls is DCAPE, based on the same launch level - - iclosure = .true. - call buoyan_dilute(lchnk ,ncol , & - q ,t ,p ,z ,pf , & - tp ,qstp ,tl ,rl ,cape , & - pblt ,lcl ,lel ,lon ,maxi , & - rgas ,grav ,cpres ,msg , & - tpert ,iclosure) - - ! if (trigdcape_ull) then - ! if (.not. allocated(dcapemx)) then - ! allocate (dcapemx(pcols), stat=ierror) - ! if ( ierror /= 0 ) call endrun('ZM_CONVR error: allocation error dcapemx') - ! endif - ! dcapemx(:ncol) = maxi(:ncol) - ! endif - - if(trigmem)then - call buoyan_dilute(lchnk ,ncol , & - qm1 ,tm1 ,p ,z ,pf , & - tpm1 ,qstpm1 ,tlm1 ,rl ,capem1 , & - pblt ,lclm1 ,lelm1 ,lonm1 ,maxim1 , & - rgas ,grav ,cpres ,msg , & - tpert ,iclosure) - - do i=1,ncol - dcape(i) = (cape(i)-capem1(i))/(delt*2._r8) - end do - endif - - !DCAPE-ULL - if (.not. is_first_step .and. trigdcape_ull) then - iclosure = .false. - call buoyan_dilute(lchnk ,ncol , & - q_star ,t_star ,p ,z ,pf , & - tpm1 ,qstpm1 ,tlm1 ,rl ,capem1 , & - pblt ,lclm1 ,lelm1 ,lonm1 ,maxim1 , & - rgas ,grav ,cpres ,msg , & - tpert ,iclosure) - - dcape(:ncol) = (cape(:ncol)-capem1(:ncol))/(delt*2._r8) - endif - -! -! determine whether grid points will undergo some deep convection -! (ideep=1) or not (ideep=0), based on values of cape,lcl,lel -! (require cape.gt. 0 and lel capelmt) then - lengath = lengath + 1 - index(lengath) = i - end if - else - if (dcape(i) > dcapelmt) then - lengath = lengath + 1 - index(lengath) = i - end if - end if - else if (trigdcape_ull) then - ! DCAPE-ULL - if (is_first_step) then - !Will this cause restart to be non-BFB - if (cape(i) > capelmt) then - lengath = lengath + 1 - index(lengath) = i - end if - else if (cape(i) > 0.0_r8 .and. dcape(i) > trigdcapelmt) then - ! use constant 0 or a separate threshold for capt because capelmt is for default trigger - lengath = lengath + 1 - index(lengath) = i - endif - else - if (cape(i) > capelmt) then - lengath = lengath + 1 - index(lengath) = i - end if - end if -!>songxl 2014-05-20---------------- - end do - - if (lengath.eq.0) return - do ii=1,lengath - i=index(ii) - ideep(ii)=i - end do -! -! obtain gathered arrays necessary for ensuing calculations. -! - do k = 1,pver - do i = 1,lengath - dp(i,k) = 0.01_r8*dpp(ideep(i),k) - qg(i,k) = q(ideep(i),k) - tg(i,k) = t(ideep(i),k) - pg(i,k) = p(ideep(i),k) - zg(i,k) = z(ideep(i),k) - sg(i,k) = s(ideep(i),k) - tpg(i,k) = tp(ideep(i),k) - zfg(i,k) = zf(ideep(i),k) - qstpg(i,k) = qstp(ideep(i),k) - ug(i,k) = 0._r8 - vg(i,k) = 0._r8 - if(trigmem)hu_nm1g(i,k) = hu_nm1(ideep(i),k) !songxl 2014-05-20 - end do - end do -! - do i = 1,lengath - zfg(i,pver+1) = zf(ideep(i),pver+1) - end do - do i = 1,lengath - capeg(i) = cape(ideep(i)) - lclg(i) = lcl(ideep(i)) - lelg(i) = lel(ideep(i)) - maxg(i) = maxi(ideep(i)) - tlg(i) = tl(ideep(i)) - landfracg(i) = landfrac(ideep(i)) - end do -! -! calculate sub-cloud layer pressure "thickness" for use in -! closure and tendency routines. -! - do k = msg + 1,pver - do i = 1,lengath - if (k >= maxg(i)) then - dsubcld(i) = dsubcld(i) + dp(i,k) - end if - end do - end do -! -! define array of factors (alpha) which defines interfacial -! values, as well as interfacial values for (q,s) used in -! subsequent routines. -! - do k = msg + 2,pver - do i = 1,lengath -! alpha(i,k) = 0.5 - sdifr = 0._r8 - qdifr = 0._r8 - if (sg(i,k) > 0._r8 .or. sg(i,k-1) > 0._r8) & - sdifr = abs((sg(i,k)-sg(i,k-1))/max(sg(i,k-1),sg(i,k))) - if (qg(i,k) > 0._r8 .or. qg(i,k-1) > 0._r8) & - qdifr = abs((qg(i,k)-qg(i,k-1))/max(qg(i,k-1),qg(i,k))) - if (sdifr > 1.E-6_r8) then - shat(i,k) = log(sg(i,k-1)/sg(i,k))*sg(i,k-1)*sg(i,k)/(sg(i,k-1)-sg(i,k)) - else - shat(i,k) = 0.5_r8* (sg(i,k)+sg(i,k-1)) - end if - if (qdifr > 1.E-6_r8) then - qhat(i,k) = log(qg(i,k-1)/qg(i,k))*qg(i,k-1)*qg(i,k)/(qg(i,k-1)-qg(i,k)) - else - qhat(i,k) = 0.5_r8* (qg(i,k)+qg(i,k-1)) - end if - end do - end do -! -! obtain cloud properties. -! - - call cldprp(lchnk , & - qg ,tg ,ug ,vg ,pg , & - zg ,sg ,mu ,eu ,du , & - md ,ed ,sd ,qd ,mc , & - qu ,su ,zfg ,qs ,hmn , & - hsat ,shat ,qlg , & - cmeg ,maxg ,lelg ,jt ,jlcl , & - maxg ,j0 ,jd ,rl ,lengath , & - rgas ,grav ,cpres ,msg , & - pflxg ,evpg ,cug ,rprdg ,limcnv , & - landfracg, hu_nm1g, tpert) !songxl 2014-05-20 - !PMA adds tpert to the calculation -! -! convert detrainment from units of "1/m" to "1/mb". -! - do k = msg + 1,pver - do i = 1,lengath - du (i,k) = du (i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - eu (i,k) = eu (i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - ed (i,k) = ed (i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - cug (i,k) = cug (i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - cmeg (i,k) = cmeg (i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - rprdg(i,k) = rprdg(i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - evpg (i,k) = evpg (i,k)* (zfg(i,k)-zfg(i,k+1))/dp(i,k) - end do - end do - - call closure(lchnk , & - qg ,tg ,pg ,zg ,sg , & - tpg ,qs ,qu ,su ,mc , & - du ,mu ,md ,qd ,sd , & - qhat ,shat ,dp ,qstpg ,zfg , & - qlg ,dsubcld ,mb ,capeg ,tlg , & - lclg ,lelg ,jt ,maxg ,1 , & - lengath ,rgas ,grav ,cpres ,rl , & - msg ,capelmt_wk ) -! -! limit cloud base mass flux to theoretical upper bound. -! - do i=1,lengath - mumax(i) = 0 - end do - do k=msg + 2,pver - do i=1,lengath - mumax(i) = max(mumax(i), mu(i,k)/dp(i,k)) - end do - end do - - do i=1,lengath - if (mumax(i) > 0._r8) then - mb(i) = min(mb(i),0.5_r8/(delt*mumax(i))) - else - mb(i) = 0._r8 - endif - end do - ! If no_deep_pbl = .true., don't allow convection entirely - ! within PBL (suggestion of Bjorn Stevens, 8-2000) - - if (no_deep_pbl) then - do i=1,lengath - if (zm(ideep(i),jt(i)) < pblh(ideep(i))) mb(i) = 0 - end do - end if - - - do k=msg+1,pver - do i=1,lengath - mu (i,k) = mu (i,k)*mb(i) - md (i,k) = md (i,k)*mb(i) - mc (i,k) = mc (i,k)*mb(i) - du (i,k) = du (i,k)*mb(i) - eu (i,k) = eu (i,k)*mb(i) - ed (i,k) = ed (i,k)*mb(i) - cmeg (i,k) = cmeg (i,k)*mb(i) - rprdg(i,k) = rprdg(i,k)*mb(i) - cug (i,k) = cug (i,k)*mb(i) - evpg (i,k) = evpg (i,k)*mb(i) - pflxg(i,k+1)= pflxg(i,k+1)*mb(i)*100._r8/grav - end do - end do -! -! compute temperature and moisture changes due to convection. -! - call q1q2_pjr(lchnk , & - dqdt ,dsdt ,qg ,qs ,qu , & - su ,du ,qhat ,shat ,dp , & - mu ,md ,sd ,qd ,qlg , & - dsubcld ,jt ,maxg ,1 ,lengath , & - cpres ,rl ,msg , & - dlg ,evpg ,cug ) -! -!songxl 2014-05-20----------------- - -! gather back temperature and mixing ratio. -! - do k = msg + 1,pver -!DIR$ CONCURRENT - do i = 1,lengath -! -! q is updated to compute net precip. -! - q(ideep(i),k) = qh(ideep(i),k) + 2._r8*delt*dqdt(i,k) - qtnd(ideep(i),k) = dqdt (i,k) - cme (ideep(i),k) = cmeg (i,k) - rprd(ideep(i),k) = rprdg(i,k) - zdu (ideep(i),k) = du (i,k) - mcon(ideep(i),k) = mc (i,k) - heat(ideep(i),k) = dsdt (i,k)*cpres - dlf (ideep(i),k) = dlg (i,k) - pflx(ideep(i),k) = pflxg(i,k) - ql (ideep(i),k) = qlg (i,k) -!songxl 2014-05-20 - end do - end do -! -!DIR$ CONCURRENT - do i = 1,lengath - jctop(ideep(i)) = jt(i) -!++bee - jcbot(ideep(i)) = maxg(i) -!--bee - pflx(ideep(i),pverp) = pflxg(i,pverp) - end do - -! Compute precip by integrating change in water vapor minus detrained cloud water - do k = pver,msg + 1,-1 - do i = 1,ncol - prec(i) = prec(i) - dpp(i,k)* (q(i,k)-qh(i,k)) - dpp(i,k)*dlf(i,k)*2*delt - end do - end do - -! obtain final precipitation rate in m/s. - do i = 1,ncol - prec(i) = rgrav*max(prec(i),0._r8)/ (2._r8*delt)/1000._r8 - end do - -! Compute reserved liquid (not yet in cldliq) for energy integrals. -! Treat rliq as flux out bottom, to be added back later. - do k = 1, pver - do i = 1, ncol - rliq(i) = rliq(i) + dlf(i,k)*dpp(i,k)/gravit - end do - end do - rliq(:ncol) = rliq(:ncol) /1000._r8 - - is_first_step_local = .false. - return -end subroutine zm_convr - -!=============================================================================== -subroutine zm_conv_evap(ncol,lchnk, & - t,pmid,pdel,q, & - tend_s, tend_s_snwprd, tend_s_snwevmlt, tend_q, & - prdprec, cldfrc, deltat, & - prec, snow, ntprprd, ntsnprd, flxprec, flxsnow ) - -!----------------------------------------------------------------------- -! Compute tendencies due to evaporation of rain from ZM scheme -!-- -! Compute the total precipitation and snow fluxes at the surface. -! Add in the latent heat of fusion for snow formation and melt, since it not dealt with -! in the Zhang-MacFarlane parameterization. -! Evaporate some of the precip directly into the environment using a Sundqvist type algorithm -!----------------------------------------------------------------------- - -! use wv_saturation, only: qsat - !use phys_grid, only: get_rlat_all_p - -!------------------------------Arguments-------------------------------- - integer,intent(in) :: ncol, lchnk ! number of columns and chunk index - real(r8),intent(in), dimension(pcols,pver) :: t ! temperature (K) - real(r8),intent(in), dimension(pcols,pver) :: pmid ! midpoint pressure (Pa) - real(r8),intent(in), dimension(pcols,pver) :: pdel ! layer thickness (Pa) - real(r8),intent(in), dimension(pcols,pver) :: q ! water vapor (kg/kg) - real(r8),intent(inout), dimension(pcols,pver) :: tend_s ! heating rate (J/kg/s) - real(r8),intent(inout), dimension(pcols,pver) :: tend_q ! water vapor tendency (kg/kg/s) - real(r8),intent(out ), dimension(pcols,pver) :: tend_s_snwprd ! Heating rate of snow production - real(r8),intent(out ), dimension(pcols,pver) :: tend_s_snwevmlt ! Heating rate of evap/melting of snow - - - - real(r8), intent(in ) :: prdprec(pcols,pver)! precipitation production (kg/ks/s) - real(r8), intent(in ) :: cldfrc(pcols,pver) ! cloud fraction - real(r8), intent(in ) :: deltat ! time step - - real(r8), intent(inout) :: prec(pcols) ! Convective-scale preciptn rate - real(r8), intent(out) :: snow(pcols) ! Convective-scale snowfall rate -! -!---------------------------Local storage------------------------------- - - real(r8) :: es (pcols,pver) ! Saturation vapor pressure - real(r8) :: fice (pcols,pver) ! ice fraction in precip production - real(r8) :: fsnow_conv(pcols,pver) ! snow fraction in precip production - real(r8) :: qs (pcols,pver) ! saturation specific humidity - real(r8),intent(out) :: flxprec(pcols,pverp) ! Convective-scale flux of precip at interfaces (kg/m2/s) - real(r8),intent(out) :: flxsnow(pcols,pverp) ! Convective-scale flux of snow at interfaces (kg/m2/s) - real(r8),intent(out) :: ntprprd(pcols,pver) ! net precip production in layer - real(r8),intent(out) :: ntsnprd(pcols,pver) ! net snow production in layer - real(r8) :: work1 ! temp variable (pjr) - real(r8) :: work2 ! temp variable (pjr) - - real(r8) :: evpvint(pcols) ! vertical integral of evaporation - real(r8) :: evpprec(pcols) ! evaporation of precipitation (kg/kg/s) - real(r8) :: evpsnow(pcols) ! evaporation of snowfall (kg/kg/s) - real(r8) :: snowmlt(pcols) ! snow melt tendency in layer - real(r8) :: flxsntm(pcols) ! flux of snow into layer, after melting - - real(r8) :: evplimit ! temp variable for evaporation limits - - integer :: i,k ! longitude,level indices - - -!----------------------------------------------------------------------- - -! convert input precip to kg/m2/s - prec(:ncol) = prec(:ncol)*1000._r8 - -! determine saturation vapor pressure - do i = 1,ncol - do k = 1,pver - call qsat(t(i,k), pmid(i,k), & - es(i,k), qs(i,k), 0) - end do - end do - -! determine ice fraction in rain production (use cloud water parameterization fraction at present) - call cldfrc_fice(ncol, t, fice, fsnow_conv) - -! zero the flux integrals on the top boundary - flxprec(:ncol,1) = 0._r8 - flxsnow(:ncol,1) = 0._r8 - evpvint(:ncol) = 0._r8 - - do k = 1, pver - do i = 1, ncol - -! Melt snow falling into layer, if necessary. - if (t(i,k) > tmelt) then - flxsntm(i) = 0._r8 - snowmlt(i) = flxsnow(i,k) * gravit/ pdel(i,k) - else - flxsntm(i) = flxsnow(i,k) - snowmlt(i) = 0._r8 - end if - -! relative humidity depression must be > 0 for evaporation - evplimit = max(1._r8 - q(i,k)/qs(i,k), 0._r8) - -! total evaporation depends on flux in the top of the layer -! flux prec is the net production above layer minus evaporation into environmet - evpprec(i) = ke * (1._r8 - cldfrc(i,k)) * evplimit * sqrt(flxprec(i,k)) -!********************************************************** -!! evpprec(i) = 0. ! turn off evaporation for now -!********************************************************** - -! Don't let evaporation supersaturate layer (approx). Layer may already be saturated. -! Currently does not include heating/cooling change to qs - evplimit = max(0._r8, (qs(i,k)-q(i,k)) / deltat) - -! Don't evaporate more than is falling into the layer - do not evaporate rain formed -! in this layer but if precip production is negative, remove from the available precip -! Negative precip production occurs because of evaporation in downdrafts. -!!$ evplimit = flxprec(i,k) * gravit / pdel(i,k) + min(prdprec(i,k), 0.) - evplimit = min(evplimit, flxprec(i,k) * gravit / pdel(i,k)) - -! Total evaporation cannot exceed input precipitation - evplimit = min(evplimit, (prec(i) - evpvint(i)) * gravit / pdel(i,k)) - - evpprec(i) = min(evplimit, evpprec(i)) - -! evaporation of snow depends on snow fraction of total precipitation in the top after melting - if (flxprec(i,k) > 0._r8) then -! evpsnow(i) = evpprec(i) * flxsntm(i) / flxprec(i,k) -! prevent roundoff problems - work1 = min(max(0._r8,flxsntm(i)/flxprec(i,k)),1._r8) - evpsnow(i) = evpprec(i) * work1 - else - evpsnow(i) = 0._r8 - end if - -! vertically integrated evaporation - evpvint(i) = evpvint(i) + evpprec(i) * pdel(i,k)/gravit - -! net precip production is production - evaporation - ntprprd(i,k) = prdprec(i,k) - evpprec(i) -! net snow production is precip production * ice fraction - evaporation - melting -!pjrworks ntsnprd(i,k) = prdprec(i,k)*fice(i,k) - evpsnow(i) - snowmlt(i) -!pjrwrks2 ntsnprd(i,k) = prdprec(i,k)*fsnow_conv(i,k) - evpsnow(i) - snowmlt(i) -! the small amount added to flxprec in the work1 expression has been increased from -! 1e-36 to 8.64e-11 (1e-5 mm/day). This causes the temperature based partitioning -! scheme to be used for small flxprec amounts. This is to address error growth problems. -#ifdef PERGRO - work1 = min(max(0._r8,flxsnow(i,k)/(flxprec(i,k)+8.64e-11_r8)),1._r8) -#else - if (flxprec(i,k).gt.0._r8) then - work1 = min(max(0._r8,flxsnow(i,k)/flxprec(i,k)),1._r8) - else - work1 = 0._r8 - endif -#endif - work2 = max(fsnow_conv(i,k), work1) - if (snowmlt(i).gt.0._r8) work2 = 0._r8 -! work2 = fsnow_conv(i,k) - ntsnprd(i,k) = prdprec(i,k)*work2 - evpsnow(i) - snowmlt(i) - tend_s_snwprd (i,k) = prdprec(i,k)*work2*latice - tend_s_snwevmlt(i,k) = - ( evpsnow(i) + snowmlt(i) )*latice - -! precipitation fluxes - flxprec(i,k+1) = flxprec(i,k) + ntprprd(i,k) * pdel(i,k)/gravit - flxsnow(i,k+1) = flxsnow(i,k) + ntsnprd(i,k) * pdel(i,k)/gravit - -! protect against rounding error - flxprec(i,k+1) = max(flxprec(i,k+1), 0._r8) - flxsnow(i,k+1) = max(flxsnow(i,k+1), 0._r8) -! more protection (pjr) -! flxsnow(i,k+1) = min(flxsnow(i,k+1), flxprec(i,k+1)) - -! heating (cooling) and moistening due to evaporation -! - latent heat of vaporization for precip production has already been accounted for -! - snow is contained in prec - tend_s(i,k) =-evpprec(i)*latvap + ntsnprd(i,k)*latice - tend_q(i,k) = evpprec(i) - end do - end do - -! set output precipitation rates (m/s) - prec(:ncol) = flxprec(:ncol,pver+1) / 1000._r8 - snow(:ncol) = flxsnow(:ncol,pver+1) / 1000._r8 - -!********************************************************** -!!$ tend_s(:ncol,:) = 0. ! turn heating off -!********************************************************** - - end subroutine zm_conv_evap - - -! AaronDonahue - skip convtran and momtran for now -subroutine convtran(lchnk , & - doconvtran,q ,ncnst ,mu ,md , & - du ,eu ,ed ,dp ,dsubcld , & - jt ,mx ,ideep ,il1g ,il2g , & - nstep ,fracis ,dqdt ,dpdry ) -!----------------------------------------------------------------------- -! -! Purpose: -! Convective transport of trace species -! -! Mixing ratios may be with respect to either dry or moist air -! -! Method: -! -! -! -! Author: P. Rasch -! -!----------------------------------------------------------------------- -! use constituents, only: cnst_get_type_byind -! use ppgrid - use scream_abortutils, only: endscreamrun - - implicit none -!----------------------------------------------------------------------- -! -! Input arguments -! - integer, intent(in) :: lchnk ! chunk identifier - integer, intent(in) :: ncnst ! number of tracers to transport - logical, intent(in) :: doconvtran(ncnst) ! flag for doing convective transport - real(r8), intent(in) :: q(pcols,pver,ncnst) ! Tracer array including moisture - real(r8), intent(in) :: mu(pcols,pver) ! Mass flux up - real(r8), intent(in) :: md(pcols,pver) ! Mass flux down - real(r8), intent(in) :: du(pcols,pver) ! Mass detraining from updraft - real(r8), intent(in) :: eu(pcols,pver) ! Mass entraining from updraft - real(r8), intent(in) :: ed(pcols,pver) ! Mass entraining from downdraft - real(r8), intent(in) :: dp(pcols,pver) ! Delta pressure between interfaces - real(r8), intent(in) :: dsubcld(pcols) ! Delta pressure from cloud base to sfc - real(r8), intent(in) :: fracis(pcols,pver,ncnst) ! fraction of tracer that is insoluble - - integer, intent(in) :: jt(pcols) ! Index of cloud top for each column - integer, intent(in) :: mx(pcols) ! Index of cloud top for each column - integer, intent(in) :: ideep(pcols) ! Gathering array - integer, intent(in) :: il1g ! Gathered min lon indices over which to operate - integer, intent(in) :: il2g ! Gathered max lon indices over which to operate - integer, intent(in) :: nstep ! Time step index - - real(r8), intent(in) :: dpdry(pcols,pver) ! Delta pressure between interfaces - - -! input/output - - real(r8), intent(out) :: dqdt(pcols,pver,ncnst) ! Tracer tendency array - -!--------------------------Local Variables------------------------------ - - integer i ! Work index - integer k ! Work index - integer kbm ! Highest altitude index of cloud base - integer kk ! Work index - integer kkp1 ! Work index - integer km1 ! Work index - integer kp1 ! Work index - integer ktm ! Highest altitude index of cloud top - integer m ! Work index - - real(r8) cabv ! Mix ratio of constituent above - real(r8) cbel ! Mix ratio of constituent below - real(r8) cdifr ! Normalized diff between cabv and cbel - real(r8) chat(pcols,pver) ! Mix ratio in env at interfaces - real(r8) cond(pcols,pver) ! Mix ratio in downdraft at interfaces - real(r8) const(pcols,pver) ! Gathered tracer array - real(r8) fisg(pcols,pver) ! gathered insoluble fraction of tracer - real(r8) conu(pcols,pver) ! Mix ratio in updraft at interfaces - real(r8) dcondt(pcols,pver) ! Gathered tend array - real(r8) small ! A small number - real(r8) mbsth ! Threshold for mass fluxes - real(r8) mupdudp ! A work variable - real(r8) minc ! A work variable - real(r8) maxc ! A work variable - real(r8) fluxin ! A work variable - real(r8) fluxout ! A work variable - real(r8) netflux ! A work variable - - real(r8) dutmp(pcols,pver) ! Mass detraining from updraft - real(r8) eutmp(pcols,pver) ! Mass entraining from updraft - real(r8) edtmp(pcols,pver) ! Mass entraining from downdraft - real(r8) dptmp(pcols,pver) ! Delta pressure between interfaces -!----------------------------------------------------------------------- -! - small = 1.e-36_r8 -! mbsth is the threshold below which we treat the mass fluxes as zero (in mb/s) - mbsth = 1.e-15_r8 - -! Find the highest level top and bottom levels of convection - ktm = pver - kbm = pver - do i = il1g, il2g - ktm = min(ktm,jt(i)) - kbm = min(kbm,mx(i)) - end do - -! Loop ever each constituent - do m = 2, ncnst - if (doconvtran(m)) then - -! TODO: Meredith Fossitt, find a way to sub dependency for -! cnst_get_type_blind -! if (cnst_get_type_byind(m).eq.'dry') then - if (.true.) then - do k = 1,pver - do i =il1g,il2g - dptmp(i,k) = dpdry(i,k) - dutmp(i,k) = du(i,k)*dp(i,k)/dpdry(i,k) - eutmp(i,k) = eu(i,k)*dp(i,k)/dpdry(i,k) - edtmp(i,k) = ed(i,k)*dp(i,k)/dpdry(i,k) - end do - end do - else - do k = 1,pver - do i =il1g,il2g - dptmp(i,k) = dp(i,k) - dutmp(i,k) = du(i,k) - eutmp(i,k) = eu(i,k) - edtmp(i,k) = ed(i,k) - end do - end do - endif -! dptmp = dp - -! Gather up the constituent and set tend to zero - do k = 1,pver - do i =il1g,il2g - const(i,k) = q(ideep(i),k,m) - fisg(i,k) = fracis(ideep(i),k,m) - end do - end do - -! From now on work only with gathered data - -! Interpolate environment tracer values to interfaces - do k = 1,pver - km1 = max(1,k-1) - do i = il1g, il2g - minc = min(const(i,km1),const(i,k)) - maxc = max(const(i,km1),const(i,k)) - if (minc < 0) then - cdifr = 0._r8 - else - cdifr = abs(const(i,k)-const(i,km1))/max(maxc,small) - endif - -! If the two layers differ significantly use a geometric averaging -! procedure - if (cdifr > 1.E-6_r8) then - cabv = max(const(i,km1),maxc*1.e-12_r8) - cbel = max(const(i,k),maxc*1.e-12_r8) - chat(i,k) = log(cabv/cbel)/(cabv-cbel)*cabv*cbel - - else ! Small diff, so just arithmetic mean - chat(i,k) = 0.5_r8* (const(i,k)+const(i,km1)) - end if - -! Provisional up and down draft values - conu(i,k) = chat(i,k) - cond(i,k) = chat(i,k) - -! provisional tends - dcondt(i,k) = 0._r8 - - end do - end do - -! Do levels adjacent to top and bottom - k = 2 - km1 = 1 - kk = pver - do i = il1g,il2g - mupdudp = mu(i,kk) + dutmp(i,kk)*dptmp(i,kk) - if (mupdudp > mbsth) then - conu(i,kk) = (+eutmp(i,kk)*fisg(i,kk)*const(i,kk)*dptmp(i,kk))/mupdudp - endif - if (md(i,k) < -mbsth) then - cond(i,k) = (-edtmp(i,km1)*fisg(i,km1)*const(i,km1)*dptmp(i,km1))/md(i,k) - endif - end do - -! Updraft from bottom to top - do kk = pver-1,1,-1 - kkp1 = min(pver,kk+1) - do i = il1g,il2g - mupdudp = mu(i,kk) + dutmp(i,kk)*dptmp(i,kk) - if (mupdudp > mbsth) then - conu(i,kk) = ( mu(i,kkp1)*conu(i,kkp1)+eutmp(i,kk)*fisg(i,kk)* & - const(i,kk)*dptmp(i,kk) )/mupdudp - endif - end do - end do - -! Downdraft from top to bottom - do k = 3,pver - km1 = max(1,k-1) - do i = il1g,il2g - if (md(i,k) < -mbsth) then - cond(i,k) = ( md(i,km1)*cond(i,km1)-edtmp(i,km1)*fisg(i,km1)*const(i,km1) & - *dptmp(i,km1) )/md(i,k) - endif - end do - end do - - - do k = ktm,pver - km1 = max(1,k-1) - kp1 = min(pver,k+1) - do i = il1g,il2g - -! version 1 hard to check for roundoff errors -! dcondt(i,k) = -! $ +(+mu(i,kp1)* (conu(i,kp1)-chat(i,kp1)) -! $ -mu(i,k)* (conu(i,k)-chat(i,k)) -! $ +md(i,kp1)* (cond(i,kp1)-chat(i,kp1)) -! $ -md(i,k)* (cond(i,k)-chat(i,k)) -! $ )/dp(i,k) - -! version 2 hard to limit fluxes -! fluxin = mu(i,kp1)*conu(i,kp1) + mu(i,k)*chat(i,k) -! $ -(md(i,k) *cond(i,k) + md(i,kp1)*chat(i,kp1)) -! fluxout = mu(i,k)*conu(i,k) + mu(i,kp1)*chat(i,kp1) -! $ -(md(i,kp1)*cond(i,kp1) + md(i,k)*chat(i,k)) - -! version 3 limit fluxes outside convection to mass in appropriate layer -! these limiters are probably only safe for positive definite quantitities -! it assumes that mu and md already satify a courant number limit of 1 - fluxin = mu(i,kp1)*conu(i,kp1)+ mu(i,k)*min(chat(i,k),const(i,km1)) & - -(md(i,k) *cond(i,k) + md(i,kp1)*min(chat(i,kp1),const(i,kp1))) - fluxout = mu(i,k)*conu(i,k) + mu(i,kp1)*min(chat(i,kp1),const(i,k)) & - -(md(i,kp1)*cond(i,kp1) + md(i,k)*min(chat(i,k),const(i,k))) - - netflux = fluxin - fluxout - if (abs(netflux) < max(fluxin,fluxout)*1.e-12_r8) then - netflux = 0._r8 - endif - dcondt(i,k) = netflux/dptmp(i,k) - end do - end do -! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -! -!DIR$ NOINTERCHANGE - do k = kbm,pver - km1 = max(1,k-1) - do i = il1g,il2g - if (k == mx(i)) then - -! version 1 -! dcondt(i,k) = (1./dsubcld(i))* -! $ (-mu(i,k)*(conu(i,k)-chat(i,k)) -! $ -md(i,k)*(cond(i,k)-chat(i,k)) -! $ ) - -! version 2 -! fluxin = mu(i,k)*chat(i,k) - md(i,k)*cond(i,k) -! fluxout = mu(i,k)*conu(i,k) - md(i,k)*chat(i,k) -! version 3 - fluxin = mu(i,k)*min(chat(i,k),const(i,km1)) - md(i,k)*cond(i,k) - fluxout = mu(i,k)*conu(i,k) - md(i,k)*min(chat(i,k),const(i,k)) - - netflux = fluxin - fluxout - if (abs(netflux) < max(fluxin,fluxout)*1.e-12_r8) then - netflux = 0._r8 - endif -! dcondt(i,k) = netflux/dsubcld(i) - dcondt(i,k) = netflux/dptmp(i,k) - else if (k > mx(i)) then -! dcondt(i,k) = dcondt(i,k-1) - dcondt(i,k) = 0._r8 - end if - end do - end do - -! Initialize to zero everywhere, then scatter tendency back to full array - dqdt(:,:,m) = 0._r8 - do k = 1,pver - kp1 = min(pver,k+1) -!DIR$ CONCURRENT - do i = il1g,il2g - dqdt(ideep(i),k,m) = dcondt(i,k) - end do - end do - - end if ! for doconvtran - - end do - - return -end subroutine convtran - -!========================================================================================= - -subroutine momtran(lchnk, ncol, & - domomtran,q ,ncnst ,mu ,md , & - du ,eu ,ed ,dp ,dsubcld , & - jt ,mx ,ideep ,il1g ,il2g , & - nstep ,dqdt ,pguall ,pgdall, icwu, icwd, dt, seten ) -!----------------------------------------------------------------------- -! -! Purpose: -! Convective transport of momentum -! -! Mixing ratios may be with respect to either dry or moist air -! -! Method: -! Based on the convtran subroutine by P. Rasch -! -! -! Author: J. Richter and P. Rasch -! -!----------------------------------------------------------------------- - use physics_utils, only: r8 => rtype, rtype8, itype, btype -! use constituents, only: cnst_get_type_byind -! use ppgrid - use scream_abortutils, only: endscreamrun - implicit none -!----------------------------------------------------------------------- -! -! Input arguments -! - integer, intent(in) :: lchnk ! chunk identifier - integer, intent(in) :: ncol ! number of atmospheric columns - integer, intent(in) :: ncnst ! number of tracers to transport - logical, intent(in) :: domomtran(ncnst) ! flag for doing convective transport - real(r8), intent(in) :: q(pcols,pver,ncnst) ! Wind array - real(r8), intent(in) :: mu(pcols,pver) ! Mass flux up - real(r8), intent(in) :: md(pcols,pver) ! Mass flux down - real(r8), intent(in) :: du(pcols,pver) ! Mass detraining from updraft - real(r8), intent(in) :: eu(pcols,pver) ! Mass entraining from updraft - real(r8), intent(in) :: ed(pcols,pver) ! Mass entraining from downdraft - real(r8), intent(in) :: dp(pcols,pver) ! Delta pressure between interfaces - real(r8), intent(in) :: dsubcld(pcols) ! Delta pressure from cloud base to sfc - real(r8), intent(in) :: dt ! time step in seconds : 2*delta_t - - integer, intent(in) :: jt(pcols) ! Index of cloud top for each column - integer, intent(in) :: mx(pcols) ! Index of cloud top for each column - integer, intent(in) :: ideep(pcols) ! Gathering array - integer, intent(in) :: il1g ! Gathered min lon indices over which to operate - integer, intent(in) :: il2g ! Gathered max lon indices over which to operate - integer, intent(in) :: nstep ! Time step index - - - -! input/output - - real(r8), intent(out) :: dqdt(pcols,pver,ncnst) ! Tracer tendency array - -!--------------------------Local Variables------------------------------ - - integer i ! Work index - integer k ! Work index - integer kbm ! Highest altitude index of cloud base - integer kk ! Work index - integer kkp1 ! Work index - integer kkm1 ! Work index - integer km1 ! Work index - integer kp1 ! Work index - integer ktm ! Highest altitude index of cloud top - integer m ! Work index - integer ii ! Work index - - real(r8) chat(pcols,pver) ! Mix ratio in env at interfaces - real(r8) cond(pcols,pver) ! Mix ratio in downdraft at interfaces - real(r8) const(pcols,pver) ! Gathered wind array - real(r8) conu(pcols,pver) ! Mix ratio in updraft at interfaces - real(r8) dcondt(pcols,pver) ! Gathered tend array - real(r8) small ! A small number - real(r8) mbsth ! Threshold for mass fluxes - real(r8) mupdudp ! A work variable - - real(r8) momcu ! constant for updraft pressure gradient term - real(r8) momcd ! constant for downdraft pressure gradient term - real(r8) sum ! sum - real(r8) sum2 ! sum2 - - real(r8) mududp(pcols,pver) ! working variable - real(r8) mddudp(pcols,pver) ! working variable - - real(r8) pgu(pcols,pver) ! Pressure gradient term for updraft - real(r8) pgd(pcols,pver) ! Pressure gradient term for downdraft - - real(r8),intent(out) :: pguall(pcols,pver,ncnst) ! Apparent force from updraft PG - real(r8),intent(out) :: pgdall(pcols,pver,ncnst) ! Apparent force from downdraft PG - - real(r8),intent(out) :: icwu(pcols,pver,ncnst) ! In-cloud winds in updraft - real(r8),intent(out) :: icwd(pcols,pver,ncnst) ! In-cloud winds in downdraft - - real(r8),intent(out) :: seten(pcols,pver) ! Dry static energy tendency - real(r8) gseten(pcols,pver) ! Gathered dry static energy tendency - - real(r8) mflux(pcols,pverp,ncnst) ! Gathered momentum flux - - real(r8) wind0(pcols,pver,ncnst) ! gathered wind before time step - real(r8) windf(pcols,pver,ncnst) ! gathered wind after time step - real(r8) fkeb, fket, ketend_cons, ketend, utop, ubot, vtop, vbot, gset2 - - -!----------------------------------------------------------------------- -! - -! Initialize outgoing fields - pguall(:,:,:) = 0.0_r8 - pgdall(:,:,:) = 0.0_r8 -! Initialize in-cloud winds to environmental wind - icwu(:ncol,:,:) = q(:ncol,:,:) - icwd(:ncol,:,:) = q(:ncol,:,:) - -! Initialize momentum flux and final winds - mflux(:,:,:) = 0.0_r8 - wind0(:,:,:) = 0.0_r8 - windf(:,:,:) = 0.0_r8 - -! Initialize dry static energy - - seten(:,:) = 0.0_r8 - gseten(:,:) = 0.0_r8 - -! Define constants for parameterization - - momcu = 0.4_r8 - momcd = 0.4_r8 - - small = 1.e-36_r8 -! mbsth is the threshold below which we treat the mass fluxes as zero (in mb/s) - mbsth = 1.e-15_r8 - -! Find the highest level top and bottom levels of convection - ktm = pver - kbm = pver - do i = il1g, il2g - ktm = min(ktm,jt(i)) - kbm = min(kbm,mx(i)) - end do - -! Loop ever each wind component - do m = 1, ncnst !start at m = 1 to transport momentum - if (domomtran(m)) then - -! Gather up the winds and set tend to zero - do k = 1,pver - do i =il1g,il2g - const(i,k) = q(ideep(i),k,m) - wind0(i,k,m) = const(i,k) - end do - end do - - -! From now on work only with gathered data - -! Interpolate winds to interfaces - - do k = 1,pver - km1 = max(1,k-1) - do i = il1g, il2g - - ! use arithmetic mean - chat(i,k) = 0.5_r8* (const(i,k)+const(i,km1)) - -! Provisional up and down draft values - conu(i,k) = chat(i,k) - cond(i,k) = chat(i,k) - -! provisional tends - dcondt(i,k) = 0._r8 - - end do - end do - - -! -! Pressure Perturbation Term -! - - !Top boundary: assume mu is zero - - k=1 - pgu(:il2g,k) = 0.0_r8 - pgd(:il2g,k) = 0.0_r8 - - do k=2,pver-1 - km1 = max(1,k-1) - kp1 = min(pver,k+1) - do i = il1g,il2g - - !interior points - - mududp(i,k) = ( mu(i,k) * (const(i,k)- const(i,km1))/dp(i,km1) & - + mu(i,kp1) * (const(i,kp1) - const(i,k))/dp(i,k)) - - pgu(i,k) = - momcu * 0.5_r8 * mududp(i,k) - - - mddudp(i,k) = ( md(i,k) * (const(i,k)- const(i,km1))/dp(i,km1) & - + md(i,kp1) * (const(i,kp1) - const(i,k))/dp(i,k)) - - pgd(i,k) = - momcd * 0.5_r8 * mddudp(i,k) - - - end do - end do - - ! bottom boundary - k = pver - km1 = max(1,k-1) - do i=il1g,il2g - - mududp(i,k) = mu(i,k) * (const(i,k)- const(i,km1))/dp(i,km1) - pgu(i,k) = - momcu * mududp(i,k) - - mddudp(i,k) = md(i,k) * (const(i,k)- const(i,km1))/dp(i,km1) - - pgd(i,k) = - momcd * mddudp(i,k) - - end do - - -! -! In-cloud velocity calculations -! - -! Do levels adjacent to top and bottom - k = 2 - km1 = 1 - kk = pver - kkm1 = max(1,kk-1) - do i = il1g,il2g - mupdudp = mu(i,kk) + du(i,kk)*dp(i,kk) - if (mupdudp > mbsth) then - - conu(i,kk) = (+eu(i,kk)*const(i,kk)*dp(i,kk)+pgu(i,kk)*dp(i,kk))/mupdudp - endif - if (md(i,k) < -mbsth) then - cond(i,k) = (-ed(i,km1)*const(i,km1)*dp(i,km1))-pgd(i,km1)*dp(i,km1)/md(i,k) - endif - - - end do - - - -! Updraft from bottom to top - do kk = pver-1,1,-1 - kkm1 = max(1,kk-1) - kkp1 = min(pver,kk+1) - do i = il1g,il2g - mupdudp = mu(i,kk) + du(i,kk)*dp(i,kk) - if (mupdudp > mbsth) then - - conu(i,kk) = ( mu(i,kkp1)*conu(i,kkp1)+eu(i,kk)* & - const(i,kk)*dp(i,kk)+pgu(i,kk)*dp(i,kk))/mupdudp - endif - end do - - end do - - -! Downdraft from top to bottom - do k = 3,pver - km1 = max(1,k-1) - do i = il1g,il2g - if (md(i,k) < -mbsth) then - - cond(i,k) = ( md(i,km1)*cond(i,km1)-ed(i,km1)*const(i,km1) & - *dp(i,km1)-pgd(i,km1)*dp(i,km1) )/md(i,k) - - endif - end do - end do - - - sum = 0._r8 - sum2 = 0._r8 - - - do k = ktm,pver - km1 = max(1,k-1) - kp1 = min(pver,k+1) - do i = il1g,il2g - ii = ideep(i) - -! version 1 hard to check for roundoff errors - dcondt(i,k) = & - +(mu(i,kp1)* (conu(i,kp1)-chat(i,kp1)) & - -mu(i,k)* (conu(i,k)-chat(i,k)) & - +md(i,kp1)* (cond(i,kp1)-chat(i,kp1)) & - -md(i,k)* (cond(i,k)-chat(i,k)) & - )/dp(i,k) - - end do - end do - - ! dcont for bottom layer - ! - !DIR$ NOINTERCHANGE - do k = kbm,pver - km1 = max(1,k-1) - do i = il1g,il2g - if (k == mx(i)) then - - ! version 1 - dcondt(i,k) = (1._r8/dp(i,k))* & - (-mu(i,k)*(conu(i,k)-chat(i,k)) & - -md(i,k)*(cond(i,k)-chat(i,k)) & - ) - end if - end do - end do - -! Initialize to zero everywhere, then scatter tendency back to full array - dqdt(:,:,m) = 0._r8 - - do k = 1,pver - do i = il1g,il2g - ii = ideep(i) - dqdt(ii,k,m) = dcondt(i,k) - ! Output apparent force on the mean flow from pressure gradient - pguall(ii,k,m) = -pgu(i,k) - pgdall(ii,k,m) = -pgd(i,k) - icwu(ii,k,m) = conu(i,k) - icwd(ii,k,m) = cond(i,k) - end do - end do - - ! Calculate momentum flux in units of mb*m/s2 - - do k = ktm,pver - do i = il1g,il2g - ii = ideep(i) - mflux(i,k,m) = & - -mu(i,k)* (conu(i,k)-chat(i,k)) & - -md(i,k)* (cond(i,k)-chat(i,k)) - end do - end do - - - ! Calculate winds at the end of the time step - - do k = ktm,pver - do i = il1g,il2g - ii = ideep(i) - km1 = max(1,k-1) - kp1 = k+1 - windf(i,k,m) = const(i,k) - (mflux(i,kp1,m) - mflux(i,k,m)) * dt /dp(i,k) - - end do - end do - - end if ! for domomtran - end do - - ! Need to add an energy fix to account for the dissipation of kinetic energy - ! Formulation follows from Boville and Bretherton (2003) - ! formulation by PJR - - do k = ktm,pver - km1 = max(1,k-1) - kp1 = min(pver,k+1) - do i = il1g,il2g - - ii = ideep(i) - - ! calculate the KE fluxes at top and bot of layer - ! based on a discrete approximation to b&b eq(35) F_KE = u*F_u + v*F_v at interface - utop = (wind0(i,k,1)+wind0(i,km1,1))/2._r8 - vtop = (wind0(i,k,2)+wind0(i,km1,2))/2._r8 - ubot = (wind0(i,kp1,1)+wind0(i,k,1))/2._r8 - vbot = (wind0(i,kp1,2)+wind0(i,k,2))/2._r8 - fket = utop*mflux(i,k,1) + vtop*mflux(i,k,2) ! top of layer - fkeb = ubot*mflux(i,k+1,1) + vbot*mflux(i,k+1,2) ! bot of layer - - ! divergence of these fluxes should give a conservative redistribution of KE - ketend_cons = (fket-fkeb)/dp(i,k) - - ! tendency in kinetic energy resulting from the momentum transport - ketend = ((windf(i,k,1)**2 + windf(i,k,2)**2) - (wind0(i,k,1)**2 + wind0(i,k,2)**2))*0.5_r8/dt - - ! the difference should be the dissipation - gset2 = ketend_cons - ketend - gseten(i,k) = gset2 - - end do - - end do - - ! Scatter dry static energy to full array - do k = 1,pver - do i = il1g,il2g - ii = ideep(i) - seten(ii,k) = gseten(i,k) - - end do - end do - - return -end subroutine momtran -!endif -! AaronDonahue - skip convtran and momtran for now -!========================================================================================= - -subroutine buoyan(lchnk ,ncol , & - q ,t ,p ,z ,pf , & - tp ,qstp ,tl ,rl ,cape , & - pblt ,lcl ,lel ,lon ,mx , & - rd ,grav ,cp ,msg , & - tpert ) -!----------------------------------------------------------------------- -! -! Purpose: -! -! -! Method: -! -! -! -! Author: -! This is contributed code not fully standardized by the CCM core group. -! The documentation has been enhanced to the degree that we are able. -! Reviewed: P. Rasch, April 1996 -! -!----------------------------------------------------------------------- - implicit none -!----------------------------------------------------------------------- -! -! input arguments -! - integer, intent(in) :: lchnk ! chunk identifier - integer, intent(in) :: ncol ! number of atmospheric columns - - real(r8), intent(in) :: q(pcols,pver) ! spec. humidity - real(r8), intent(in) :: t(pcols,pver) ! temperature - real(r8), intent(in) :: p(pcols,pver) ! pressure - real(r8), intent(in) :: z(pcols,pver) ! height - real(r8), intent(in) :: pf(pcols,pver+1) ! pressure at interfaces - real(r8), intent(in) :: pblt(pcols) ! index of pbl depth - real(r8), intent(in) :: tpert(pcols) ! perturbation temperature by pbl processes - -! -! output arguments -! - real(r8), intent(out) :: tp(pcols,pver) ! parcel temperature - real(r8), intent(out) :: qstp(pcols,pver) ! saturation mixing ratio of parcel - real(r8), intent(out) :: tl(pcols) ! parcel temperature at lcl - real(r8), intent(out) :: cape(pcols) ! convective aval. pot. energy. - integer lcl(pcols) ! - integer lel(pcols) ! - integer lon(pcols) ! level of onset of deep convection - integer mx(pcols) ! level of max moist static energy -! -!--------------------------Local Variables------------------------------ -! - real(r8) capeten(pcols,5) ! provisional value of cape - real(r8) tv(pcols,pver) ! - real(r8) tpv(pcols,pver) ! - real(r8) buoy(pcols,pver) - - real(r8) a1(pcols) - real(r8) a2(pcols) - real(r8) estp(pcols) - real(r8) pl(pcols) - real(r8) plexp(pcols) - real(r8) hmax(pcols) - real(r8) hmn(pcols) - real(r8) y(pcols) - - logical plge600(pcols) - integer knt(pcols) - integer lelten(pcols,5) - - real(r8) cp - real(r8) e - real(r8) grav - - integer i - integer k - integer msg - integer n - - real(r8) rd - real(r8) rl -#ifdef PERGRO - real(r8) rhd -#endif -! -!----------------------------------------------------------------------- -! - do n = 1,5 - do i = 1,ncol - lelten(i,n) = pver - capeten(i,n) = 0._r8 - end do - end do -! - do i = 1,ncol - lon(i) = pver - knt(i) = 0 - lel(i) = pver - mx(i) = lon(i) - cape(i) = 0._r8 - hmax(i) = 0._r8 - end do - - tp(:ncol,:) = t(:ncol,:) - qstp(:ncol,:) = q(:ncol,:) - -!!! RBN - Initialize tv and buoy for output. -!!! tv=tv : tpv=tpv : qstp=q : buoy=0. - tv(:ncol,:) = t(:ncol,:) *(1._r8+1.608_r8*q(:ncol,:))/ (1._r8+q(:ncol,:)) - tpv(:ncol,:) = tv(:ncol,:) - buoy(:ncol,:) = 0._r8 - -! -! set "launching" level(mx) to be at maximum moist static energy. -! search for this level stops at planetary boundary layer top. -! -#ifdef PERGRO - do k = pver,msg + 1,-1 - do i = 1,ncol - hmn(i) = cp*t(i,k) + grav*z(i,k) + rl*q(i,k) -! -! Reset max moist static energy level when relative difference exceeds 1.e-4 -! - rhd = (hmn(i) - hmax(i))/(hmn(i) + hmax(i)) - if (k >= nint(pblt(i)) .and. k <= lon(i) .and. rhd > -1.e-4_r8) then - hmax(i) = hmn(i) - mx(i) = k - end if - end do - end do -#else - do k = pver,msg + 1,-1 - do i = 1,ncol - hmn(i) = cp*t(i,k) + grav*z(i,k) + rl*q(i,k) - if (k >= nint(pblt(i)) .and. k <= lon(i) .and. hmn(i) > hmax(i)) then - hmax(i) = hmn(i) - mx(i) = k - end if - end do - end do -#endif -! - do i = 1,ncol - lcl(i) = mx(i) - e = p(i,mx(i))*q(i,mx(i))/ (eps1+q(i,mx(i))) - tl(i) = 2840._r8/ (3.5_r8*log(t(i,mx(i)))-log(e)-4.805_r8) + 55._r8 - if (tl(i) < t(i,mx(i))) then - plexp(i) = (1._r8/ (0.2854_r8* (1._r8-0.28_r8*q(i,mx(i))))) - pl(i) = p(i,mx(i))* (tl(i)/t(i,mx(i)))**plexp(i) - else - tl(i) = t(i,mx(i)) - pl(i) = p(i,mx(i)) - end if - end do - -! -! calculate lifting condensation level (lcl). -! - do k = pver,msg + 2,-1 - do i = 1,ncol - if (k <= mx(i) .and. (p(i,k) > pl(i) .and. p(i,k-1) <= pl(i))) then - lcl(i) = k - 1 - end if - end do - end do -! -! if lcl is above the nominal level of non-divergence (600 mbs), -! no deep convection is permitted (ensuing calculations -! skipped and cape retains initialized value of zero). -! - do i = 1,ncol - plge600(i) = pl(i).ge.600._r8 - end do -! -! initialize parcel properties in sub-cloud layer below lcl. -! - do k = pver,msg + 1,-1 - do i=1,ncol - if (k > lcl(i) .and. k <= mx(i) .and. plge600(i)) then - tv(i,k) = t(i,k)* (1._r8+1.608_r8*q(i,k))/ (1._r8+q(i,k)) - qstp(i,k) = q(i,mx(i)) - tp(i,k) = t(i,mx(i))* (p(i,k)/p(i,mx(i)))**(0.2854_r8* (1._r8-0.28_r8*q(i,mx(i)))) -! -! buoyancy is increased by 0.5 k as in tiedtke -! -!-jjh tpv (i,k)=tp(i,k)*(1.+1.608*q(i,mx(i)))/ -!-jjh 1 (1.+q(i,mx(i))) - tpv(i,k) = (tp(i,k)+tpert(i))*(1._r8+1.608_r8*q(i,mx(i)))/ (1._r8+q(i,mx(i))) - buoy(i,k) = tpv(i,k) - tv(i,k) + tiedke_add - end if - end do - end do - -! -! define parcel properties at lcl (i.e. level immediately above pl). -! - do k = pver,msg + 1,-1 - do i=1,ncol - if (k == lcl(i) .and. plge600(i)) then - tv(i,k) = t(i,k)* (1._r8+1.608_r8*q(i,k))/ (1._r8+q(i,k)) - qstp(i,k) = q(i,mx(i)) - tp(i,k) = tl(i)* (p(i,k)/pl(i))**(0.2854_r8* (1._r8-0.28_r8*qstp(i,k))) -! estp(i) =exp(21.656_r8 - 5418._r8/tp(i,k)) -! use of different formulas for es has about 1 g/kg difference -! in qs at t= 300k, and 0.02 g/kg at t=263k, with the formula -! above giving larger qs. - call qsat_hPa(tp(i,k), p(i,k), estp(i), qstp(i,k)) - a1(i) = cp / rl + qstp(i,k) * (1._r8+ qstp(i,k) / eps1) * rl * eps1 / & - (rd * tp(i,k) ** 2) - a2(i) = .5_r8* (qstp(i,k)* (1._r8+2._r8/eps1*qstp(i,k))* & - (1._r8+qstp(i,k)/eps1)*eps1**2*rl*rl/ & - (rd**2*tp(i,k)**4)-qstp(i,k)* & - (1._r8+qstp(i,k)/eps1)*2._r8*eps1*rl/ & - (rd*tp(i,k)**3)) - a1(i) = 1._r8/a1(i) - a2(i) = -a2(i)*a1(i)**3 - y(i) = q(i,mx(i)) - qstp(i,k) - tp(i,k) = tp(i,k) + a1(i)*y(i) + a2(i)*y(i)**2 - call qsat_hPa(tp(i,k), p(i,k), estp(i), qstp(i,k)) -! -! buoyancy is increased by 0.5 k in cape calculation. -! dec. 9, 1994 -!-jjh tpv(i,k) =tp(i,k)*(1.+1.608*qstp(i,k))/(1.+q(i,mx(i))) -! - tpv(i,k) = (tp(i,k)+tpert(i))* (1._r8+1.608_r8*qstp(i,k)) / (1._r8+q(i,mx(i))) - buoy(i,k) = tpv(i,k) - tv(i,k) + tiedke_add - end if - end do - end do -! -! main buoyancy calculation. -! - do k = pver - 1,msg + 1,-1 - do i=1,ncol - if (k < lcl(i) .and. plge600(i)) then - tv(i,k) = t(i,k)* (1._r8+1.608_r8*q(i,k))/ (1._r8+q(i,k)) - qstp(i,k) = qstp(i,k+1) - tp(i,k) = tp(i,k+1)* (p(i,k)/p(i,k+1))**(0.2854_r8* (1._r8-0.28_r8*qstp(i,k))) - call qsat_hPa(tp(i,k), p(i,k), estp(i), qstp(i,k)) - a1(i) = cp/rl + qstp(i,k)* (1._r8+qstp(i,k)/eps1)*rl*eps1/ (rd*tp(i,k)**2) - a2(i) = .5_r8* (qstp(i,k)* (1._r8+2._r8/eps1*qstp(i,k))* & - (1._r8+qstp(i,k)/eps1)*eps1**2*rl*rl/ & - (rd**2*tp(i,k)**4)-qstp(i,k)* & - (1._r8+qstp(i,k)/eps1)*2._r8*eps1*rl/ & - (rd*tp(i,k)**3)) - a1(i) = 1._r8/a1(i) - a2(i) = -a2(i)*a1(i)**3 - y(i) = qstp(i,k+1) - qstp(i,k) - tp(i,k) = tp(i,k) + a1(i)*y(i) + a2(i)*y(i)**2 - call qsat_hPa(tp(i,k), p(i,k), estp(i), qstp(i,k)) -!-jjh tpv(i,k) =tp(i,k)*(1.+1.608*qstp(i,k))/ -!jt (1.+q(i,mx(i))) - tpv(i,k) = (tp(i,k)+tpert(i))* (1._r8+1.608_r8*qstp(i,k))/(1._r8+q(i,mx(i))) - buoy(i,k) = tpv(i,k) - tv(i,k) + tiedke_add - end if - end do - end do - -! - do k = msg + 2,pver - do i = 1,ncol - if (k < lcl(i) .and. plge600(i)) then - if (buoy(i,k+1) > 0._r8 .and. buoy(i,k) <= 0._r8) then - knt(i) = min(5,knt(i) + 1) - lelten(i,knt(i)) = k - end if - end if - end do - end do -! -! calculate convective available potential energy (cape). -! - do n = 1,5 - do k = msg + 1,pver - do i = 1,ncol - if (plge600(i) .and. k <= mx(i) .and. k > lelten(i,n)) then - capeten(i,n) = capeten(i,n) + rd*buoy(i,k)*log(pf(i,k+1)/pf(i,k)) - end if - end do - end do - end do -! -! find maximum cape from all possible tentative capes from -! one sounding, -! and use it as the final cape, april 26, 1995 -! - do n = 1,5 - do i = 1,ncol - if (capeten(i,n) > cape(i)) then - cape(i) = capeten(i,n) - lel(i) = lelten(i,n) - end if - end do - end do -! -! put lower bound on cape for diagnostic purposes. -! - do i = 1,ncol - cape(i) = max(cape(i), 0._r8) - end do -! - return -end subroutine buoyan - -subroutine cldprp(lchnk , & - q ,t ,u ,v ,p , & - z ,s ,mu ,eu ,du , & - md ,ed ,sd ,qd ,mc , & - qu ,su ,zf ,qst ,hmn , & - hsat ,shat ,ql , & - cmeg ,jb ,lel ,jt ,jlcl , & - mx ,j0 ,jd ,rl ,il2g , & - rd ,grav ,cp ,msg , & -!songxl 2014-05-20------- -!----------------------------------------------------------------------- -! -! Purpose: -! -! -! Method: -! may 09/91 - guang jun zhang, m.lazare, n.mcfarlane. -! original version cldprop. -! -! Author: See above, modified by P. Rasch -! This is contributed code not fully standardized by the CCM core group. -! -! this code is very much rougher than virtually anything else in the CCM -! there are debug statements left strewn about and code segments disabled -! these are to facilitate future development. We expect to release a -! cleaner code in a future release -! -! the documentation has been enhanced to the degree that we are able -! -!----------------------------------------------------------------------- - - implicit none - -!------------------------------------------------------------------------------ -! -! Input arguments -! - integer, intent(in) :: lchnk ! chunk identifier - - real(r8), intent(in) :: q(pcols,pver) ! spec. humidity of env - real(r8), intent(in) :: t(pcols,pver) ! temp of env - real(r8), intent(in) :: p(pcols,pver) ! pressure of env - real(r8), intent(in) :: z(pcols,pver) ! height of env - real(r8), intent(in) :: s(pcols,pver) ! normalized dry static energy of env - real(r8), intent(in) :: zf(pcols,pverp) ! height of interfaces - real(r8), intent(in) :: u(pcols,pver) ! zonal velocity of env - real(r8), intent(in) :: v(pcols,pver) ! merid. velocity of env - - real(r8), intent(in) :: landfrac(pcols) ! RBN Landfrac - - integer, intent(in) :: jb(pcols) ! updraft base level - integer, intent(in) :: lel(pcols) ! updraft launch level - integer, intent(out) :: jt(pcols) ! updraft plume top - integer, intent(out) :: jlcl(pcols) ! updraft lifting cond level - integer, intent(in) :: mx(pcols) ! updraft base level (same is jb) - integer, intent(out) :: j0(pcols) ! level where updraft begins detraining - integer, intent(out) :: jd(pcols) ! level of downdraft - integer, intent(in) :: limcnv ! convection limiting level - integer, intent(in) :: il2g !CORE GROUP REMOVE - integer, intent(in) :: msg ! missing moisture vals (always 0) - real(r8), intent(in) :: rl ! latent heat of vap - real(r8), intent(in) :: shat(pcols,pver) ! interface values of dry stat energy - real(r8), intent(in) :: tpert(pcols) -! -! output -! - real(r8), intent(out) :: rprd(pcols,pver) ! rate of production of precip at that layer - real(r8), intent(out) :: du(pcols,pver) ! detrainement rate of updraft - real(r8), intent(out) :: ed(pcols,pver) ! entrainment rate of downdraft - real(r8), intent(out) :: eu(pcols,pver) ! entrainment rate of updraft - real(r8), intent(out) :: hmn(pcols,pver) ! moist stat energy of env - real(r8), intent(out) :: hsat(pcols,pver) ! sat moist stat energy of env - real(r8), intent(out) :: mc(pcols,pver) ! net mass flux - real(r8), intent(out) :: md(pcols,pver) ! downdraft mass flux - real(r8), intent(out) :: mu(pcols,pver) ! updraft mass flux - real(r8), intent(out) :: pflx(pcols,pverp) ! precipitation flux thru layer - real(r8), intent(out) :: qd(pcols,pver) ! spec humidity of downdraft - real(r8), intent(out) :: ql(pcols,pver) ! liq water of updraft - real(r8), intent(out) :: qst(pcols,pver) ! saturation mixing ratio of env. - real(r8), intent(out) :: qu(pcols,pver) ! spec hum of updraft - real(r8), intent(out) :: sd(pcols,pver) ! normalized dry stat energy of downdraft - real(r8), intent(out) :: su(pcols,pver) ! normalized dry stat energy of updraft - -!songxl 2014-05-20---------------- - - real(r8) rd ! gas constant for dry air - real(r8) grav ! gravity - real(r8) cp ! heat capacity of dry air - -! -! Local workspace -! - real(r8) gamma(pcols,pver) - real(r8) dz(pcols,pver) - real(r8) iprm(pcols,pver) - real(r8) hu(pcols,pver) - real(r8) hd(pcols,pver) - real(r8) eps(pcols,pver) - real(r8) f(pcols,pver) - real(r8) k1(pcols,pver) - real(r8) i2(pcols,pver) - real(r8) ihat(pcols,pver) - real(r8) i3(pcols,pver) - real(r8) idag(pcols,pver) - real(r8) i4(pcols,pver) - real(r8) qsthat(pcols,pver) - real(r8) hsthat(pcols,pver) - real(r8) gamhat(pcols,pver) - real(r8) cu(pcols,pver) - real(r8) evp(pcols,pver) - real(r8) cmeg(pcols,pver) - real(r8) qds(pcols,pver) -! RBN For c0mask - real(r8) c0mask(pcols) - - real(r8) hmin(pcols) - real(r8) expdif(pcols) - real(r8) expnum(pcols) - real(r8) ftemp(pcols) - real(r8) eps0(pcols) - real(r8) rmue(pcols) - real(r8) zuef(pcols) - real(r8) zdef(pcols) - real(r8) epsm(pcols) - real(r8) ratmjb(pcols) - real(r8) est(pcols) - real(r8) totpcp(pcols) - real(r8) totevp(pcols) - real(r8) alfa(pcols) - real(r8) ql1 - real(r8) tu - real(r8) estu - real(r8) qstu - - real(r8) small - real(r8) mdt - - integer khighest - integer klowest - integer kount - integer i,k - - logical doit(pcols) - logical done(pcols) - -!songxl 2014-05-20---------------- -! -!------------------------------------------------------------------------------ -! - do i = 1,il2g - ftemp(i) = 0._r8 - expnum(i) = 0._r8 - expdif(i) = 0._r8 - c0mask(i) = c0_ocn * (1._r8-landfrac(i)) + c0_lnd * landfrac(i) - end do -! -!jr Change from msg+1 to 1 to prevent blowup -! - do k = 1,pver - do i = 1,il2g - dz(i,k) = zf(i,k) - zf(i,k+1) - end do - end do - -! -! initialize many output and work variables to zero -! - pflx(:il2g,1) = 0 - - do k = 1,pver - do i = 1,il2g - k1(i,k) = 0._r8 - i2(i,k) = 0._r8 - i3(i,k) = 0._r8 - i4(i,k) = 0._r8 - mu(i,k) = 0._r8 - f(i,k) = 0._r8 - eps(i,k) = 0._r8 - eu(i,k) = 0._r8 - du(i,k) = 0._r8 - ql(i,k) = 0._r8 - cu(i,k) = 0._r8 - evp(i,k) = 0._r8 - cmeg(i,k) = 0._r8 - qds(i,k) = q(i,k) - md(i,k) = 0._r8 - ed(i,k) = 0._r8 - sd(i,k) = s(i,k) - qd(i,k) = q(i,k) - mc(i,k) = 0._r8 - qu(i,k) = q(i,k) - su(i,k) = s(i,k) - call qsat_hPa(t(i,k), p(i,k), est(i), qst(i,k)) -!++bee - if ( p(i,k)-est(i) <= 0._r8 ) then - qst(i,k) = 1.0_r8 - end if -!--bee - gamma(i,k) = qst(i,k)*(1._r8 + qst(i,k)/eps1)*eps1*rl/(rd*t(i,k)**2)*rl/cp - hmn(i,k) = cp*t(i,k) + grav*z(i,k) + rl*q(i,k) - hsat(i,k) = cp*t(i,k) + grav*z(i,k) + rl*qst(i,k) - hu(i,k) = hmn(i,k) - hd(i,k) = hmn(i,k) - rprd(i,k) = 0._r8 - end do - end do -! -!jr Set to zero things which make this routine blow up -! - do k=1,msg - do i=1,il2g - rprd(i,k) = 0._r8 - end do - end do -! -! interpolate the layer values of qst, hsat and gamma to -! layer interfaces -! - do k = 1, msg+1 - do i = 1,il2g - hsthat(i,k) = hsat(i,k) - qsthat(i,k) = qst(i,k) - gamhat(i,k) = gamma(i,k) - end do - end do - do i = 1,il2g - totpcp(i) = 0._r8 - totevp(i) = 0._r8 - end do - do k = msg + 2,pver - do i = 1,il2g - if (abs(qst(i,k-1)-qst(i,k)) > 1.E-6_r8) then - qsthat(i,k) = log(qst(i,k-1)/qst(i,k))*qst(i,k-1)*qst(i,k)/ (qst(i,k-1)-qst(i,k)) - else - qsthat(i,k) = qst(i,k) - end if - hsthat(i,k) = cp*shat(i,k) + rl*qsthat(i,k) - if (abs(gamma(i,k-1)-gamma(i,k)) > 1.E-6_r8) then - gamhat(i,k) = log(gamma(i,k-1)/gamma(i,k))*gamma(i,k-1)*gamma(i,k)/ & - (gamma(i,k-1)-gamma(i,k)) - else - gamhat(i,k) = gamma(i,k) - end if - end do - end do -! -! initialize cloud top to highest plume top. -!jr changed hard-wired 4 to limcnv+1 (not to exceed pver) -! - jt(:) = pver - do i = 1,il2g - jt(i) = max(lel(i),limcnv+1) - jt(i) = min(jt(i),pver) - jd(i) = pver - jlcl(i) = lel(i) - hmin(i) = 1.E6_r8 - end do -! -! find the level of minimum hsat, where detrainment starts -! - - do k = msg + 1,pver - do i = 1,il2g - if (hsat(i,k) <= hmin(i) .and. k >= jt(i) .and. k <= jb(i)) then - hmin(i) = hsat(i,k) - j0(i) = k - end if - end do - end do - do i = 1,il2g - j0(i) = min(j0(i),jb(i)-2) - j0(i) = max(j0(i),jt(i)+2) -! -! Fix from Guang Zhang to address out of bounds array reference -! - j0(i) = min(j0(i),pver) - end do -! -! Initialize certain arrays inside cloud -! - do k = msg + 1,pver - do i = 1,il2g - if (k >= jt(i) .and. k <= jb(i)) then - hu(i,k) = hmn(i,mx(i)) + cp*(tiedke_add+tp_fac*tpert(i)) !PMA - su(i,k) = s(i,mx(i)) + tiedke_add+tp_fac*tpert(i) - end if - end do - end do -! -! ********************************************************* -! compute taylor series for approximate eps(z) below -! ********************************************************* -! - do k = pver - 1,msg + 1,-1 - do i = 1,il2g - if (k < jb(i) .and. k >= jt(i)) then - k1(i,k) = k1(i,k+1) + (hmn(i,mx(i))-hmn(i,k))*dz(i,k) - ihat(i,k) = 0.5_r8* (k1(i,k+1)+k1(i,k)) - i2(i,k) = i2(i,k+1) + ihat(i,k)*dz(i,k) - idag(i,k) = 0.5_r8* (i2(i,k+1)+i2(i,k)) - i3(i,k) = i3(i,k+1) + idag(i,k)*dz(i,k) - iprm(i,k) = 0.5_r8* (i3(i,k+1)+i3(i,k)) - i4(i,k) = i4(i,k+1) + iprm(i,k)*dz(i,k) - end if - end do - end do -! -! re-initialize hmin array for ensuing calculation. -! - do i = 1,il2g - hmin(i) = 1.E6_r8 - end do - do k = msg + 1,pver - do i = 1,il2g - if (k >= j0(i) .and. k <= jb(i) .and. hmn(i,k) <= hmin(i)) then - hmin(i) = hmn(i,k) - expdif(i) = hmn(i,mx(i)) - hmin(i) - end if - end do - end do -! -! ********************************************************* -! compute approximate eps(z) using above taylor series -! ********************************************************* -! - do k = msg + 2,pver - do i = 1,il2g - expnum(i) = 0._r8 - ftemp(i) = 0._r8 - if (k < jt(i) .or. k >= jb(i)) then - k1(i,k) = 0._r8 - expnum(i) = 0._r8 - else - expnum(i) = hmn(i,mx(i)) - (hsat(i,k-1)*(zf(i,k)-z(i,k)) + & - hsat(i,k)* (z(i,k-1)-zf(i,k)))/(z(i,k-1)-z(i,k)) - end if - if ((expdif(i) > 100._r8 .and. expnum(i) > 0._r8) .and. & - k1(i,k) > expnum(i)*dz(i,k)) then - ftemp(i) = expnum(i)/k1(i,k) - f(i,k) = ftemp(i) + i2(i,k)/k1(i,k)*ftemp(i)**2 + & - (2._r8*i2(i,k)**2-k1(i,k)*i3(i,k))/k1(i,k)**2* & - ftemp(i)**3 + (-5._r8*k1(i,k)*i2(i,k)*i3(i,k)+ & - 5._r8*i2(i,k)**3+k1(i,k)**2*i4(i,k))/ & - k1(i,k)**3*ftemp(i)**4 - f(i,k) = max(f(i,k),0._r8) - f(i,k) = min(f(i,k),0.0002_r8) - end if - end do - end do - do i = 1,il2g - if (j0(i) < jb(i)) then - if (f(i,j0(i)) < 1.E-6_r8 .and. f(i,j0(i)+1) > f(i,j0(i))) j0(i) = j0(i) + 1 - end if - end do - do k = msg + 2,pver - do i = 1,il2g - if (k >= jt(i) .and. k <= j0(i)) then - f(i,k) = max(f(i,k),f(i,k-1)) - end if - end do - end do - do i = 1,il2g - eps0(i) = f(i,j0(i)) - eps(i,jb(i)) = eps0(i) - end do -! -! This is set to match the Rasch and Kristjansson paper -! - do k = pver,msg + 1,-1 - do i = 1,il2g - if (k >= j0(i) .and. k <= jb(i)) then - eps(i,k) = f(i,j0(i)) - end if - end do - end do - do k = pver,msg + 1,-1 - do i = 1,il2g - if (k < j0(i) .and. k >= jt(i)) eps(i,k) = f(i,k) - end do - end do -! -! specify the updraft mass flux mu, entrainment eu, detrainment du -! and moist static energy hu. -! here and below mu, eu,du, md and ed are all normalized by mb -! - do i = 1,il2g - if (eps0(i) > 0._r8) then - mu(i,jb(i)) = 1._r8 - eu(i,jb(i)) = mu(i,jb(i))/dz(i,jb(i)) - end if - end do - do k = pver,msg + 1,-1 - do i = 1,il2g - if (eps0(i) > 0._r8 .and. (k >= jt(i) .and. k < jb(i))) then - zuef(i) = zf(i,k) - zf(i,jb(i)) - rmue(i) = (1._r8/eps0(i))* (exp(eps(i,k+1)*zuef(i))-1._r8)/zuef(i) - mu(i,k) = (1._r8/eps0(i))* (exp(eps(i,k )*zuef(i))-1._r8)/zuef(i) - eu(i,k) = (rmue(i)-mu(i,k+1))/dz(i,k) - du(i,k) = (rmue(i)-mu(i,k))/dz(i,k) - end if - end do - end do -! - khighest = pverp - klowest = 1 - do i=1,il2g - khighest = min(khighest,lel(i)) - klowest = max(klowest,jb(i)) - end do - -!songxl 2014-05-20-------------- - - do k = klowest-1,khighest,-1 - do i = 1,il2g - if (k <= jb(i)-1 .and. k >= lel(i) .and. eps0(i) > 0._r8) then - if (mu(i,k) < 0.02_r8) then - hu(i,k) = hmn(i,k) - mu(i,k) = 0._r8 - eu(i,k) = 0._r8 - du(i,k) = mu(i,k+1)/dz(i,k) - else -!songxl 2014-05-20-------------- - end if - end if - end do - end do -! -! -! reset cloud top index beginning from two layers above the -! cloud base (i.e. if cloud is only one layer thick, top is not reset -! - do i=1,il2g - doit(i) = .true. - end do - do k=klowest-2,khighest-1,-1 - do i=1,il2g - if (doit(i) .and. k <= jb(i)-2 .and. k >= lel(i)-1) then - if(trigmem)then - if (hu(i,k) <= hsthat(i,k) .and. hu(i,k+1) > hsthat(i,k+1) & - .and. mu(i,k) >= 0.02_r8) then - if (hu(i,k)-hsthat(i,k) < -2000._r8) then - jt(i) = k + 1 - doit(i) = .false. - else - jt(i) = k - doit(i) = .false. - end if - else - if (hu_tot(i).eq.hmn_tot(i)) then - if (hu(i,k) > hu(i,jb(i)) .or. mu(i,k) < 0.02_r8) then - jt(i) = k + 1 - doit(i) = .false. - end if - else - if ( mu(i,k) < 0.02_r8) then - jt(i) = k + 1 - doit(i) = .false. - end if - end if - end if - else ! not trigmem - if (hu(i,k) <= hsthat(i,k) .and. hu(i,k+1) > hsthat(i,k+1) & - .and. mu(i,k) >= 0.02_r8) then - if (hu(i,k)-hsthat(i,k) < -2000._r8) then - jt(i) = k + 1 - doit(i) = .false. - else - jt(i) = k - doit(i) = .false. - end if - else if (hu(i,k) > hu(i,jb(i)) .or. mu(i,k) < 0.02_r8) then - jt(i) = k + 1 - doit(i) = .false. - end if - end if ! end trigmem -!>songxl 2014-06-20------------------------ - end if - end do - end do - do k = pver,msg + 1,-1 - do i = 1,il2g - if (k >= lel(i) .and. k <= jt(i) .and. eps0(i) > 0._r8) then - mu(i,k) = 0._r8 - eu(i,k) = 0._r8 - du(i,k) = 0._r8 - hu(i,k) = hmn(i,k) - end if - if (k == jt(i) .and. eps0(i) > 0._r8) then - du(i,k) = mu(i,k+1)/dz(i,k) - eu(i,k) = 0._r8 - mu(i,k) = 0._r8 - end if - end do - end do -! -!songxl 2014-05-20----------------- - -! specify downdraft properties (no downdrafts if jd.ge.jb). -! scale down downward mass flux profile so that net flux -! (up-down) at cloud base in not negative. -! - do i = 1,il2g -! -! in normal downdraft strength run alfa=0.2. In test4 alfa=0.1 -! - alfa(i) = alfa_scalar - jt(i) = min(jt(i),jb(i)-1) - jd(i) = max(j0(i),jt(i)+1) - jd(i) = min(jd(i),jb(i)) - hd(i,jd(i)) = hmn(i,jd(i)-1) - if (jd(i) < jb(i) .and. eps0(i) > 0._r8) then - epsm(i) = eps0(i) - md(i,jd(i)) = -alfa(i)*epsm(i)/eps0(i) - end if - end do - do k = msg + 1,pver - do i = 1,il2g - if ((k > jd(i) .and. k <= jb(i)) .and. eps0(i) > 0._r8) then - zdef(i) = zf(i,jd(i)) - zf(i,k) - md(i,k) = -alfa(i)/ (2._r8*eps0(i))*(exp(2._r8*epsm(i)*zdef(i))-1._r8)/zdef(i) - end if - end do - end do - do k = msg + 1,pver - do i = 1,il2g - if ((k >= jt(i) .and. k <= jb(i)) .and. eps0(i) > 0._r8 .and. jd(i) < jb(i)) then - ratmjb(i) = min(abs(mu(i,jb(i))/md(i,jb(i))),1._r8) - md(i,k) = md(i,k)*ratmjb(i) - end if - end do - end do - - small = 1.e-20_r8 - do k = msg + 1,pver - do i = 1,il2g - if ((k >= jt(i) .and. k <= pver) .and. eps0(i) > 0._r8) then - ed(i,k-1) = (md(i,k-1)-md(i,k))/dz(i,k-1) - mdt = min(md(i,k),-small) - hd(i,k) = (md(i,k-1)*hd(i,k-1) - dz(i,k-1)*ed(i,k-1)*hmn(i,k-1))/mdt - end if - end do - end do -! -! calculate updraft and downdraft properties. -! - do k = msg + 2,pver - do i = 1,il2g - if ((k >= jd(i) .and. k <= jb(i)) .and. eps0(i) > 0._r8 .and. jd(i) < jb(i)) then - qds(i,k) = qsthat(i,k) + gamhat(i,k)*(hd(i,k)-hsthat(i,k))/ & - (rl*(1._r8 + gamhat(i,k))) - end if - end do - end do -! - do i = 1,il2g - done(i) = .false. - end do - kount = 0 - do k = pver,msg + 2,-1 - do i = 1,il2g - if (k == jb(i) .and. eps0(i) > 0._r8) then - qu(i,k) = q(i,mx(i)) - su(i,k) = (hu(i,k)-rl*qu(i,k))/cp - end if - if (( .not. done(i) .and. k > jt(i) .and. k < jb(i)) .and. eps0(i) > 0._r8) then - su(i,k) = mu(i,k+1)/mu(i,k)*su(i,k+1) + & - dz(i,k)/mu(i,k)* (eu(i,k)-du(i,k))*s(i,k) - qu(i,k) = mu(i,k+1)/mu(i,k)*qu(i,k+1) + dz(i,k)/mu(i,k)* (eu(i,k)*q(i,k)- & - du(i,k)*qst(i,k)) - tu = su(i,k) - grav/cp*zf(i,k) - call qsat_hPa(tu, (p(i,k)+p(i,k-1))/2._r8, estu, qstu) - if (qu(i,k) >= qstu) then - jlcl(i) = k - kount = kount + 1 - done(i) = .true. - end if - end if - end do - if (kount >= il2g) goto 690 - end do -690 continue - do k = msg + 2,pver - do i = 1,il2g - if ((k > jt(i) .and. k <= jlcl(i)) .and. eps0(i) > 0._r8) then - su(i,k) = shat(i,k) + (hu(i,k)-hsthat(i,k))/(cp* (1._r8+gamhat(i,k))) - qu(i,k) = qsthat(i,k) + gamhat(i,k)*(hu(i,k)-hsthat(i,k))/ & - (rl* (1._r8+gamhat(i,k))) - end if - end do - end do - -! compute condensation in updraft - do k = pver,msg + 2,-1 - do i = 1,il2g - if (k >= jt(i) .and. k < jb(i) .and. eps0(i) > 0._r8) then - cu(i,k) = ((mu(i,k)*su(i,k)-mu(i,k+1)*su(i,k+1))/ & - dz(i,k)- (eu(i,k)-du(i,k))*s(i,k))/(rl/cp) - if (k == jt(i)) cu(i,k) = 0._r8 - cu(i,k) = max(0._r8,cu(i,k)) - end if - end do - end do - -! compute condensed liquid, rain production rate -! accumulate total precipitation (condensation - detrainment of liquid) -! Note ql1 = ql(k) + rprd(k)*dz(k)/mu(k) -! The differencing is somewhat strange (e.g. du(i,k)*ql(i,k+1)) but is -! consistently applied. -! mu, ql are interface quantities -! cu, du, eu, rprd are midpoint quantites - do k = pver,msg + 2,-1 - do i = 1,il2g - rprd(i,k) = 0._r8 - if (k >= jt(i) .and. k < jb(i) .and. eps0(i) > 0._r8 .and. mu(i,k) >= 0.0_r8) then - if (mu(i,k) > 0._r8) then - ql1 = 1._r8/mu(i,k)* (mu(i,k+1)*ql(i,k+1)- & - dz(i,k)*du(i,k)*ql(i,k+1)+dz(i,k)*cu(i,k)) - ql(i,k) = ql1/ (1._r8+dz(i,k)*c0mask(i)) - else - ql(i,k) = 0._r8 - end if - totpcp(i) = totpcp(i) + dz(i,k)*(cu(i,k)-du(i,k)*ql(i,k+1)) - rprd(i,k) = c0mask(i)*mu(i,k)*ql(i,k) - end if - end do - end do -! - do i = 1,il2g - qd(i,jd(i)) = qds(i,jd(i)) - sd(i,jd(i)) = (hd(i,jd(i)) - rl*qd(i,jd(i)))/cp - end do -! - do k = msg + 2,pver - do i = 1,il2g - if (k >= jd(i) .and. k < jb(i) .and. eps0(i) > 0._r8) then - qd(i,k+1) = qds(i,k+1) - evp(i,k) = -ed(i,k)*q(i,k) + (md(i,k)*qd(i,k)-md(i,k+1)*qd(i,k+1))/dz(i,k) - evp(i,k) = max(evp(i,k),0._r8) - mdt = min(md(i,k+1),-small) - sd(i,k+1) = ((rl/cp*evp(i,k)-ed(i,k)*s(i,k))*dz(i,k) + md(i,k)*sd(i,k))/mdt - totevp(i) = totevp(i) - dz(i,k)*ed(i,k)*q(i,k) - end if - end do - end do - do i = 1,il2g -!*guang totevp(i) = totevp(i) + md(i,jd(i))*q(i,jd(i)-1) - - totevp(i) = totevp(i) + md(i,jd(i))*qd(i,jd(i)) - md(i,jb(i))*qd(i,jb(i)) - end do -!!$ if (.true.) then - if (.false.) then - do i = 1,il2g - k = jb(i) - if (eps0(i) > 0._r8) then - evp(i,k) = -ed(i,k)*q(i,k) + (md(i,k)*qd(i,k))/dz(i,k) - evp(i,k) = max(evp(i,k),0._r8) - totevp(i) = totevp(i) - dz(i,k)*ed(i,k)*q(i,k) - end if - end do - endif - - do i = 1,il2g - totpcp(i) = max(totpcp(i),0._r8) - totevp(i) = max(totevp(i),0._r8) - end do -! - do k = msg + 2,pver - do i = 1,il2g - if (totevp(i) > 0._r8 .and. totpcp(i) > 0._r8) then - md(i,k) = md (i,k)*min(1._r8, totpcp(i)/(totevp(i)+totpcp(i))) - ed(i,k) = ed (i,k)*min(1._r8, totpcp(i)/(totevp(i)+totpcp(i))) - evp(i,k) = evp(i,k)*min(1._r8, totpcp(i)/(totevp(i)+totpcp(i))) - else - md(i,k) = 0._r8 - ed(i,k) = 0._r8 - evp(i,k) = 0._r8 - end if -! cmeg is the cloud water condensed - rain water evaporated -! rprd is the cloud water converted to rain - (rain evaporated) - cmeg(i,k) = cu(i,k) - evp(i,k) - rprd(i,k) = rprd(i,k)-evp(i,k) - end do - end do - -! compute the net precipitation flux across interfaces - pflx(:il2g,1) = 0._r8 - do k = 2,pverp - do i = 1,il2g - pflx(i,k) = pflx(i,k-1) + rprd(i,k-1)*dz(i,k-1) - end do - end do -! - do k = msg + 1,pver - do i = 1,il2g - mc(i,k) = mu(i,k) + md(i,k) - end do - end do -! - return -end subroutine cldprp - -subroutine closure(lchnk , & - q ,t ,p ,z ,s , & - tp ,qs ,qu ,su ,mc , & - du ,mu ,md ,qd ,sd , & - qhat ,shat ,dp ,qstp ,zf , & - ql ,dsubcld ,mb ,cape ,tl , & - lcl ,lel ,jt ,mx ,il1g , & - il2g ,rd ,grav ,cp ,rl , & - msg ,capelmt ) -!----------------------------------------------------------------------- -! -! Purpose: -! -! -! Method: -! -! -! -! Author: G. Zhang and collaborators. CCM contact:P. Rasch -! This is contributed code not fully standardized by the CCM core group. -! -! this code is very much rougher than virtually anything else in the CCM -! We expect to release cleaner code in a future release -! -! the documentation has been enhanced to the degree that we are able -! -!----------------------------------------------------------------------- - - implicit none - -! -!-----------------------------Arguments--------------------------------- -! - integer, intent(in) :: lchnk ! chunk identifier - - real(r8), intent(inout) :: q(pcols,pver) ! spec humidity - real(r8), intent(inout) :: t(pcols,pver) ! temperature - real(r8), intent(inout) :: p(pcols,pver) ! pressure (mb) - real(r8), intent(inout) :: mb(pcols) ! cloud base mass flux - real(r8), intent(in) :: z(pcols,pver) ! height (m) - real(r8), intent(in) :: s(pcols,pver) ! normalized dry static energy - real(r8), intent(in) :: tp(pcols,pver) ! parcel temp - real(r8), intent(in) :: qs(pcols,pver) ! sat spec humidity - real(r8), intent(in) :: qu(pcols,pver) ! updraft spec. humidity - real(r8), intent(in) :: su(pcols,pver) ! normalized dry stat energy of updraft - real(r8), intent(in) :: mc(pcols,pver) ! net convective mass flux - real(r8), intent(in) :: du(pcols,pver) ! detrainment from updraft - real(r8), intent(in) :: mu(pcols,pver) ! mass flux of updraft - real(r8), intent(in) :: md(pcols,pver) ! mass flux of downdraft - real(r8), intent(in) :: qd(pcols,pver) ! spec. humidity of downdraft - real(r8), intent(in) :: sd(pcols,pver) ! dry static energy of downdraft - real(r8), intent(in) :: qhat(pcols,pver) ! environment spec humidity at interfaces - real(r8), intent(in) :: shat(pcols,pver) ! env. normalized dry static energy at intrfcs - real(r8), intent(in) :: dp(pcols,pver) ! pressure thickness of layers - real(r8), intent(in) :: qstp(pcols,pver) ! spec humidity of parcel - real(r8), intent(in) :: zf(pcols,pver+1) ! height of interface levels - real(r8), intent(in) :: ql(pcols,pver) ! liquid water mixing ratio - - real(r8), intent(in) :: cape(pcols) ! available pot. energy of column - real(r8), intent(in) :: tl(pcols) - real(r8), intent(in) :: dsubcld(pcols) ! thickness of subcloud layer - - integer, intent(in) :: lcl(pcols) ! index of lcl - integer, intent(in) :: lel(pcols) ! index of launch leve - integer, intent(in) :: jt(pcols) ! top of updraft - integer, intent(in) :: mx(pcols) ! base of updraft -! -!--------------------------Local variables------------------------------ -! - real(r8) dtpdt(pcols,pver) - real(r8) dqsdtp(pcols,pver) - real(r8) dtmdt(pcols,pver) - real(r8) dqmdt(pcols,pver) - real(r8) dboydt(pcols,pver) - real(r8) thetavp(pcols,pver) - real(r8) thetavm(pcols,pver) - - real(r8) dtbdt(pcols),dqbdt(pcols),dtldt(pcols) - real(r8) beta - real(r8) capelmt - real(r8) cp - real(r8) dadt(pcols) - real(r8) debdt - real(r8) dltaa - real(r8) eb - real(r8) grav - - integer i - integer il1g - integer il2g - integer k, kmin, kmax - integer msg - - real(r8) rd - real(r8) rl -! change of subcloud layer properties due to convection is -! related to cumulus updrafts and downdrafts. -! mc(z)=f(z)*mb, mub=betau*mb, mdb=betad*mb are used -! to define betau, betad and f(z). -! note that this implies all time derivatives are in effect -! time derivatives per unit cloud-base mass flux, i.e. they -! have units of 1/mb instead of 1/sec. -! - do i = il1g,il2g - mb(i) = 0._r8 - eb = p(i,mx(i))*q(i,mx(i))/ (eps1+q(i,mx(i))) - dtbdt(i) = (1._r8/dsubcld(i))* (mu(i,mx(i))*(shat(i,mx(i))-su(i,mx(i)))+ & - md(i,mx(i))* (shat(i,mx(i))-sd(i,mx(i)))) - dqbdt(i) = (1._r8/dsubcld(i))* (mu(i,mx(i))*(qhat(i,mx(i))-qu(i,mx(i)))+ & - md(i,mx(i))* (qhat(i,mx(i))-qd(i,mx(i)))) - debdt = eps1*p(i,mx(i))/ (eps1+q(i,mx(i)))**2*dqbdt(i) - dtldt(i) = -2840._r8* (3.5_r8/t(i,mx(i))*dtbdt(i)-debdt/eb)/ & - (3.5_r8*log(t(i,mx(i)))-log(eb)-4.805_r8)**2 - end do -! -! dtmdt and dqmdt are cumulus heating and drying. -! - do k = msg + 1,pver - do i = il1g,il2g - dtmdt(i,k) = 0._r8 - dqmdt(i,k) = 0._r8 - end do - end do -! - do k = msg + 1,pver - 1 - do i = il1g,il2g - if (k == jt(i)) then - dtmdt(i,k) = (1._r8/dp(i,k))*(mu(i,k+1)* (su(i,k+1)-shat(i,k+1)- & - rl/cp*ql(i,k+1))+md(i,k+1)* (sd(i,k+1)-shat(i,k+1))) - dqmdt(i,k) = (1._r8/dp(i,k))*(mu(i,k+1)* (qu(i,k+1)- & - qhat(i,k+1)+ql(i,k+1))+md(i,k+1)*(qd(i,k+1)-qhat(i,k+1))) - end if - end do - end do -! - beta = 0._r8 - do k = msg + 1,pver - 1 - do i = il1g,il2g - if (k > jt(i) .and. k < mx(i)) then - dtmdt(i,k) = (mc(i,k)* (shat(i,k)-s(i,k))+mc(i,k+1)* (s(i,k)-shat(i,k+1)))/ & - dp(i,k) - rl/cp*du(i,k)*(beta*ql(i,k)+ (1-beta)*ql(i,k+1)) -! dqmdt(i,k)=(mc(i,k)*(qhat(i,k)-q(i,k)) -! 1 +mc(i,k+1)*(q(i,k)-qhat(i,k+1)))/dp(i,k) -! 2 +du(i,k)*(qs(i,k)-q(i,k)) -! 3 +du(i,k)*(beta*ql(i,k)+(1-beta)*ql(i,k+1)) - - dqmdt(i,k) = (mu(i,k+1)* (qu(i,k+1)-qhat(i,k+1)+cp/rl* (su(i,k+1)-s(i,k)))- & - mu(i,k)* (qu(i,k)-qhat(i,k)+cp/rl*(su(i,k)-s(i,k)))+md(i,k+1)* & - (qd(i,k+1)-qhat(i,k+1)+cp/rl*(sd(i,k+1)-s(i,k)))-md(i,k)* & - (qd(i,k)-qhat(i,k)+cp/rl*(sd(i,k)-s(i,k))))/dp(i,k) + & - du(i,k)* (beta*ql(i,k)+(1-beta)*ql(i,k+1)) - end if - end do - end do -! - do k = msg + 1,pver - do i = il1g,il2g - if (k >= lel(i) .and. k <= lcl(i)) then - thetavp(i,k) = tp(i,k)* (1000._r8/p(i,k))** (rd/cp)*(1._r8+1.608_r8*qstp(i,k)-q(i,mx(i))) - thetavm(i,k) = t(i,k)* (1000._r8/p(i,k))** (rd/cp)*(1._r8+0.608_r8*q(i,k)) - dqsdtp(i,k) = qstp(i,k)* (1._r8+qstp(i,k)/eps1)*eps1*rl/(rd*tp(i,k)**2) -! -! dtpdt is the parcel temperature change due to change of -! subcloud layer properties during convection. -! - dtpdt(i,k) = tp(i,k)/ (1._r8+rl/cp* (dqsdtp(i,k)-qstp(i,k)/tp(i,k)))* & - (dtbdt(i)/t(i,mx(i))+rl/cp* (dqbdt(i)/tl(i)-q(i,mx(i))/ & - tl(i)**2*dtldt(i))) -! -! dboydt is the integrand of cape change. -! - dboydt(i,k) = ((dtpdt(i,k)/tp(i,k)+1._r8/(1._r8+1.608_r8*qstp(i,k)-q(i,mx(i)))* & - (1.608_r8 * dqsdtp(i,k) * dtpdt(i,k) -dqbdt(i))) - (dtmdt(i,k)/t(i,k)+0.608_r8/ & - (1._r8+0.608_r8*q(i,k))*dqmdt(i,k)))*grav*thetavp(i,k)/thetavm(i,k) - end if - end do - end do -! - do k = msg + 1,pver - do i = il1g,il2g - if (k > lcl(i) .and. k < mx(i)) then - thetavp(i,k) = tp(i,k)* (1000._r8/p(i,k))** (rd/cp)*(1._r8+0.608_r8*q(i,mx(i))) - thetavm(i,k) = t(i,k)* (1000._r8/p(i,k))** (rd/cp)*(1._r8+0.608_r8*q(i,k)) -! -! dboydt is the integrand of cape change. -! - dboydt(i,k) = (dtbdt(i)/t(i,mx(i))+0.608_r8/ (1._r8+0.608_r8*q(i,mx(i)))*dqbdt(i)- & - dtmdt(i,k)/t(i,k)-0.608_r8/ (1._r8+0.608_r8*q(i,k))*dqmdt(i,k))* & - grav*thetavp(i,k)/thetavm(i,k) - end if - end do - end do - -! -! buoyant energy change is set to 2/3*excess cape per 3 hours -! - dadt(il1g:il2g) = 0._r8 - kmin = minval(lel(il1g:il2g)) - kmax = maxval(mx(il1g:il2g)) - 1 - do k = kmin, kmax - do i = il1g,il2g - if ( k >= lel(i) .and. k <= mx(i) - 1) then - dadt(i) = dadt(i) + dboydt(i,k)* (zf(i,k)-zf(i,k+1)) - endif - end do - end do - do i = il1g,il2g - dltaa = -1._r8* (cape(i)-capelmt) - if (dadt(i) /= 0._r8) mb(i) = max(dltaa/tau/dadt(i),0._r8) - end do -! - return -end subroutine closure - -subroutine q1q2_pjr(lchnk , & - dqdt ,dsdt ,q ,qs ,qu , & - su ,du ,qhat ,shat ,dp , & - mu ,md ,sd ,qd ,ql , & - dsubcld ,jt ,mx ,il1g ,il2g , & - cp ,rl ,msg , & - dl ,evp ,cu ) - - - implicit none - -!----------------------------------------------------------------------- -! -! Purpose: -! -! -! Method: -! -! -! -! Author: phil rasch dec 19 1995 -! -!----------------------------------------------------------------------- - - - real(r8), intent(in) :: cp - - integer, intent(in) :: lchnk ! chunk identifier - integer, intent(in) :: il1g - integer, intent(in) :: il2g - integer, intent(in) :: msg - - real(r8), intent(in) :: q(pcols,pver) - real(r8), intent(in) :: qs(pcols,pver) - real(r8), intent(in) :: qu(pcols,pver) - real(r8), intent(in) :: su(pcols,pver) - real(r8), intent(in) :: du(pcols,pver) - real(r8), intent(in) :: qhat(pcols,pver) - real(r8), intent(in) :: shat(pcols,pver) - real(r8), intent(in) :: dp(pcols,pver) - real(r8), intent(in) :: mu(pcols,pver) - real(r8), intent(in) :: md(pcols,pver) - real(r8), intent(in) :: sd(pcols,pver) - real(r8), intent(in) :: qd(pcols,pver) - real(r8), intent(in) :: ql(pcols,pver) - real(r8), intent(in) :: evp(pcols,pver) - real(r8), intent(in) :: cu(pcols,pver) - real(r8), intent(in) :: dsubcld(pcols) - - real(r8),intent(out) :: dqdt(pcols,pver),dsdt(pcols,pver) - real(r8),intent(out) :: dl(pcols,pver) - integer kbm - integer ktm - integer jt(pcols) - integer mx(pcols) -! -! work fields: -! - integer i - integer k - - real(r8) emc - real(r8) rl -!------------------------------------------------------------------- - do k = msg + 1,pver - do i = il1g,il2g - dsdt(i,k) = 0._r8 - dqdt(i,k) = 0._r8 - dl(i,k) = 0._r8 - end do - end do -! -! find the highest level top and bottom levels of convection -! - ktm = pver - kbm = pver - do i = il1g, il2g - ktm = min(ktm,jt(i)) - kbm = min(kbm,mx(i)) - end do - - do k = ktm,pver-1 - do i = il1g,il2g - emc = -cu (i,k) & ! condensation in updraft - +evp(i,k) ! evaporating rain in downdraft - - dsdt(i,k) = -rl/cp*emc & - + (+mu(i,k+1)* (su(i,k+1)-shat(i,k+1)) & - -mu(i,k)* (su(i,k)-shat(i,k)) & - +md(i,k+1)* (sd(i,k+1)-shat(i,k+1)) & - -md(i,k)* (sd(i,k)-shat(i,k)) & - )/dp(i,k) - - dqdt(i,k) = emc + & - (+mu(i,k+1)* (qu(i,k+1)-qhat(i,k+1)) & - -mu(i,k)* (qu(i,k)-qhat(i,k)) & - +md(i,k+1)* (qd(i,k+1)-qhat(i,k+1)) & - -md(i,k)* (qd(i,k)-qhat(i,k)) & - )/dp(i,k) - - dl(i,k) = du(i,k)*ql(i,k+1) - - end do - end do - -! -!DIR$ NOINTERCHANGE! - do k = kbm,pver - do i = il1g,il2g - if (k == mx(i)) then - dsdt(i,k) = (1._r8/dsubcld(i))* & - (-mu(i,k)* (su(i,k)-shat(i,k)) & - -md(i,k)* (sd(i,k)-shat(i,k)) & - ) - dqdt(i,k) = (1._r8/dsubcld(i))* & - (-mu(i,k)*(qu(i,k)-qhat(i,k)) & - -md(i,k)*(qd(i,k)-qhat(i,k)) & - ) - else if (k > mx(i)) then - dsdt(i,k) = dsdt(i,k-1) - dqdt(i,k) = dqdt(i,k-1) - end if - end do - end do -! - return -end subroutine q1q2_pjr - -subroutine buoyan_dilute(lchnk ,ncol , & - q ,t ,p ,z ,pf , & - tp ,qstp ,tl ,rl ,cape , & - pblt ,lcl ,lel ,lon ,mx , & - rd ,grav ,cp ,msg , & - tpert ,iclosure) -!----------------------------------------------------------------------- -! -! Purpose: -! Calculates CAPE the lifting condensation level and the convective top -! where buoyancy is first -ve. -! -! Method: Calculates the parcel temperature based on a simple constant -! entraining plume model. CAPE is integrated from buoyancy. -! 09/09/04 - Simplest approach using an assumed entrainment rate for -! testing (dmpdp). -! 08/04/05 - Swap to convert dmpdz to dmpdp -! -! SCAM Logical Switches - DILUTE:RBN - Now Disabled -! --------------------- -! switch(1) = .T. - Uses the dilute parcel calculation to obtain tendencies. -! switch(2) = .T. - Includes entropy/q changes due to condensate loss and freezing. -! switch(3) = .T. - Adds the PBL Tpert for the parcel temperature at all levels. -! -! References: -! Raymond and Blythe (1992) JAS -! -! Author: -! Richard Neale - September 2004 -! -!----------------------------------------------------------------------- - implicit none -!----------------------------------------------------------------------- -! -! input arguments -! - integer, intent(in) :: lchnk ! chunk identifier - integer, intent(in) :: ncol ! number of atmospheric columns - - real(r8), intent(in) :: q(pcols,pver) ! spec. humidity - real(r8), intent(in) :: t(pcols,pver) ! temperature - real(r8), intent(in) :: p(pcols,pver) ! pressure - real(r8), intent(in) :: z(pcols,pver) ! height - real(r8), intent(in) :: pf(pcols,pver+1) ! pressure at interfaces - real(r8), intent(in) :: pblt(pcols) ! index of pbl depth - real(r8), intent(in) :: tpert(pcols) ! perturbation temperature by pbl processes - logical, intent(in) :: iclosure ! true for normal procedure, otherwise use dcapemx from 1st call -! -! output arguments -! - real(r8), intent(out) :: tp(pcols,pver) ! parcel temperature - real(r8), intent(out) :: qstp(pcols,pver) ! saturation mixing ratio of parcel (only above lcl, just q below). - real(r8), intent(out) :: tl(pcols) ! parcel temperature at lcl - real(r8), intent(out) :: cape(pcols) ! convective aval. pot. energy. - integer lcl(pcols) ! - integer lel(pcols) ! - integer lon(pcols) ! level of onset of deep convection - integer mx(pcols) ! level of max moist static energy -! -!--------------------------Local Variables------------------------------ -! - real(r8) capeten(pcols,num_cin) ! provisional value of cape - real(r8) tv(pcols,pver) ! - real(r8) tpv(pcols,pver) ! - real(r8) buoy(pcols,pver) - - real(r8) pl(pcols) - real(r8) hmax(pcols) - real(r8) hmn(pcols) - - logical plge600(pcols) - integer knt(pcols) - integer lelten(pcols,num_cin) - -! DCAPE-ULL - real(r8) pblt600(pcols) - - real(r8) cp - real(r8) grav - - integer i - integer k - integer msg - integer n - integer bot_layer - - real(r8) rd - real(r8) rl -#ifdef PERGRO - real(r8) rhd -#endif -! -!----------------------------------------------------------------------- -! - do n = 1,num_cin - do i = 1,ncol - lelten(i,n) = pver - capeten(i,n) = 0._r8 - end do - end do -! - do i = 1,ncol - lon(i) = pver - knt(i) = 0 - lel(i) = pver - mx(i) = lon(i) - cape(i) = 0._r8 - hmax(i) = 0._r8 - end do - - tp(:ncol,:) = t(:ncol,:) - qstp(:ncol,:) = q(:ncol,:) - -!DCAPE-ULL - if (trigdcape_ull) then - do k = pver - 1,msg + 1,-1 - do i = 1,ncol - if ((p(i,k).le.600._r8) .and. (p(i,k+1).gt.600._r8)) pblt600(i) = dble(k) - end do - end do - endif - -!!! RBN - Initialize tv and buoy for output. -!!! tv=tv : tpv=tpv : qstp=q : buoy=0. - tv(:ncol,:) = t(:ncol,:) *(1._r8+1.608_r8*q(:ncol,:))/ (1._r8+q(:ncol,:)) - tpv(:ncol,:) = tv(:ncol,:) - buoy(:ncol,:) = 0._r8 - -! -! set "launching" level(mx) to be at maximum moist static energy. -! search for this level stops at planetary boundary layer top. -! - bot_layer = pver - mx_bot_lyr_adj - -! DCAPE-ULL - if (trigdcape_ull .and. (.not. iclosure)) then - mx(:ncol) = dcapemx(:ncol) - else -#ifdef PERGRO - do k = bot_layer,msg + 1,-1 - do i = 1,ncol - hmn(i) = cp*t(i,k) + grav*z(i,k) + rl*q(i,k) -! -! Reset max moist static energy level when relative difference exceeds 1.e-4 -! - rhd = (hmn(i) - hmax(i))/(hmn(i) + hmax(i)) - - !DCAPE-ULL - if (trigdcape_ull) then - if (k >= nint(pblt600(i)) .and. k <= lon(i) .and. rhd > -1.e-4_r8) then - hmax(i) = hmn(i) - mx(i) = k - end if - else - if (k >= nint(pblt(i)) .and. k <= lon(i) .and. rhd > -1.e-4_r8) then - hmax(i) = hmn(i) - mx(i) = k - end if - end if - end do - end do -#else - do k = bot_layer,msg + 1,-1 - do i = 1,ncol - hmn(i) = cp*t(i,k) + grav*z(i,k) + rl*q(i,k) - - !DCAPE-ULL - if (trigdcape_ull) then - if (k >= nint(pblt600(i)) .and. k <= lon(i) .and. hmn(i) > hmax(i)) then - hmax(i) = hmn(i) - mx(i) = k - end if - else - if (k >= nint(pblt(i)) .and. k <= lon(i) .and. hmn(i) > hmax(i)) then - hmax(i) = hmn(i) - mx(i) = k - end if - end if - end do - end do -#endif - - end if - -! LCL dilute calculation - initialize to mx(i) -! Determine lcl in parcel_dilute and get pl,tl after parcel_dilute -! Original code actually sets LCL as level above wher condensate forms. -! Therefore in parcel_dilute lcl(i) will be at first level where qsmix < qtmix. - - do i = 1,ncol ! Initialise LCL variables. - lcl(i) = mx(i) - tl(i) = t(i,mx(i)) - pl(i) = p(i,mx(i)) - end do - -! -! main buoyancy calculation. -! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!! DILUTE PLUME CALCULATION USING ENTRAINING PLUME !!! -!!! RBN 9/9/04 !!! - - call parcel_dilute(lchnk, ncol, msg, mx, p, t, q, tpert, tp, tpv, qstp, pl, tl, lcl) - -! If lcl is above the nominal level of non-divergence (600 mbs), -! no deep convection is permitted (ensuing calculations -! skipped and cape retains initialized value of zero). -! - do i = 1,ncol - plge600(i) = pl(i).ge.600._r8 ! Just change to always allow buoy calculation. - end do - -! -! Main buoyancy calculation. -! - do k = pver,msg + 1,-1 - do i=1,ncol - if (k <= mx(i) .and. plge600(i)) then ! Define buoy from launch level to cloud top. - tv(i,k) = t(i,k)* (1._r8+1.608_r8*q(i,k))/ (1._r8+q(i,k)) - buoy(i,k) = tpv(i,k) - tv(i,k) + tiedke_add ! +0.5K or not? - else - qstp(i,k) = q(i,k) - tp(i,k) = t(i,k) - tpv(i,k) = tv(i,k) - endif - end do - end do - - - -!------------------------------------------------------------------------------- - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - - -! - do k = msg + 2,pver - do i = 1,ncol - if (k < lcl(i) .and. plge600(i)) then - if (buoy(i,k+1) > 0._r8 .and. buoy(i,k) <= 0._r8) then - knt(i) = min(num_cin,knt(i) + 1) - lelten(i,knt(i)) = k - end if - end if - end do - end do -! -! calculate convective available potential energy (cape). -! - do n = 1,num_cin - do k = msg + 1,pver - do i = 1,ncol - if (plge600(i) .and. k <= mx(i) .and. k > lelten(i,n)) then - capeten(i,n) = capeten(i,n) + rd*buoy(i,k)*log(pf(i,k+1)/pf(i,k)) - end if - end do - end do - end do -! -! find maximum cape from all possible tentative capes from -! one sounding, -! and use it as the final cape, april 26, 1995 -! - do n = 1,num_cin - do i = 1,ncol - if (capeten(i,n) > cape(i)) then - cape(i) = capeten(i,n) - lel(i) = lelten(i,n) - end if - end do - end do -! -! put lower bound on cape for diagnostic purposes. -! - do i = 1,ncol - cape(i) = max(cape(i), 0._r8) - end do -! - return -end subroutine buoyan_dilute - -subroutine parcel_dilute (lchnk, ncol, msg, klaunch, p, t, q, tpert, tp, tpv, qstp, pl, tl, lcl) -! Routine to determine -! 1. Tp - Parcel temperature -! 2. qstp - Saturated mixing ratio at the parcel temperature. - -!-------------------- -implicit none -!-------------------- - -integer, intent(in) :: lchnk -integer, intent(in) :: ncol -integer, intent(in) :: msg - -integer, intent(in), dimension(pcols) :: klaunch(pcols) - -real(r8), intent(in), dimension(pcols,pver) :: p -real(r8), intent(in), dimension(pcols,pver) :: t -real(r8), intent(in), dimension(pcols,pver) :: q -real(r8), intent(in), dimension(pcols) :: tpert ! PBL temperature perturbation. - -real(r8), intent(inout), dimension(pcols,pver) :: tp ! Parcel temp. -real(r8), intent(inout), dimension(pcols,pver) :: qstp ! Parcel water vapour (sat value above lcl). -real(r8), intent(inout), dimension(pcols) :: tl ! Actual temp of LCL. -real(r8), intent(inout), dimension(pcols) :: pl ! Actual pressure of LCL. - -integer, intent(inout), dimension(pcols) :: lcl ! Lifting condesation level (first model level with saturation). - -real(r8), intent(out), dimension(pcols,pver) :: tpv ! Define tpv within this routine. - -!-------------------- - -! Have to be careful as s is also dry static energy. - - -! If we are to retain the fact that CAM loops over grid-points in the internal -! loop then we need to dimension sp,atp,mp,xsh2o with ncol. - - -real(r8) tmix(pcols,pver) ! Tempertaure of the entraining parcel. -real(r8) qtmix(pcols,pver) ! Total water of the entraining parcel. -real(r8) qsmix(pcols,pver) ! Saturated mixing ratio at the tmix. -real(r8) smix(pcols,pver) ! Entropy of the entraining parcel. -real(r8) xsh2o(pcols,pver) ! Precipitate lost from parcel. -real(r8) ds_xsh2o(pcols,pver) ! Entropy change due to loss of condensate. -real(r8) ds_freeze(pcols,pver) ! Entropy change sue to freezing of precip. - -real(r8) mp(pcols) ! Parcel mass flux. -real(r8) qtp(pcols) ! Parcel total water. -real(r8) sp(pcols) ! Parcel entropy. - -real(r8) sp0(pcols) ! Parcel launch entropy. -real(r8) qtp0(pcols) ! Parcel launch total water. -real(r8) mp0(pcols) ! Parcel launch relative mass flux. - -real(r8) lwmax ! Maximum condesate that can be held in cloud before rainout. -real(r8) dmpdp ! Parcel fractional mass entrainment rate (/mb). -!real(r8) dmpdpc ! In cloud parcel mass entrainment rate (/mb). -!real(r8) dmpdz ! Parcel fractional mass entrainment rate (/m) -real(r8) dpdz,dzdp ! Hydrstatic relation and inverse of. -real(r8) senv ! Environmental entropy at each grid point. -real(r8) qtenv ! Environmental total water " " ". -real(r8) penv ! Environmental total pressure " " ". -real(r8) tenv ! Environmental total temperature " " ". -real(r8) new_s ! Hold value for entropy after condensation/freezing adjustments. -real(r8) new_q ! Hold value for total water after condensation/freezing adjustments. -real(r8) dp ! Layer thickness (center to center) -real(r8) tfguess ! First guess for entropy inversion - crucial for efficiency! -real(r8) tscool ! Super cooled temperature offset (in degC) (eg -35). - -real(r8) qxsk, qxskp1 ! LCL excess water (k, k+1) -real(r8) dsdp, dqtdp, dqxsdp ! LCL s, qt, p gradients (k, k+1) -real(r8) slcl,qtlcl,qslcl ! LCL s, qt, qs values. - -integer rcall ! Number of ientropy call for errors recording -integer nit_lheat ! Number of iterations for condensation/freezing loop. -integer i,k,ii ! Loop counters. - -!====================================================================== -! SUMMARY -! -! 9/9/04 - Assumes parcel is initiated from level of maxh (klaunch) -! and entrains at each level with a specified entrainment rate. -! -! 15/9/04 - Calculates lcl(i) based on k where qsmix is first < qtmix. -! -!====================================================================== -! -! Set some values that may be changed frequently. -! - -nit_lheat = 2 ! iterations for ds,dq changes from condensation freezing. - - -!dmpdpc = 3.e-2_r8 ! In cloud entrainment rate (/mb). -lwmax = 1.e-3_r8 ! Need to put formula in for this. -tscool = 0.0_r8 ! Temp at which water loading freezes in the cloud. - -qtmix=0._r8 -smix=0._r8 - -qtenv = 0._r8 -senv = 0._r8 -tenv = 0._r8 -penv = 0._r8 - -qtp0 = 0._r8 -sp0 = 0._r8 -mp0 = 0._r8 - -qtp = 0._r8 -sp = 0._r8 -mp = 0._r8 - -new_q = 0._r8 -new_s = 0._r8 - -! **** Begin loops **** - -do k = pver, msg+1, -1 - do i=1,ncol - -! Initialize parcel values at launch level. - - if (k == klaunch(i)) then - qtp0(i) = q(i,k) ! Parcel launch total water (assuming subsaturated) - OK????. - sp0(i) = entropy(t(i,k),p(i,k),qtp0(i)) ! Parcel launch entropy. - mp0(i) = 1._r8 ! Parcel launch relative mass (i.e. 1 parcel stays 1 parcel for dmpdp=0, undilute). - smix(i,k) = sp0(i) - qtmix(i,k) = qtp0(i) - tfguess = t(i,k) - rcall = 1 - call ientropy (rcall,i,lchnk,smix(i,k),p(i,k),qtmix(i,k),tmix(i,k),qsmix(i,k),tfguess) - end if - -! Entraining levels - - if (k < klaunch(i)) then - -! Set environmental values for this level. - - dp = (p(i,k)-p(i,k+1)) ! In -ve mb as p decreasing with height - difference between center of layers. - qtenv = 0.5_r8*(q(i,k)+q(i,k+1)) ! Total water of environment. - tenv = 0.5_r8*(t(i,k)+t(i,k+1)) - penv = 0.5_r8*(p(i,k)+p(i,k+1)) - - senv = entropy(tenv,penv,qtenv) ! Entropy of environment. - -! Determine fractional entrainment rate /pa given value /m. - - dpdz = -(penv*grav)/(rgas*tenv) ! in mb/m since p in mb. - dzdp = 1._r8/dpdz ! in m/mb - dmpdp = dmpdz*dzdp ! /mb Fractional entrainment - -! Sum entrainment to current level -! entrains q,s out of intervening dp layers, in which linear variation is assumed -! so really it entrains the mean of the 2 stored values. - - sp(i) = sp(i) - dmpdp*dp*senv - qtp(i) = qtp(i) - dmpdp*dp*qtenv - mp(i) = mp(i) - dmpdp*dp - -! Entrain s and qt to next level. - - smix(i,k) = (sp0(i) + sp(i)) / (mp0(i) + mp(i)) - qtmix(i,k) = (qtp0(i) + qtp(i)) / (mp0(i) + mp(i)) - -! Invert entropy from s and q to determine T and saturation-capped q of mixture. -! t(i,k) used as a first guess so that it converges faster. - - tfguess = tmix(i,k+1) - rcall = 2 - call ientropy(rcall,i,lchnk,smix(i,k),p(i,k),qtmix(i,k),tmix(i,k),qsmix(i,k),tfguess) - -! -! Determine if this is lcl of this column if qsmix <= qtmix. -! FIRST LEVEL where this happens on ascending. - - if (qsmix(i,k) <= qtmix(i,k) .and. qsmix(i,k+1) > qtmix(i,k+1)) then - lcl(i) = k - qxsk = qtmix(i,k) - qsmix(i,k) - qxskp1 = qtmix(i,k+1) - qsmix(i,k+1) - dqxsdp = (qxsk - qxskp1)/dp - pl(i) = p(i,k+1) - qxskp1/dqxsdp ! pressure level of actual lcl. - dsdp = (smix(i,k) - smix(i,k+1))/dp - dqtdp = (qtmix(i,k) - qtmix(i,k+1))/dp - slcl = smix(i,k+1) + dsdp* (pl(i)-p(i,k+1)) - qtlcl = qtmix(i,k+1) + dqtdp*(pl(i)-p(i,k+1)) - - tfguess = tmix(i,k) - rcall = 3 - call ientropy (rcall,i,lchnk,slcl,pl(i),qtlcl,tl(i),qslcl,tfguess) - -! write(iulog,*)' ' -! write(iulog,*)' p',p(i,k+1),pl(i),p(i,lcl(i)) -! write(iulog,*)' t',tmix(i,k+1),tl(i),tmix(i,lcl(i)) -! write(iulog,*)' s',smix(i,k+1),slcl,smix(i,lcl(i)) -! write(iulog,*)'qt',qtmix(i,k+1),qtlcl,qtmix(i,lcl(i)) -! write(iulog,*)'qs',qsmix(i,k+1),qslcl,qsmix(i,lcl(i)) - - endif -! - end if ! k < klaunch - - - end do ! Levels loop -end do ! Columns loop - -!!!!!!!!!!!!!!!!!!!!!!!!!!END ENTRAINMENT LOOP!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -!! Could stop now and test with this as it will provide some estimate of buoyancy -!! without the effects of freezing/condensation taken into account for tmix. - -!! So we now have a profile of entropy and total water of the entraining parcel -!! Varying with height from the launch level klaunch parcel=environment. To the -!! top allowed level for the existence of convection. - -!! Now we have to adjust these values such that the water held in vaopor is < or -!! = to qsmix. Therefore, we assume that the cloud holds a certain amount of -!! condensate (lwmax) and the rest is rained out (xsh2o). This, obviously -!! provides latent heating to the mixed parcel and so this has to be added back -!! to it. But does this also increase qsmix as well? Also freezing processes - - -xsh2o = 0._r8 -ds_xsh2o = 0._r8 -ds_freeze = 0._r8 - -!!!!!!!!!!!!!!!!!!!!!!!!!PRECIPITATION/FREEZING LOOP!!!!!!!!!!!!!!!!!!!!!!!!!! -!! Iterate solution twice for accuracy - - - -do k = pver, msg+1, -1 - do i=1,ncol - -! Initialize variables at k=klaunch - - if (k == klaunch(i)) then - -! Set parcel values at launch level assume no liquid water. - - tp(i,k) = tmix(i,k) - qstp(i,k) = q(i,k) - tpv(i,k) = (tp(i,k) + tp_fac*tpert(i)) * (1._r8+1.608_r8*qstp(i,k)) / (1._r8+qstp(i,k)) - - end if - - if (k < klaunch(i)) then - -! Initiaite loop if switch(2) = .T. - RBN:DILUTE - TAKEN OUT BUT COULD BE RETURNED LATER. - -! Iterate nit_lheat times for s,qt changes. - - do ii=0,nit_lheat-1 - -! Rain (xsh2o) is excess condensate, bar LWMAX (Accumulated loss from qtmix). - - xsh2o(i,k) = max (0._r8, qtmix(i,k) - qsmix(i,k) - lwmax) - -! Contribution to ds from precip loss of condensate (Accumulated change from smix).(-ve) - - ds_xsh2o(i,k) = ds_xsh2o(i,k+1) - cpliq * log (tmix(i,k)/tfreez) * max(0._r8,(xsh2o(i,k)-xsh2o(i,k+1))) -! -! Entropy of freezing: latice times amount of water involved divided by T. -! - - if (tmix(i,k) <= tfreez+tscool .and. ds_freeze(i,k+1) == 0._r8) then ! One off freezing of condensate. - ds_freeze(i,k) = (latice/tmix(i,k)) * max(0._r8,qtmix(i,k)-qsmix(i,k)-xsh2o(i,k)) ! Gain of LH - end if - - if (tmix(i,k) <= tfreez+tscool .and. ds_freeze(i,k+1) /= 0._r8) then ! Continual freezing of additional condensate. - ds_freeze(i,k) = ds_freeze(i,k+1)+(latice/tmix(i,k)) * max(0._r8,(qsmix(i,k+1)-qsmix(i,k))) - end if - -! Adjust entropy and accordingly to sum of ds (be careful of signs). - - new_s = smix(i,k) + ds_xsh2o(i,k) + ds_freeze(i,k) - -! Adjust liquid water and accordingly to xsh2o. - - new_q = qtmix(i,k) - xsh2o(i,k) - -! Invert entropy to get updated Tmix and qsmix of parcel. - - tfguess = tmix(i,k) - rcall =4 - call ientropy (rcall,i,lchnk,new_s, p(i,k), new_q, tmix(i,k), qsmix(i,k), tfguess) - - end do ! Iteration loop for freezing processes. - -! tp - Parcel temp is temp of mixture. -! tpv - Parcel v. temp should be density temp with new_q total water. - - tp(i,k) = tmix(i,k) - -! tpv = tprho in the presence of condensate (i.e. when new_q > qsmix) - - if (new_q > qsmix(i,k)) then ! Super-saturated so condensate present - reduces buoyancy. - qstp(i,k) = qsmix(i,k) - else ! Just saturated/sub-saturated - no condensate virtual effects. - qstp(i,k) = new_q - end if - - tpv(i,k) = (tp(i,k)+tp_fac*tpert(i))* (1._r8+1.608_r8*qstp(i,k)) / (1._r8+ new_q) - - end if ! k < klaunch - - end do ! Loop for columns - -end do ! Loop for vertical levels. - - -return -end subroutine parcel_dilute - -!----------------------------------------------------------------------------------------- -real(r8) function entropy(TK,p,qtot) -!----------------------------------------------------------------------------------------- -! -! TK(K),p(mb),qtot(kg/kg) -! from Raymond and Blyth 1992 -! - real(r8), intent(in) :: p,qtot,TK - real(r8) :: qv,qst,e,est,L - real(r8), parameter :: pref = 1000._r8 - -L = rl - (cpliq - cpwv)*(TK-tfreez) ! T IN CENTIGRADE - -call qsat_hPa(TK, p, est, qst) - -qv = min(qtot,qst) ! Partition qtot into vapor part only. -e = qv*p / (eps1 +qv) - -entropy = (cpres + qtot*cpliq)*log( TK/tfreez) - rgas*log( (p-e)/pref ) + & - L*qv/TK - qv*rh2o*log(qv/qst) - -end FUNCTION entropy - -! -!----------------------------------------------------------------------------------------- -SUBROUTINE ientropy (rcall,icol,lchnk,s,p,qt,T,qst,Tfg) -!----------------------------------------------------------------------------------------- -! -! p(mb), Tfg/T(K), qt/qv(kg/kg), s(J/kg). -! Inverts entropy, pressure and total water qt -! for T and saturated vapor mixing ratio -! - -! use phys_grid, only: get_rlon_p, get_rlat_p - - integer, intent(in) :: icol, lchnk, rcall - real(r8), intent(in) :: s, p, Tfg, qt - real(r8), intent(out) :: qst, T - real(r8) :: est - real(r8) :: a,b,c,d,ebr,fa,fb,fc,pbr,qbr,rbr,sbr,tol1,xm,tol - integer :: i - - logical :: converged - - ! Max number of iteration loops. - integer, parameter :: LOOPMAX = 100 - real(r8), parameter :: EPS = 3.e-8_r8 - - converged = .false. - - ! Invert the entropy equation -- use Brent's method - ! Brent, R. P. Ch. 3-4 in Algorithms for Minimization Without Derivatives. Englewood Cliffs, NJ: Prentice-Hall, 1973. - - T = Tfg ! Better first guess based on Tprofile from conv. - - a = Tfg-10 !low bracket - b = Tfg+10 !high bracket - - fa = entropy(a, p, qt) - s - fb = entropy(b, p, qt) - s - - c=b - fc=fb - tol=0.001_r8 - - converge: do i=0, LOOPMAX - if ((fb > 0.0_r8 .and. fc > 0.0_r8) .or. & - (fb < 0.0_r8 .and. fc < 0.0_r8)) then - c=a - fc=fa - d=b-a - ebr=d - end if - if (abs(fc) < abs(fb)) then - a=b - b=c - c=a - fa=fb - fb=fc - fc=fa - end if - - tol1=2.0_r8*EPS*abs(b)+0.5_r8*tol - xm=0.5_r8*(c-b) - converged = (abs(xm) <= tol1 .or. fb == 0.0_r8) - if (converged) exit converge - - if (abs(ebr) >= tol1 .and. abs(fa) > abs(fb)) then - sbr=fb/fa - if (a == c) then - pbr=2.0_r8*xm*sbr - qbr=1.0_r8-sbr - else - qbr=fa/fc - rbr=fb/fc - pbr=sbr*(2.0_r8*xm*qbr*(qbr-rbr)-(b-a)*(rbr-1.0_r8)) - qbr=(qbr-1.0_r8)*(rbr-1.0_r8)*(sbr-1.0_r8) - end if - if (pbr > 0.0_r8) qbr=-qbr - pbr=abs(pbr) - if (2.0_r8*pbr < min(3.0_r8*xm*qbr-abs(tol1*qbr),abs(ebr*qbr))) then - ebr=d - d=pbr/qbr - else - d=xm - ebr=d - end if - else - d=xm - ebr=d - end if - a=b - fa=fb - b=b+merge(d,sign(tol1,xm), abs(d) > tol1 ) - - fb = entropy(b, p, qt) - s - - end do converge - - T = b - call qsat_hPa(T, p, est, qst) - -! if (.not. converged) then -! this_lat = get_rlat_p(lchnk, icol)*57.296_r8 -! this_lon = get_rlon_p(lchnk, icol)*57.296_r8 -! write(*,*) '*** ZM_CONV: IENTROPY: Failed and about to exit, info follows ****' -! write(*,*) 'ZM_CONV: IENTROPY. Details: call#,lchnk,icol= ',rcall,lchnk,icol, & -! ' lat: ',this_lat,' lon: ',this_lon, & -! ' P(mb)= ', p, ' Tfg(K)= ', Tfg, ' qt(g/kg) = ', 1000._r8*qt, & -! ' qst(g/kg) = ', 1000._r8*qst,', s(J/kg) = ',s -! call endrun('**** ZM_CONV IENTROPY: Tmix did not converge ****') -! end if - -100 format (A,I1,I4,I4,7(A,F6.2)) - -end SUBROUTINE ientropy - -! Wrapper for qsat_water that does translation between Pa and hPa -! qsat_water uses Pa internally, so get it right, need to pass in Pa. -! Afterward, set es back to hPa. -subroutine qsat_hPa(t, p, es, qm) -! use wv_saturation, only: qsat_water - - ! Inputs - real(r8), intent(in) :: t ! Temperature (K) - real(r8), intent(in) :: p ! Pressure (hPa) - ! Outputs - real(r8), intent(out) :: es ! Saturation vapor pressure (hPa) - real(r8), intent(out) :: qm ! Saturation mass mixing ratio - ! (vapor mass over dry mass, kg/kg) - - call qsat(t, p*100._r8, es, qm, 0) - - es = es*0.01_r8 - -end subroutine qsat_hPa - -! AaronDonahue - Dummy function to handle whether or not this is the first -! step, until AD can provide this information directly. -function is_first_step() result(val) - logical :: val - val = is_first_step_local -end function is_first_step - -function is_first_restart_step() result(val) - logical :: val - val = .false. -end function is_first_restart_step -! AaronDonahue - Taken from cloud_fraction.F90 and adapted for local use. -! TODO, we should probably use the ice cloud fraction from macrophysics - subroutine cldfrc_fice(ncol, t, fice, fsnow) -! -! Compute the fraction of the total cloud water which is in ice phase. -! The fraction depends on temperature only. -! This is the form that was used for radiation, the code came from cldefr originally -! -! Author: B. A. Boville Sept 10, 2002 -! modified: PJR 3/13/03 (added fsnow to ascribe snow production for convection ) -! -! AaronDonahue - taken from original location as a stopgap to get ZM working -! locally. -!----------------------------------------------------------------------- - -! Arguments - integer, intent(in) :: ncol ! number of active columns - real(r8), intent(in) :: t(pcols,pver) ! temperature - - real(r8), intent(out) :: fice(pcols,pver) ! Fractional ice content within cloud - real(r8), intent(out) :: fsnow(pcols,pver) ! Fractional snow content for convection - -! Local variables - real(r8) :: tmax_fice ! max temperature for cloud ice formation - real(r8) :: tmin_fice ! min temperature for cloud ice formation - real(r8) :: tmax_fsnow ! max temperature for transition to convective snow - real(r8) :: tmin_fsnow ! min temperature for transition to convective snow - - integer :: i,k ! loop indexes - ! Top level - integer :: top_lev = 1 - -!----------------------------------------------------------------------- - - tmax_fice = tmelt - 10._r8 ! max temperature for cloud ice formation - tmin_fice = tmax_fice - 30._r8 ! min temperature for cloud ice formation - tmax_fsnow = tmelt ! max temperature for transition to convective snow - tmin_fsnow = tmelt - 5._r8 ! min temperature for transition to convective snow - - fice(:,:top_lev-1) = 0._r8 - fsnow(:,:top_lev-1) = 0._r8 - -! Define fractional amount of cloud that is ice - do k=top_lev,pver - do i=1,ncol - -! If warmer than tmax then water phase - if (t(i,k) > tmax_fice) then - fice(i,k) = 0.0_r8 - -! If colder than tmin then ice phase - else if (t(i,k) < tmin_fice) then - fice(i,k) = 1.0_r8 - -! Otherwise mixed phase, with ice fraction decreasing linearly from tmin to tmax - else - fice(i,k) =(tmax_fice - t(i,k)) / (tmax_fice - tmin_fice) - end if - -! snow fraction partitioning - -! If warmer than tmax then water phase - if (t(i,k) > tmax_fsnow) then - fsnow(i,k) = 0.0_r8 - -! If colder than tmin then ice phase - else if (t(i,k) < tmin_fsnow) then - fsnow(i,k) = 1.0_r8 - -! Otherwise mixed phase, with ice fraction decreasing linearly from tmin to tmax - else - fsnow(i,k) =(tmax_fsnow - t(i,k)) / (tmax_fsnow - tmin_fsnow) - end if - - end do - end do - - end subroutine cldfrc_fice -!--------------------------------------------------------------------- -! QSAT (SPECIFIC HUMIDITY) PROCEDURES -!--------------------------------------------------------------------- -! AaronDonahue - Taken from micro_p3.F90 to get ZM working locally. -! TODO - A unified qsat calculation that can be used by multiple physics -! processes? -!=========================================================================================== - - subroutine qsat(t_atm,p_atm,e_pres,qv_sat, i_wrt) - - !------------------------------------------------------------------------------------ - ! Calls polysvp1 to obtain the saturation vapor pressure, and then computes - ! and returns the saturation mixing ratio, with respect to either liquid or ice, - ! depending on value of 'i_wrt' - !------------------------------------------------------------------------------------ - - implicit none - - !Calling parameters: - real(r8), intent(IN) :: t_atm !temperature [K] - real(r8), intent(IN) :: p_atm !pressure [Pa] - real(r8), intent(OUT) :: e_pres !saturation vapor pressure [Pa] - real(r8), intent(OUT) :: qv_sat !saturation mixing ratio - integer, intent(IN) :: i_wrt !index, 0 = w.r.t. liquid, 1 = w.r.t. ice - - !Local variables: - real(r8) :: ep_2 - - !------------------ - ep_2 = epsilo - e_pres = polysvp1(t_atm,i_wrt) - qv_sat = ep_2*e_pres/max(1.e-3_r8,(p_atm-e_pres)) - - return - - end subroutine qsat -!==========================================================================================! - !_rtype - real(r8) function polysvp1(t,i_type) - - !------------------------------------------- - ! COMPUTE SATURATION VAPOR PRESSURE - ! POLYSVP1 RETURNED IN UNITS OF PA. - ! T IS INPUT IN UNITS OF K. - ! i_type REFERS TO SATURATION WITH RESPECT TO LIQUID (0) OR ICE (1) - !------------------------------------------- - - use scream_abortutils, only : endscreamrun - - implicit none - - real(r8), intent(in) :: t - integer, intent(in) :: i_type - - ! REPLACE GOFF-GRATCH WITH FASTER FORMULATION FROM FLATAU ET AL. 1992, TABLE 4 (RIGHT-HAND COLUMN) - - !local variables - character(len=1000) :: err_msg - - ! ice - real(r8) a0i,a1i,a2i,a3i,a4i,a5i,a6i,a7i,a8i - data a0i,a1i,a2i,a3i,a4i,a5i,a6i,a7i,a8i /& - 6.11147274_r8, 0.503160820_r8, 0.188439774e-1_r8, & - 0.420895665e-3_r8, 0.615021634e-5_r8, 0.602588177e-7_r8, & - 0.385852041e-9_r8, 0.146898966e-11_r8, 0.252751365e-14_r8/ - - ! liquid - real(r8) a0,a1,a2,a3,a4,a5,a6,a7,a8 - - ! V1.7 - data a0,a1,a2,a3,a4,a5,a6,a7,a8 /& - 6.11239921_r8, 0.443987641_r8, 0.142986287e-1_r8, & - 0.264847430e-3_r8, 0.302950461e-5_r8, 0.206739458e-7_r8, & - 0.640689451e-10_r8,-0.952447341e-13_r8,-0.976195544e-15_r8/ - real(r8) dt - real(r8) zerodegc - !------------------------------------------- - zerodegc = tmelt - if (i_type.eq.1 .and. t.lt.zerodegc) then - ! ICE - - ! Flatau formulation: - dt = max(-80._r8,t-273.15_r8) - polysvp1 = a0i + dt*(a1i+dt*(a2i+dt*(a3i+dt*(a4i+dt*(a5i+dt*(a6i+dt*(a7i+ & - a8i*dt))))))) - polysvp1 = polysvp1*100._r8 - - ! Goff-Gratch formulation: - ! POLYSVP1 = 10.**(-9.09718*(273.16/T-1.)-3.56654* & - ! log10(273.16/T)+0.876793*(1.-T/273.16)+ & - ! log10(6.1071))*100. - - - elseif (i_type.eq.0 .or. t.ge.zerodegc) then - ! LIQUID - - ! Flatau formulation: - dt = max(-80._r8,t-273.15_r8) - polysvp1 = a0 + dt*(a1+dt*(a2+dt*(a3+dt*(a4+dt*(a5+dt*(a6+dt*(a7+a8*dt))))))) - polysvp1 = polysvp1*100._r8 - - ! Goff-Gratch formulation: - ! POLYSVP1 = 10.**(-7.90298*(373.16/T-1.)+ & - ! 5.02808*log10(373.16/T)- & - ! 1.3816E-7*(10**(11.344*(1.-T/373.16))-1.)+ & - ! 8.1328E-3*(10**(-3.49149*(373.16/T-1.))-1.)+ & - ! log10(1013.246))*100. - - !PMC added error checking - else - - write(err_msg,*)'** polysvp1 i_type must be 0 or 1 but is: ', & - i_type,' temperature is:',t,' in file: ',__FILE__,' at line:',__LINE__ - - call endscreamrun(err_msg) - endif - - return - - end function polysvp1 -! Below is what was originally used by ZM and is taken from wv_saturation -!elemental subroutine qsat(t, p, es, qs, gam, dqsdt, enthalpy) -! !------------------------------------------------------------------! -! ! Purpose: ! -! ! Look up and return saturation vapor pressure from precomputed ! -! ! table, then calculate and return saturation specific humidity. ! -! ! Optionally return various temperature derivatives or enthalpy ! -! ! at saturation. ! -! !------------------------------------------------------------------! -! -! ! Inputs -! real(r8), intent(in) :: t ! Temperature -! real(r8), intent(in) :: p ! Pressure -! ! Outputs -! real(r8), intent(out) :: es ! Saturation vapor pressure -! real(r8), intent(out) :: qs ! Saturation specific humidity -! -! real(r8), intent(out), optional :: gam ! (l/cpair)*(d(qs)/dt) -! real(r8), intent(out), optional :: dqsdt ! (d(qs)/dt) -! real(r8), intent(out), optional :: enthalpy ! cpair*t + hltalt*q -! -! ! Local variables -! real(r8) :: hltalt ! Modified latent heat for T derivatives -! real(r8) :: tterm ! Account for d(es)/dT in transition region -! -! es = estblf(t) -! -! qs = svp_to_qsat(es, p) -! -! ! Ensures returned es is consistent with limiters on qs. -! es = min(es, p) -! -! ! Calculate optional arguments. -! if (present(gam) .or. present(dqsdt) .or. present(enthalpy)) then -! -! ! "generalized" analytic expression for t derivative of es -! ! accurate to within 1 percent for 173.16 < t < 373.16 -! call calc_hltalt(t, hltalt, tterm) -! -! if (present(enthalpy)) enthalpy = tq_enthalpy(t, qs, hltalt) -! -! call deriv_outputs(t, p, es, qs, hltalt, tterm, & -! gam=gam, dqsdt=dqsdt) -! -! end if -! -!end subroutine qsat -! -!elemental subroutine qsat_water(t, p, es, qs, gam, dqsdt, enthalpy) -! !------------------------------------------------------------------! -! ! Purpose: ! -! ! Calculate SVP over water at a given temperature, and then ! -! ! calculate and return saturation specific humidity. ! -! ! Optionally return various temperature derivatives or enthalpy ! -! ! at saturation. ! -! !------------------------------------------------------------------! -! -!! use wv_sat_methods, only: wv_sat_qsat_water -! -! ! Inputs -! real(r8), intent(in) :: t ! Temperature -! real(r8), intent(in) :: p ! Pressure -! ! Outputs -! real(r8), intent(out) :: es ! Saturation vapor pressure -! real(r8), intent(out) :: qs ! Saturation specific humidity -! -! real(r8), intent(out), optional :: gam ! (l/cpair)*(d(qs)/dt) -! real(r8), intent(out), optional :: dqsdt ! (d(qs)/dt) -! real(r8), intent(out), optional :: enthalpy ! cpair*t + hltalt*q -! -! ! Local variables -! real(r8) :: hltalt ! Modified latent heat for T derivatives -! -! call wv_sat_qsat_water(t, p, es, qs) -! -! if (present(gam) .or. present(dqsdt) .or. present(enthalpy)) then -! -! ! "generalized" analytic expression for t derivative of es -! ! accurate to within 1 percent for 173.16 < t < 373.16 -! call no_ip_hltalt(t, hltalt) -! -! if (present(enthalpy)) enthalpy = tq_enthalpy(t, qs, hltalt) -! -! ! For pure water/ice transition term is 0. -! call deriv_outputs(t, p, es, qs, hltalt, 0._r8, & -! gam=gam, dqsdt=dqsdt) -! -! end if -! -!end subroutine qsat_water -! -!elemental subroutine wv_sat_qsat_water(t, p, es, qs, idx) -! !------------------------------------------------------------------! -! ! Purpose: ! -! ! Calculate SVP over water at a given temperature, and then ! -! ! calculate and return saturation specific humidity. ! -! !------------------------------------------------------------------! -! -! ! Inputs -! real(r8), intent(in) :: t ! Temperature -! real(r8), intent(in) :: p ! Pressure -! ! Outputs -! real(r8), intent(out) :: es ! Saturation vapor pressure -! real(r8), intent(out) :: qs ! Saturation specific humidity -! -! integer, intent(in), optional :: idx ! Scheme index -! -! es = wv_sat_svp_water(t, idx) -! -! qs = wv_sat_svp_to_qsat(es, p) -! -! ! Ensures returned es is consistent with limiters on qs. -! es = min(es, p) -! -!end subroutine wv_sat_qsat_water -!--------------------------------------------------------------------- - -end module zm_conv From 0a967e006df9a386ae039fdaaccffa158c763bed Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Thu, 30 May 2024 09:40:50 -0700 Subject: [PATCH 373/476] Add documentation for COSP in EAMxx --- components/eamxx/docs/user/cosp.md | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 components/eamxx/docs/user/cosp.md diff --git a/components/eamxx/docs/user/cosp.md b/components/eamxx/docs/user/cosp.md new file mode 100644 index 000000000000..aff778ce1fff --- /dev/null +++ b/components/eamxx/docs/user/cosp.md @@ -0,0 +1,47 @@ +# CFMIP Observation Simulator Package (COSP) in EAMxx +[COSP](https://github.com/CFMIP/COSPv2.0) is partially implemented and supported in EAMxx. Currently, minimal outputs from the ISCCP, MODIS, and MISR simulators have been enabled. + +## Running with COSP +Turning COSP on simply requires adding the `cosp` process to `atm_procs_list` via `atmchange` in a case directory: +``` +./atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,cosp" +``` +Additionally, the frequency at which COSP is run can be configured via `atmchange`: +``` +./atmchange physics::cosp::cosp_frequency_units="steps" +./atmchange physics::cosp::cosp_frequency=1 +``` +Output streams need to be added manually. A minimal example: +``` +./atmchange output_yaml_files=scream_daily_output.yaml +cat << EOF > scream_cosp_daily_output.yaml +Averaging Type: Average +Fields: + Physics PG2: + Field Names: + - isccp_cldtot + - isccp_ctptau + - modis_ctptau + - misr_cthtau + - cosp_sunlit +Max Snapshots Per File: 1 +filename_prefix: eamxx +output_control: + Frequency: 1 + frequency_units: ndays +EOF +``` + +## Available output fields +The following output fields are available: + + - isccp_cldtot (total cloud area from ISCCP simulator) + - isccp_ctptau (ISCCP-simulated cloud top pressure/optical depth joint histogram) + - modis_ctptau (MODIS-simulated cloud top pressure/optical depth joint histogram) + - misr_cthtau (MISR-simulated cloud top height/optical depth joint histogram) + - cosp_sunlit (sunlit flag aggregated at COSP frequency for renormalizing daytime averages) + +ISCCP, MODIS, and MISR outputs are valid only for daytime/sunlit columns (to be consistent with available satellite retrievals). In order to aggregate only daytime columns in time averages, these outputs are multiplied by the sunlit flag (0 or 1) at each COSP calculation time. Time averages of these quantities are then aggregated, along with the cosp sunlit flag each time COSP is called. In order to back out the daytime-only time averages from the outputs, one needs to divide the output fields by `cosp_sunlit`. E.g., +``` +isccp_ctptau = mean(isccp_ctptau) / mean(cosp_sunlit) +``` From e5598fdcbe1ae812cf0207fc24e5c67bad581701 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Thu, 30 May 2024 09:49:35 -0700 Subject: [PATCH 374/476] Add documentation about cosp_subcolumns --- components/eamxx/docs/user/cosp.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/components/eamxx/docs/user/cosp.md b/components/eamxx/docs/user/cosp.md index aff778ce1fff..9807e10a2e15 100644 --- a/components/eamxx/docs/user/cosp.md +++ b/components/eamxx/docs/user/cosp.md @@ -11,6 +11,17 @@ Additionally, the frequency at which COSP is run can be configured via `atmchang ./atmchange physics::cosp::cosp_frequency_units="steps" ./atmchange physics::cosp::cosp_frequency=1 ``` + +COSP can be run with or without subcolumn sampling. This is configured by changing the `cosp_subcolumns` namelist variable via `atmchange`. A value of 1 implies *no* subcolumn sampling, while values greater than 1 specify the number of subcolumns to use for subcolumn sampling (assuming maximum-random overlap). E.g., +``` +./atmchange physics::cosp:cosp_subcolumns=1 +``` +would disable subcolumn sampling, while +``` +./atmchange physics::cosp::cosp_subcolumns=10 +``` +would use 10 subcolumns for the COSP internal subcolumn sampling using `SCOPS`/`PREC_SCOPS`. The default for high resolution cases (e.g., ne1024) should be to *not* use subcolumns, while lower resolutions (e.g., ne30) should enable subcolumn sampling. + Output streams need to be added manually. A minimal example: ``` ./atmchange output_yaml_files=scream_daily_output.yaml From 98f37c1f6e4f3d227f0bd6db75c3506e3f34ec8e Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Thu, 30 May 2024 10:38:24 -0700 Subject: [PATCH 375/476] Add COSP to side panel --- components/eamxx/mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/mkdocs.yml b/components/eamxx/mkdocs.yml index bbf0bc6f0876..0ed6cfef9184 100644 --- a/components/eamxx/mkdocs.yml +++ b/components/eamxx/mkdocs.yml @@ -9,6 +9,7 @@ nav: - 'Model output': 'user/model_output.md' - 'Nudging': 'user/nudging.md' - 'Extra radiation calls': 'user/clean_clear_sky.md' + - 'COSP': 'user/cosp.md' - 'Developer Guide': - 'Overview': 'developer/index.md' - 'Installation': 'common/installation.md' From b065a7f196fba713c0b43052323c22322b6699e5 Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Thu, 30 May 2024 12:35:40 -0600 Subject: [PATCH 376/476] Fix grid def for cosp standalone --- components/eamxx/tests/single-process/cosp/input.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/eamxx/tests/single-process/cosp/input.yaml b/components/eamxx/tests/single-process/cosp/input.yaml index e3a80db73f09..5b3b2787943d 100644 --- a/components/eamxx/tests/single-process/cosp/input.yaml +++ b/components/eamxx/tests/single-process/cosp/input.yaml @@ -14,18 +14,17 @@ atmosphere_processes: grids_manager: Type: Mesh Free geo_data_source: IC_FILE - grids_names: [Physics] - Physics: - aliases: [Point Grid] + grids_names: [Physics GLL] + Physics GLL: type: point_grid + aliases: [Physics] number_of_global_columns: 218 number_of_vertical_levels: 72 - initial_conditions: # The name of the file containing the initial conditions for this test. Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} - topography_filename: ${TOPO_DATA_DIR}/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} dtau067: 1.0 dtau105: 1.0 cldfrac_rad: 0.5 From 844311b1451fc4300ce37364087f9d67d9a926d0 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 30 May 2024 15:46:03 -0600 Subject: [PATCH 377/476] Move IOP from control/ to share/iop/ --- components/eamxx/src/control/CMakeLists.txt | 2 -- components/eamxx/src/control/atmosphere_driver.hpp | 2 +- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 2 +- components/eamxx/src/physics/spa/spa_functions.hpp | 2 +- components/eamxx/src/physics/spa/tests/CMakeLists.txt | 4 ++-- components/eamxx/src/share/CMakeLists.txt | 1 + .../eamxx/src/share/atm_process/atmosphere_process.hpp | 3 +-- .../iop}/intensive_observation_period.cpp | 7 ++----- .../iop}/intensive_observation_period.hpp | 0 9 files changed, 9 insertions(+), 14 deletions(-) rename components/eamxx/src/{control => share/iop}/intensive_observation_period.cpp (99%) rename components/eamxx/src/{control => share/iop}/intensive_observation_period.hpp (100%) diff --git a/components/eamxx/src/control/CMakeLists.txt b/components/eamxx/src/control/CMakeLists.txt index 19ce3b6dc318..050f0d1e959e 100644 --- a/components/eamxx/src/control/CMakeLists.txt +++ b/components/eamxx/src/control/CMakeLists.txt @@ -2,14 +2,12 @@ set(SCREAM_CONTROL_SOURCES atmosphere_driver.cpp atmosphere_surface_coupling_importer.cpp atmosphere_surface_coupling_exporter.cpp - intensive_observation_period.cpp surface_coupling_utils.cpp ) set(SCREAM_CONTROL_HEADERS atmosphere_driver.hpp atmosphere_surface_coupling.hpp - intensive_observation_period.hpp surface_coupling_utils.hpp ) diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 1cd8d4db08e7..d43d233f4e97 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -2,7 +2,7 @@ #define SCREAM_ATMOSPHERE_DRIVER_HPP #include "control/surface_coupling_utils.hpp" -#include "control/intensive_observation_period.hpp" +#include "share/iop/intensive_observation_period.hpp" #include "share/field/field_manager.hpp" #include "share/grid/grids_manager.hpp" #include "share/util/scream_time_stamp.hpp" diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 9e04b4a00920..66f73b5ac9a8 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -1,10 +1,10 @@ #include "eamxx_homme_process_interface.hpp" // EAMxx includes -#include "control/intensive_observation_period.hpp" #include "dynamics/homme/homme_dimensions.hpp" #include "dynamics/homme/homme_dynamics_helpers.hpp" #include "physics/share/physics_constants.hpp" +#include "share/iop/intensive_observation_period.hpp" #include "share/util/scream_column_ops.hpp" // Homme includes diff --git a/components/eamxx/src/physics/spa/spa_functions.hpp b/components/eamxx/src/physics/spa/spa_functions.hpp index 631aae9eee78..6bdba702e70f 100644 --- a/components/eamxx/src/physics/spa/spa_functions.hpp +++ b/components/eamxx/src/physics/spa/spa_functions.hpp @@ -1,10 +1,10 @@ #ifndef SPA_FUNCTIONS_HPP #define SPA_FUNCTIONS_HPP -#include "control/intensive_observation_period.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" +#include "share/iop/intensive_observation_period.hpp" #include "share/util/scream_time_stamp.hpp" #include "share/scream_types.hpp" diff --git a/components/eamxx/src/physics/spa/tests/CMakeLists.txt b/components/eamxx/src/physics/spa/tests/CMakeLists.txt index 5abbab29d4c4..effe97702505 100644 --- a/components/eamxx/src/physics/spa/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/spa/tests/CMakeLists.txt @@ -1,12 +1,12 @@ INCLUDE (ScreamUtils) CreateUnitTest(spa_read_data_test "spa_read_data_from_file_test.cpp" - LIBS spa scream_io scream_control + LIBS spa scream_io LABELS spa MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) CreateUnitTest(spa_one_to_one_remap_test "spa_one_to_one_remap_test.cpp" - LIBS spa scream_io scream_control + LIBS spa scream_io LABELS spa MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index 701cf89bb03f..e83f9d98ab9f 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -27,6 +27,7 @@ set(SHARE_SRC grid/remap/horiz_interp_remapper_data.cpp grid/remap/refining_remapper_p2p.cpp grid/remap/vertical_remapper.cpp + iop/intensive_observation_period.cpp property_checks/property_check.cpp property_checks/field_nan_check.cpp property_checks/field_within_interval_check.cpp diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.hpp b/components/eamxx/src/share/atm_process/atmosphere_process.hpp index 071ef906a751..a25879b80218 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.hpp @@ -1,8 +1,7 @@ #ifndef SCREAM_ATMOSPHERE_PROCESS_HPP #define SCREAM_ATMOSPHERE_PROCESS_HPP -#include "control/intensive_observation_period.hpp" - +#include "share/iop/intensive_observation_period.hpp" #include "share/atm_process/atmosphere_process_utils.hpp" #include "share/atm_process/ATMBufferManager.hpp" #include "share/atm_process/SCDataManager.hpp" diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp similarity index 99% rename from components/eamxx/src/control/intensive_observation_period.cpp rename to components/eamxx/src/share/iop/intensive_observation_period.cpp index ab01bfc7e3ed..061e7022649b 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -1,14 +1,11 @@ -#include "control/intensive_observation_period.hpp" - #include "share/grid/point_grid.hpp" #include "share/io/scorpio_input.hpp" +#include "share/iop/intensive_observation_period.hpp" #include "share/util/scream_vertical_interpolation.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_lin_interp.hpp" -#include "pio.h" - #include // Extend ekat mpi type for pairs, @@ -250,7 +247,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, if (scorpio::has_dim(iop_file, "time")) time_dimname = "time"; else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); - + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); diff --git a/components/eamxx/src/control/intensive_observation_period.hpp b/components/eamxx/src/share/iop/intensive_observation_period.hpp similarity index 100% rename from components/eamxx/src/control/intensive_observation_period.hpp rename to components/eamxx/src/share/iop/intensive_observation_period.hpp From 2e0ebfbc0a80fd63ef957aa006e63e752f07411c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 30 May 2024 16:22:23 -0700 Subject: [PATCH 378/476] Fixes np1_vs_npX standalone tests and adds mam4xx vars to output --- .../eamxx/cmake/machine-files/compy.cmake | 2 +- .../physics/mam/eamxx_mam_aci_functions.hpp | 1 + .../mam/eamxx_mam_aci_process_interface.cpp | 14 +++-- .../mam/shoc_cldfrac_mam4_aci_p3/input.yaml | 2 +- .../mam/shoc_cldfrac_mam4_aci_p3/output.yaml | 55 ++++++++++++++++- .../output.yaml | 55 ++++++++++++++++- .../output.yaml | 55 ++++++++++++++++- .../mam/shoc_mam4_aci/output.yaml | 55 ++++++++++++++++- .../single-process/mam/aci/CMakeLists.txt | 13 ---- .../tests/single-process/mam/aci/output.yaml | 59 +++++++++++++++++-- 10 files changed, 283 insertions(+), 28 deletions(-) diff --git a/components/eamxx/cmake/machine-files/compy.cmake b/components/eamxx/cmake/machine-files/compy.cmake index 1f156b0546c0..e8404d75559b 100644 --- a/components/eamxx/cmake/machine-files/compy.cmake +++ b/components/eamxx/cmake/machine-files/compy.cmake @@ -7,4 +7,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) #Compy SLURM specific settings -set(EKAT_MPI_NP_FLAG "-p short -n" CACHE STRING "" FORCE) +set(EKAT_MPI_NP_FLAG "-p short --mpi=pmi2 -n" CACHE STRING "" FORCE) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 87018c929527..e5368f7a360c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -547,6 +547,7 @@ void update_cloud_borne_aerosols( } // Update interstitial aerosols using tendencies - levels +KOKKOS_INLINE_FUNCTION void update_interstitial_aerosols_levs( const haero::ThreadTeam &team, const int nlev, const int icol, const int s_idx, const Real dt, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 4dcc169daa81..da50a2282c46 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -21,7 +21,11 @@ FUTURE WORK: 1. MAM4xx submodule should point to MAM4xx main branch 2. Link hetrozenous freezing outputs to microphysics 3. Add postcondition and invariant checks -4. Add variables in output.yaml files of the standalone tests +5. Resolve comments about top_lev +6. Avoid using c-style static arrays in ptend and other arrays +7. Use std::string rather than c-strings +8. Remove a Kokkos:fence and combine two kernels while computing w_sec_int_ +9. Fix double counting of tracer advection by modifying SHOC. ----------------------------------------------------------------- */ @@ -145,7 +149,7 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "nmodes", "lev"}); // dry diameter of aerosols [m] add_field("dgnum", scalar4d_layout_mid, m, grid_name); @@ -564,17 +568,16 @@ void MAMAci::run_impl(const double dt) { compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, // output w_sec_int_); - + Kokkos::fence(); // wait for w_sec_int_ to be computed. compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, // output tke_); - Kokkos::fence(); // wait for for tke_ to be computed. + Kokkos::fence(); // wait for tke_ to be computed. compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, // output wsub_, wsubice_, wsig_); - Kokkos::fence(); // wait for wsig_ to be computed. // We need dry diameter for only aitken mode compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, @@ -611,6 +614,7 @@ void MAMAci::run_impl(const double dt) { // output kvh_int_); + Kokkos::fence(); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml index a8cac4b1a6c7..9c0ed4d97988 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/input.yaml @@ -48,7 +48,7 @@ grids_manager: initial_conditions: # The name of the file containing the initial conditions for this test. - Filename: /qfs/people/sing201/delete/screami_unit_tests_mam4xx_ne2np4L72_LAT_71p9201421331276_LON_286p572525837053.nc + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} #variable required for shoc diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml index f4aec357e9f6..2541ce8d9f8f 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_cldfrac_mam4_aci_p3_output Averaging Type: Instant Field Names: # SHOC @@ -38,6 +38,59 @@ Field Names: - qv # SHOC + P3 + RRTMGP - T_mid + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml index a8b22c16ba09..392e159f0a55 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp_output Averaging Type: Instant Field Names: # SHOC @@ -51,6 +51,59 @@ Field Names: - rad_heating_pdel - sfc_flux_lw_dn - sfc_flux_sw_net + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml index a8b22c16ba09..4c9f5ff5d8d1 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_cldfrac_mam4_aci_p3_rrtmgp_output Averaging Type: Instant Field Names: # SHOC @@ -51,6 +51,59 @@ Field Names: - rad_heating_pdel - sfc_flux_lw_dn - sfc_flux_sw_net + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml index 590cd86ab7ea..34626c996438 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_cld_p3_rrtmgp_output +filename_prefix: shoc_mam4_aci_output Averaging Type: Instant Field Names: # SHOC @@ -11,6 +11,59 @@ Field Names: - tke - inv_qc_relvar - pbl_height + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 25c83e362edc..0d5d60ee7d5a 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -36,19 +36,6 @@ CompareNCFilesFamilyMpi ( META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) -# Check tendency calculation -foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) - set (script ${SCREAM_BASE_DIR}/scripts/check-tendencies) - set (fname mam4_aci_standalone_output.INSTANT.nsteps_x1.np${NRANKS}.${RUN_T0}.nc) - set (tname mam4_aci_tend_check_np${NRANKS}) - add_test (NAME ${tname} - COMMAND ${script} -f ${fname} -v qc T_mid -t mam_aci_qc_tend mam_aci_T_mid_tend - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties (${tname} PROPERTIES - LABELS "mam4_aci;physics" - FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) -endforeach() - if (SCREAM_ENABLE_BASELINE_TESTS) # Compare one of the output files with the baselines. # Note: one is enough, since we already check that np1 is BFB with npX diff --git a/components/eamxx/tests/single-process/mam/aci/output.yaml b/components/eamxx/tests/single-process/mam/aci/output.yaml index 1591d5fe3313..18f64b077cf4 100644 --- a/components/eamxx/tests/single-process/mam/aci/output.yaml +++ b/components/eamxx/tests/single-process/mam/aci/output.yaml @@ -5,10 +5,61 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid - - dgnum - + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - eddy_diff_heat + - nc_nuceat_tend + - ni_activated + output_control: - Frequency: 2 + Frequency: 1 frequency_units: nsteps ... From a2afdfcb95838b1290a28fccb9edc1d0a7126a0d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 30 May 2024 17:29:42 -0700 Subject: [PATCH 379/476] Adds some TODO comments and minor cleanup --- .../eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp | 1 - .../physics/mam/eamxx_mam_aci_process_interface.cpp | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index e5368f7a360c..96e31f4d6c79 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -529,7 +529,6 @@ void call_function_dropmixnuc( // Update cloud borne aerosols void update_cloud_borne_aerosols( - haero::ThreadTeamPolicy team_policy, const MAMAci::view_2d qqcw_fld_work[mam4::ndrop::ncnst_tot], const int nlev, // output mam_coupling::AerosolState &dry_aero) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index da50a2282c46..063535ccdab9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -26,6 +26,14 @@ FUTURE WORK: 7. Use std::string rather than c-strings 8. Remove a Kokkos:fence and combine two kernels while computing w_sec_int_ 9. Fix double counting of tracer advection by modifying SHOC. +10. A git issue for computing top_lev, moving liq_cldfrac in ACI and using TKE directly +11.Replace [=] with [&] +12. Remove all set_string and use pow +13. use "aitken_dry_dia = ekat::subview_1(dgnum,mam4::ModeIndex::Aitken);" +14. Merge kernels so that we are not calling one from another +15. Use "TeamVectorRange" instead of thread +16. improve the way qqcw is populated +17.delete fence mentioned by Luca ----------------------------------------------------------------- */ @@ -663,7 +671,7 @@ void MAMAci::run_impl(const double dt) { //--------------------------------------------------------------- // Update cloud borne aerosols - update_cloud_borne_aerosols(team_policy, qqcw_fld_work_, nlev_, + update_cloud_borne_aerosols(qqcw_fld_work_, nlev_, // output dry_aero_); From 6d1cd4dcfa172ef5e2e1bbb79e268c9362d2d147 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 17:05:48 -0600 Subject: [PATCH 380/476] EAMxx: prototype of pyscream basics, via pybind11 --- components/eamxx/src/share/CMakeLists.txt | 4 + .../eamxx/src/share/pyshare/CMakeLists.txt | 6 ++ .../eamxx/src/share/pyshare/pyscream.cpp | 91 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 components/eamxx/src/share/pyshare/CMakeLists.txt create mode 100644 components/eamxx/src/share/pyshare/pyscream.cpp diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index e83f9d98ab9f..6a5620fde5a1 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -104,6 +104,10 @@ endif() # IO library add_subdirectory(io) +if (EAMXX_ENABLE_PYBIND) + add_subdirectory(pyshare) +endif() + if (NOT SCREAM_LIB_ONLY) # Create test_support lib add_library(scream_test_support diff --git a/components/eamxx/src/share/pyshare/CMakeLists.txt b/components/eamxx/src/share/pyshare/CMakeLists.txt new file mode 100644 index 000000000000..1f8f74f7a9ee --- /dev/null +++ b/components/eamxx/src/share/pyshare/CMakeLists.txt @@ -0,0 +1,6 @@ +set (pybind11_DIR /home/lbertag/.conda/envs/bartgol/lib/python3.12/site-packages) +find_package(pybind11 REQUIRED + HINTS ${pybind11_DIR}) + +pybind11_add_module(pyscream pyscream.cpp) +target_link_libraries(pyscream PUBLIC scream_share scream_io) diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/pyshare/pyscream.cpp new file mode 100644 index 000000000000..41286bbe80ad --- /dev/null +++ b/components/eamxx/src/share/pyshare/pyscream.cpp @@ -0,0 +1,91 @@ +#include "scream_session.hpp" +#include "share/field/field.hpp" +#include "share/field/field_utils.hpp" + +#include +#include +#include + +using namespace scream; +using namespace ShortFieldTagsNames; +namespace py = pybind11; + +void initialize () { + initialize_scream_session(false); +} +void finalize () { + finalize_scream_session(); +} + +struct PyField { + Field f; + + // Create field and allocate memory + PyField(const std::string& n, + const std::vector& d) + { + std::vector t (d.size(),CMP); + FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); + f = Field(fid); + f.allocate_view(); + } + + // Create field from pre-existing python nd array + PyField(const std::string& n, + py::array_t arr) + { + int rank = arr.ndim(); + std::vector d(rank,-1); + for (int n=0; n t (rank,CMP); + FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); + switch (rank) { + case 1: + { + Field::view_dev_t v(arr.mutable_data(0),d[0]); + f = Field(fid,v); + break; + } + case 2: + { + Field::view_dev_t v(arr.mutable_data(0,0),d[0],d[1]); + f = Field(fid,v); + break; + } + default: + EKAT_ERROR_MSG ("AAARGH!\n"); + } + } + + void print() const { + print_field_hyperslab(f); + } +}; + +PYBIND11_MODULE (pyscream,m) { + + m.doc() = "Basic interface to scream session initialization"; + + m.def("init",&initialize); + m.def("finalize",&finalize); + + py::enum_(m,"Tag") + .value("INV" ,INV) + .value("EL" ,EL) + .value("COL" ,COL) + .value("GP" ,GP) + .value("TL" ,TL) + .value("LEV" ,LEV) + .value("ILEV",ILEV) + .value("CMP" ,CMP); + + py::class_(m,"Field") + .def(py::init&>()) + .def(py::init + >()) + .def("print",&PyField::print); +} From 8227230491f11eefdc1ea164e67e2dc494e90bd7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 17:49:48 -0600 Subject: [PATCH 381/476] EAMxx: remove pointless stuff from pyscream --- components/eamxx/src/share/pyshare/pyscream.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/pyshare/pyscream.cpp index 41286bbe80ad..b519c3483834 100644 --- a/components/eamxx/src/share/pyshare/pyscream.cpp +++ b/components/eamxx/src/share/pyshare/pyscream.cpp @@ -62,6 +62,10 @@ struct PyField { void print() const { print_field_hyperslab(f); } + + void cleanup () { + f = Field(); + } }; PYBIND11_MODULE (pyscream,m) { @@ -71,21 +75,12 @@ PYBIND11_MODULE (pyscream,m) { m.def("init",&initialize); m.def("finalize",&finalize); - py::enum_(m,"Tag") - .value("INV" ,INV) - .value("EL" ,EL) - .value("COL" ,COL) - .value("GP" ,GP) - .value("TL" ,TL) - .value("LEV" ,LEV) - .value("ILEV",ILEV) - .value("CMP" ,CMP); - py::class_(m,"Field") .def(py::init&>()) .def(py::init >()) - .def("print",&PyField::print); + .def("print",&PyField::print) + .def("cleanup",&PyField::cleanup); } From 9e3f6f9a5702296df18034688e8fb7275a63e7b4 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 28 May 2024 21:18:18 -0500 Subject: [PATCH 382/476] add pybind11 as a submodule in externals --- .gitmodules | 3 +++ components/eamxx/src/share/pyshare/CMakeLists.txt | 2 +- externals/pybind11 | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 160000 externals/pybind11 diff --git a/.gitmodules b/.gitmodules index 69d56e140361..2db4c370ec78 100644 --- a/.gitmodules +++ b/.gitmodules @@ -83,3 +83,6 @@ path = externals/mam4xx url = git@github.com:eagles-project/mam4xx.git +[submodule "externals/pybind11"] + path = externals/pybind11 + url = git@github.com:pybind/pybind11.git diff --git a/components/eamxx/src/share/pyshare/CMakeLists.txt b/components/eamxx/src/share/pyshare/CMakeLists.txt index 1f8f74f7a9ee..6227a7321fd9 100644 --- a/components/eamxx/src/share/pyshare/CMakeLists.txt +++ b/components/eamxx/src/share/pyshare/CMakeLists.txt @@ -1,4 +1,4 @@ -set (pybind11_DIR /home/lbertag/.conda/envs/bartgol/lib/python3.12/site-packages) +set (pybind11_DIR ${SCREAM_BASE_DIR}/../../externals/pybind11) find_package(pybind11 REQUIRED HINTS ${pybind11_DIR}) diff --git a/externals/pybind11 b/externals/pybind11 new file mode 160000 index 000000000000..ae6432b8172e --- /dev/null +++ b/externals/pybind11 @@ -0,0 +1 @@ +Subproject commit ae6432b8172e24f8da60375302a2e6e9e5f714d1 From a7ac5167b412b94a6922c3848f6a5e593b6522c2 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Tue, 28 May 2024 22:40:21 -0500 Subject: [PATCH 383/476] a skeleton for a package we likely should set this up under eamxx/pippkg or something, tbd --- .gitignore | 8 +++++ .../eamxx/src/python/screaminpy/__init__.py | 7 ++++ components/eamxx/src/python/setup.py | 35 +++++++++++++++++++ .../eamxx/src/python/tests/basic_test.py | 20 +++++++++++ 4 files changed, 70 insertions(+) create mode 100644 components/eamxx/src/python/screaminpy/__init__.py create mode 100644 components/eamxx/src/python/setup.py create mode 100644 components/eamxx/src/python/tests/basic_test.py diff --git a/.gitignore b/.gitignore index c4d2a64bc994..f70951bb3578 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ *.la *.a +# Shared libraries +*.so + # Executables *.exe @@ -34,3 +37,8 @@ site components/eamxx/site/* # Ignore auto-generated eamxx_params.md file components/eamxx/docs/common/eamxx_params.md + +# Python packaging +*.egg-info +*components/eamxx/src/python/build +*components/eamxx/src/python/dist diff --git a/components/eamxx/src/python/screaminpy/__init__.py b/components/eamxx/src/python/screaminpy/__init__.py new file mode 100644 index 000000000000..1817c6642c4f --- /dev/null +++ b/components/eamxx/src/python/screaminpy/__init__.py @@ -0,0 +1,7 @@ +""" to get going """ + +__version__ = "1.0.0" + +# expose the executable we just packaged +# so that we can do `from screaminpy import pyscream` +import pyscream diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py new file mode 100644 index 000000000000..f9cc91a1e4a0 --- /dev/null +++ b/components/eamxx/src/python/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup, Distribution + + +class BinaryDistribution(Distribution): + def has_ext_modules(foo): + return True + +# For all of this to work, we either need to: +# 1. build the extension as we build the python package; or +# 2. simply package the extension as "data". +# The latter is actually quite annoying, but we may have to do it anyway. +# So, bring the extension to the dir of this file, before triggering + +setup( + name='screaminpy', + version='1.0.0', + author="screamers", + description='screaminpy wrapper', + packages=['', 'screaminpy'], + package_data={ + 'screaminpy': ['*.py'], + '': ['pyscream.*so'], + }, + distclass=BinaryDistribution +) + +# TODOs: +# 1. We need to figure out how to handle the complex env needed by scream +# 2. We need to think forward about packaging this neatly + +# for both issues, I think the most straightforward solution for now is the following: +# We simply build manually-ish for support machines and just upload the wheels +# to pypi. That way, we can ensure the build works on the machine, +# we can set its environment as we wish, etc.. +# The downside is that it is manual... \ No newline at end of file diff --git a/components/eamxx/src/python/tests/basic_test.py b/components/eamxx/src/python/tests/basic_test.py new file mode 100644 index 000000000000..256870e53de3 --- /dev/null +++ b/components/eamxx/src/python/tests/basic_test.py @@ -0,0 +1,20 @@ +# luca's verbatim example works from any dir now! +from screaminpy import pyscream as ps +import numpy as np + +ps.init() + +v = np.zeros(shape=(10,2)) + +for i in range(0,10): + for j in range(0,2): + v[i,j] = i*2 + j + 1 + +print(f"v:{v}") + +f = ps.Field("T_mid",v) + +f.print() + +f.cleanup() +ps.finalize() From f2bffcfc5e72d0231ccf2249a110ed9f9143a9ff Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 22:04:42 -0600 Subject: [PATCH 384/476] EAMxx: split pyscream.cpp into multiple files * All other files are hpp files included by pyscream.cpp * Each file handle separate EAMxx concept * Remove hard-coding of pybind11_DIR. User must provide pybind11_ROOT cmake/env var --- .../eamxx/src/share/pyshare/CMakeLists.txt | 4 +- .../eamxx/src/share/pyshare/pyfield.hpp | 62 +++++++++++++++++ .../eamxx/src/share/pyshare/pyscream.cpp | 66 +++---------------- 3 files changed, 72 insertions(+), 60 deletions(-) create mode 100644 components/eamxx/src/share/pyshare/pyfield.hpp diff --git a/components/eamxx/src/share/pyshare/CMakeLists.txt b/components/eamxx/src/share/pyshare/CMakeLists.txt index 6227a7321fd9..1284106421c0 100644 --- a/components/eamxx/src/share/pyshare/CMakeLists.txt +++ b/components/eamxx/src/share/pyshare/CMakeLists.txt @@ -1,6 +1,4 @@ -set (pybind11_DIR ${SCREAM_BASE_DIR}/../../externals/pybind11) -find_package(pybind11 REQUIRED - HINTS ${pybind11_DIR}) +find_package(pybind11 REQUIRED) pybind11_add_module(pyscream pyscream.cpp) target_link_libraries(pyscream PUBLIC scream_share scream_io) diff --git a/components/eamxx/src/share/pyshare/pyfield.hpp b/components/eamxx/src/share/pyshare/pyfield.hpp new file mode 100644 index 000000000000..2a88950fbbb5 --- /dev/null +++ b/components/eamxx/src/share/pyshare/pyfield.hpp @@ -0,0 +1,62 @@ +#include "share/field/field.hpp" +#include "share/field/field_utils.hpp" + +#include +#include +#include + +namespace scream { + +struct PyField { + Field f; + + // Create field and allocate memory + PyField(const std::string& n, + const std::vector& d) + { + std::vector t (d.size(),FieldTag::Component); + FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); + f = Field(fid); + f.allocate_view(); + } + + // Create field from pre-existing python nd array + PyField(const std::string& n, + pybind11::array_t arr) + { + int rank = arr.ndim(); + std::vector d(rank,-1); + for (int n=0; n t (rank,FieldTag::Component); + FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); + switch (rank) { + case 1: + { + Field::view_dev_t v(arr.mutable_data(0),d[0]); + f = Field(fid,v); + break; + } + case 2: + { + Field::view_dev_t v(arr.mutable_data(0,0),d[0],d[1]); + f = Field(fid,v); + v(0,0) = -1; + break; + } + default: + EKAT_ERROR_MSG ("AAARGH!\n"); + } + } + + void print() const { + print_field_hyperslab(f); + } + + void cleanup () { + f = Field(); + } +}; + +} // namespace scream diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/pyshare/pyscream.cpp index b519c3483834..44836f6cad26 100644 --- a/components/eamxx/src/share/pyshare/pyscream.cpp +++ b/components/eamxx/src/share/pyshare/pyscream.cpp @@ -1,80 +1,30 @@ #include "scream_session.hpp" -#include "share/field/field.hpp" -#include "share/field/field_utils.hpp" +#include "pyfield.hpp" #include #include #include -using namespace scream; -using namespace ShortFieldTagsNames; -namespace py = pybind11; +namespace scream { void initialize () { - initialize_scream_session(false); + initialize_scream_session(true); } void finalize () { finalize_scream_session(); } -struct PyField { - Field f; - - // Create field and allocate memory - PyField(const std::string& n, - const std::vector& d) - { - std::vector t (d.size(),CMP); - FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); - f = Field(fid); - f.allocate_view(); - } - - // Create field from pre-existing python nd array - PyField(const std::string& n, - py::array_t arr) - { - int rank = arr.ndim(); - std::vector d(rank,-1); - for (int n=0; n t (rank,CMP); - FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); - switch (rank) { - case 1: - { - Field::view_dev_t v(arr.mutable_data(0),d[0]); - f = Field(fid,v); - break; - } - case 2: - { - Field::view_dev_t v(arr.mutable_data(0,0),d[0],d[1]); - f = Field(fid,v); - break; - } - default: - EKAT_ERROR_MSG ("AAARGH!\n"); - } - } - - void print() const { - print_field_hyperslab(f); - } - - void cleanup () { - f = Field(); - } -}; +namespace py = pybind11; PYBIND11_MODULE (pyscream,m) { - m.doc() = "Basic interface to scream session initialization"; + m.doc() = "Python interfaces to certain EAMxx infrastructure code"; + // Scream Session m.def("init",&initialize); m.def("finalize",&finalize); + // Field class py::class_(m,"Field") .def(py::init&>()) @@ -84,3 +34,5 @@ PYBIND11_MODULE (pyscream,m) { .def("print",&PyField::print) .def("cleanup",&PyField::cleanup); } + +} // namespace scream From 6d4f3334b0b2e8e12f311e2c14faa6e9c406579d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 22:05:04 -0600 Subject: [PATCH 385/476] EAMxx: add PointGrid wrapper to pyscream --- components/eamxx/src/share/pyshare/pygrid.hpp | 22 +++++++++++++++++++ .../eamxx/src/share/pyshare/pyscream.cpp | 6 +++++ 2 files changed, 28 insertions(+) create mode 100644 components/eamxx/src/share/pyshare/pygrid.hpp diff --git a/components/eamxx/src/share/pyshare/pygrid.hpp b/components/eamxx/src/share/pyshare/pygrid.hpp new file mode 100644 index 000000000000..cd451bbf9941 --- /dev/null +++ b/components/eamxx/src/share/pyshare/pygrid.hpp @@ -0,0 +1,22 @@ +#include "share/grid/point_grid.hpp" + +#include + +namespace scream { + +struct PyPointGrid { + std::shared_ptr g; + + // Create field and allocate memory + PyPointGrid(const std::string& name, int ncols, int nlevs) + { + ekat::Comm comm(MPI_COMM_WORLD); + g = create_point_grid (name,ncols,nlevs,comm); + } + + void cleanup () { + g = nullptr; + } +}; + +} // namespace scream diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/pyshare/pyscream.cpp index 44836f6cad26..fc7c918827b9 100644 --- a/components/eamxx/src/share/pyshare/pyscream.cpp +++ b/components/eamxx/src/share/pyshare/pyscream.cpp @@ -1,5 +1,6 @@ #include "scream_session.hpp" #include "pyfield.hpp" +#include "pygrid.hpp" #include #include @@ -33,6 +34,11 @@ PYBIND11_MODULE (pyscream,m) { >()) .def("print",&PyField::print) .def("cleanup",&PyField::cleanup); + + // PointGrid + py::class_(m,"PointGrid") + .def(py::init()) + .def("cleanup",&PyPointGrid::cleanup); } } // namespace scream From 57919ad53fccc1a0ff3f7740ff60d68eb8a4c3f3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 23:57:08 -0600 Subject: [PATCH 386/476] EAMxx: add units to pyscream --- .../eamxx/src/share/pyshare/pyscream.cpp | 25 +++++++++- .../eamxx/src/share/pyshare/pyunits.hpp | 50 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 components/eamxx/src/share/pyshare/pyunits.hpp diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/pyshare/pyscream.cpp index fc7c918827b9..3508af86fed6 100644 --- a/components/eamxx/src/share/pyshare/pyscream.cpp +++ b/components/eamxx/src/share/pyshare/pyscream.cpp @@ -1,11 +1,13 @@ #include "scream_session.hpp" #include "pyfield.hpp" #include "pygrid.hpp" +#include "pyunits.hpp" #include #include #include +namespace py = pybind11; namespace scream { void initialize () { @@ -15,8 +17,6 @@ void finalize () { finalize_scream_session(); } -namespace py = pybind11; - PYBIND11_MODULE (pyscream,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; @@ -25,6 +25,27 @@ PYBIND11_MODULE (pyscream,m) { m.def("init",&initialize); m.def("finalize",&finalize); + // Units + py::module units = m.def_submodule("units"); + py::class_(units,"Units") + .def(py::init<>()) + .def(py::init()) + .def("str",&PyUnits::str) + .def("__mul__",[](const PyUnits& x, const PyUnits& y) { return PyUnits(x.units*y.units); }) + .def("__truediv__",[](const PyUnits& x, const PyUnits& y) { return PyUnits(x.units/y.units); }); + + units.def("pow",[](const PyUnits& x, const int n) { return PyUnits(pow(x.units,ekat::RationalConstant(n))); }); + units.def("root",[](const PyUnits& x, const int n) { return PyUnits(pow(x.units,ekat::RationalConstant(1,n))); }); + units.attr("one") = &short_units::one; + units.attr("m") = &short_units::m; + units.attr("s") = &short_units::s; + units.attr("kg") = &short_units::kg; + units.attr("K") = &short_units::K; + units.attr("N") = &short_units::N; + units.attr("J") = &short_units::J; + units.attr("W") = &short_units::W; + units.attr("Pa") = &short_units::Pa; + // Field class py::class_(m,"Field") .def(py::init + +#include + +namespace scream { + +struct PyUnits { + ekat::units::Units units; + + PyUnits () + : units(ekat::RationalConstant(0)) + { + // Nothing to do here + } + PyUnits (const ekat::units::Units& src) + : units(src) + { + // Nothing to do here + } + + std::string str() { + return units.to_string(); + } + + PyUnits (const PyUnits&) = default; +}; + +namespace short_units { + +PyUnits one(ekat::units::Units::nondimensional()); +PyUnits m(ekat::units::m); +PyUnits s(ekat::units::s); +PyUnits kg(ekat::units::kg); +PyUnits A(ekat::units::A); +PyUnits mol(ekat::units::mol); +PyUnits cd(ekat::units::cd); +PyUnits K(ekat::units::K); + +PyUnits N (ekat::units::N); +PyUnits J (ekat::units::J); +PyUnits W (ekat::units::W); +PyUnits Pa (ekat::units::Pa); +} + +} // namespace scream + +#endif // PYUNITS_HPP From ed833718bd247f1524567c6bc3558db6da33d9ea Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 23:59:13 -0600 Subject: [PATCH 387/476] EAMxx: more improvements to pyscream * Add header guards to pyxyz.hpp * Add PyGrid type * PyGrid can create managed/unmanaged fields * Add constructors to PyField (not all py-interoperable) --- .../eamxx/src/share/pyshare/CMakeLists.txt | 2 +- .../eamxx/src/share/pyshare/pyfield.hpp | 61 ++++++-- components/eamxx/src/share/pyshare/pygrid.hpp | 132 +++++++++++++++++- .../eamxx/src/share/pyshare/pyscream.cpp | 20 ++- 4 files changed, 193 insertions(+), 22 deletions(-) diff --git a/components/eamxx/src/share/pyshare/CMakeLists.txt b/components/eamxx/src/share/pyshare/CMakeLists.txt index 1284106421c0..8c1cc6bc9835 100644 --- a/components/eamxx/src/share/pyshare/CMakeLists.txt +++ b/components/eamxx/src/share/pyshare/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC scream_share scream_io) +target_link_libraries(pyscream PUBLIC scream_share scream_test_support scream_io) diff --git a/components/eamxx/src/share/pyshare/pyfield.hpp b/components/eamxx/src/share/pyshare/pyfield.hpp index 2a88950fbbb5..2891998e9d3a 100644 --- a/components/eamxx/src/share/pyshare/pyfield.hpp +++ b/components/eamxx/src/share/pyshare/pyfield.hpp @@ -1,3 +1,6 @@ +#ifndef PYFIELD_HPP +#define PYFIELD_HPP + #include "share/field/field.hpp" #include "share/field/field_utils.hpp" @@ -10,14 +13,21 @@ namespace scream { struct PyField { Field f; + // Create empty field + PyField () = default; + + PyField(const FieldIdentifier& fid) + { + create(fid); + } + // Create field and allocate memory PyField(const std::string& n, const std::vector& d) { std::vector t (d.size(),FieldTag::Component); FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); - f = Field(fid); - f.allocate_view(); + create(fid); } // Create field from pre-existing python nd array @@ -31,16 +41,49 @@ struct PyField { } std::vector t (rank,FieldTag::Component); FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); + create(fid,arr); + } + + PyField(const FieldIdentifier& fid, + pybind11::array_t arr) + { + create(fid,arr); + } + + void print() const { + print_field_hyperslab(f); + } + + void cleanup () { + f = Field(); + } + +private: + void create (const FieldIdentifier& fid) + { + f = Field(fid); + f.allocate_view(); + } + + void create (const FieldIdentifier& fid, + pybind11::array_t arr) + { + int rank = arr.ndim(); + EKAT_REQUIRE_MSG (rank==fid.get_layout().rank(), + "Error! Rank mismatch between input FieldIdentifier and pybind11::array_t.\n" + " - field name: " + fid.name() + "\n" + " - identifier rank: " + std::to_string(fid.get_layout().rank()) + "\n" + " - array_t rank : " + std::to_string(rank) + "\n"); switch (rank) { case 1: { - Field::view_dev_t v(arr.mutable_data(0),d[0]); + Field::view_dev_t v(arr.mutable_data(0),arr.shape(0)); f = Field(fid,v); break; } case 2: { - Field::view_dev_t v(arr.mutable_data(0,0),d[0],d[1]); + Field::view_dev_t v(arr.mutable_data(0,0),arr.shape(0),arr.shape(1)); f = Field(fid,v); v(0,0) = -1; break; @@ -49,14 +92,8 @@ struct PyField { EKAT_ERROR_MSG ("AAARGH!\n"); } } - - void print() const { - print_field_hyperslab(f); - } - - void cleanup () { - f = Field(); - } }; } // namespace scream + +#endif // PYFIELD_HPP diff --git a/components/eamxx/src/share/pyshare/pygrid.hpp b/components/eamxx/src/share/pyshare/pygrid.hpp index cd451bbf9941..292f27ad4aa2 100644 --- a/components/eamxx/src/share/pyshare/pygrid.hpp +++ b/components/eamxx/src/share/pyshare/pygrid.hpp @@ -1,22 +1,142 @@ -#include "share/grid/point_grid.hpp" +#ifndef PYGRID_HPP +#define PYGRID_HPP + +#include "share/grid/mesh_free_grids_manager.hpp" +#include "pyunits.hpp" +#include "pyfield.hpp" #include namespace scream { -struct PyPointGrid { - std::shared_ptr g; +struct PyGrid { + std::shared_ptr grid; + + // 2d scalar, managed/unmanaged + PyField scalar2d (const std::string& name, + const PyUnits& u) + { + FieldIdentifier fid(name,grid->get_2d_scalar_layout(),u.units,grid->name()); + return PyField(fid); + } + PyField scalar2d (const std::string& name, + const PyUnits& u, + pybind11::array_t arr) + { + FieldIdentifier fid(name,grid->get_2d_scalar_layout(),u.units,grid->name()); + return PyField(fid,arr); + } + + // 2d vector, managed/unmanaged + PyField vector2d (const std::string& name, + const PyUnits& u, + int vdim) + { + FieldIdentifier fid(name,grid->get_2d_vector_layout(vdim),u.units,grid->name()); + return PyField(fid); + } + PyField vector2d (const std::string& name, + const PyUnits& u, + int vdim, + pybind11::array_t arr) + { + FieldIdentifier fid(name,grid->get_2d_vector_layout(vdim),u.units,grid->name()); + return PyField(fid,arr); + } + + // 3d midpoints scalar, managed/unmanaged + PyField scalar3d_mid (const std::string& name, + const PyUnits& u) + { + FieldIdentifier fid(name,grid->get_3d_scalar_layout(true),u.units,grid->name()); + return PyField(fid); + } + PyField scalar3d_mid (const std::string& name, + const PyUnits& u, + pybind11::array_t arr) + { + FieldIdentifier fid(name,grid->get_3d_scalar_layout(true),u.units,grid->name()); + return PyField(fid,arr); + } + + // 3d interfaces scalar, managed/unmanaged + PyField scalar3d_int (const std::string& name, + const PyUnits& u) + { + FieldIdentifier fid(name,grid->get_3d_scalar_layout(false),u.units,grid->name()); + return PyField(fid); + } + PyField scalar3d_int (const std::string& name, + const PyUnits& u, + pybind11::array_t arr) + { + FieldIdentifier fid(name,grid->get_3d_scalar_layout(false),u.units,grid->name()); + return PyField(fid,arr); + } +private: + PyField create (const FieldIdentifier& fid, + pybind11::array_t arr) + { + const auto rank = fid.get_layout().rank(); + PyField pyf; + EKAT_REQUIRE_MSG (rank==arr.ndim(), + "Error! Rank mismatch between input FieldIdentifier and pybind11::array_t.\n" + " - field name: " + fid.name() + "\n" + " - identifier rank: " + std::to_string(rank) + "\n" + " - array_t rank : " + std::to_string(arr.ndim()) + "\n"); + switch (rank) { + case 1: + { + Field::view_dev_t v(arr.mutable_data(0),arr.shape(0)); + pyf.f = Field(fid,v); + break; + } + case 2: + { + Field::view_dev_t v(arr.mutable_data(0,0),arr.shape(0),arr.shape(1)); + pyf.f = Field(fid,v); + break; + } + default: + EKAT_ERROR_MSG ("AAARGH!\n"); + } + return pyf; + } +}; + +struct PyGridsManager { + std::shared_ptr gm; + PyGrid grid; // Create field and allocate memory - PyPointGrid(const std::string& name, int ncols, int nlevs) + PyGridsManager(const std::string& name, int ncols, int nlevs) { + ekat::ParameterList gm_pl; + auto& pg_pl = gm_pl.sublist(name); + pg_pl.set("type",std::string("point_grid")); + pg_pl.set("number_of_global_columns",ncols); + pg_pl.set("number_of_vertical_levels",nlevs); + pg_pl.set>("aliases",{"Physics"}); + + gm_pl.set("grids_names",std::vector{name}); + ekat::Comm comm(MPI_COMM_WORLD); - g = create_point_grid (name,ncols,nlevs,comm); + gm = std::make_shared(comm,gm_pl); + gm->build_grids(); + + grid.grid = gm->get_grid(name); + } + + const PyGrid& get_grid () const { + return grid; } void cleanup () { - g = nullptr; + gm = nullptr; + grid = {}; } }; } // namespace scream + +#endif // PYGRID_HPP diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/pyshare/pyscream.cpp index 3508af86fed6..ee22f3b2ee19 100644 --- a/components/eamxx/src/share/pyshare/pyscream.cpp +++ b/components/eamxx/src/share/pyshare/pyscream.cpp @@ -48,6 +48,7 @@ PYBIND11_MODULE (pyscream,m) { // Field class py::class_(m,"Field") + .def(py::init<>()) .def(py::init&>()) .def(py::init(m,"PointGrid") + // Grid + py::class_(m,"Grid") + .def(py::init<>()) + .def("scalar2d",py::overload_cast(&PyGrid::scalar2d)) + .def("scalar2d",py::overload_cast>(&PyGrid::scalar2d)) + .def("vector2d",py::overload_cast(&PyGrid::vector2d)) + .def("vector2d",py::overload_cast>(&PyGrid::vector2d)) + .def("scalar3d_mid",py::overload_cast(&PyGrid::scalar3d_mid)) + .def("scalar3d_mid",py::overload_cast>(&PyGrid::scalar3d_mid)) + .def("scalar3d_int",py::overload_cast(&PyGrid::scalar3d_int)) + .def("scalar3d_int",py::overload_cast>(&PyGrid::scalar3d_int)); + + // GridsManager + py::class_(m,"GridsManager") .def(py::init()) - .def("cleanup",&PyPointGrid::cleanup); + .def("get_grid",&PyGridsManager::get_grid) + .def("cleanup",&PyGridsManager::cleanup); } } // namespace scream From f5301efed06b6a5c150c12b2ff346c0ebfd6ecf6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 28 May 2024 23:59:30 -0600 Subject: [PATCH 388/476] EAMxx: start working on pyp3.cpp --- .../eamxx/src/physics/p3/CMakeLists.txt | 4 ++ .../src/physics/p3/python/CMakeLists.txt | 6 ++ .../eamxx/src/physics/p3/python/pyp3.cpp | 58 +++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 components/eamxx/src/physics/p3/python/CMakeLists.txt create mode 100644 components/eamxx/src/physics/p3/python/pyp3.cpp diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 7ea0aeeb8145..287bac452c1b 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -129,6 +129,10 @@ if (Kokkos_ENABLE_HIP) set_source_files_properties(p3_functions_f90.cpp PROPERTIES COMPILE_FLAGS -O0) endif() +if (EAMXX_ENABLE_PYBIND) + add_subdirectory (python) +endif() + if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/physics/p3/python/CMakeLists.txt b/components/eamxx/src/physics/p3/python/CMakeLists.txt new file mode 100644 index 000000000000..2080d0d7eb01 --- /dev/null +++ b/components/eamxx/src/physics/p3/python/CMakeLists.txt @@ -0,0 +1,6 @@ +# By now it should have already been found, but just for completeness +find_package(pybind11 REQUIRED) + +pybind11_add_module(pyp3 pyp3.cpp) +target_link_libraries(pyp3 PUBLIC p3) +target_include_directories(pyp3 PRIVATE ${SCREAM_BASE_DIR}/src/share/pyshare) diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/physics/p3/python/pyp3.cpp new file mode 100644 index 000000000000..69a7da3d4ba6 --- /dev/null +++ b/components/eamxx/src/physics/p3/python/pyp3.cpp @@ -0,0 +1,58 @@ +#include "physics/p3/eamxx_p3_process_interface.hpp" +#include "pygrid.hpp" +#include "pyfield.hpp" + +#include + +namespace scream { + +namespace py = pybind11; + +struct PyP3 { + std::shared_ptr p3; + + PyP3 (const PyGridsManager& pygm) { + ekat::Comm comm(MPI_COMM_WORLD); + ekat::ParameterList pl; + pl.set("max_total_ni",10.0); + + p3 = create_atmosphere_process(comm,pl); + p3->set_grids(pygm.gm); + } + + void set_fields(const std::vector& pyfields) { + for (const auto& pyf : pyfields) { + const auto& fid = pyf.f.get_header().get_identifier(); + if (p3->has_required_field(fid)) { + p3->set_required_field(pyf.f.get_const()); + } + if (p3->has_computed_field(fid)) { + p3->set_computed_field(pyf.f); + } + } + } + + void initialize () { + util::TimeStamp ts; + p3->initialize(ts,RunType::Initial); + } + + void cleanup () { + p3 = nullptr; + } +}; + +PYBIND11_MODULE (pyp3,m) { + + m.doc() = "Python interfaces to P3Microphysics"; + + py::class_(m,"P3") + .def(py::init()) + .def("set_fields",&PyP3::set_fields) + .def("initialize",&PyP3::initialize) + .def("cleanup",&PyP3::cleanup); + +} + +} // namespace scream + From fc66d11029a190c418a2423dc573b2ef40a049a4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 00:01:08 -0600 Subject: [PATCH 389/476] EAMxx: moved share/pyshare to share/python Start the pattern where folder blah has python subfolder if pybind hooks are supported --- components/eamxx/src/physics/p3/python/CMakeLists.txt | 2 +- components/eamxx/src/share/CMakeLists.txt | 2 +- components/eamxx/src/share/{pyshare => python}/CMakeLists.txt | 0 components/eamxx/src/share/{pyshare => python}/pyfield.hpp | 0 components/eamxx/src/share/{pyshare => python}/pygrid.hpp | 0 components/eamxx/src/share/{pyshare => python}/pyscream.cpp | 0 components/eamxx/src/share/{pyshare => python}/pyunits.hpp | 0 7 files changed, 2 insertions(+), 2 deletions(-) rename components/eamxx/src/share/{pyshare => python}/CMakeLists.txt (100%) rename components/eamxx/src/share/{pyshare => python}/pyfield.hpp (100%) rename components/eamxx/src/share/{pyshare => python}/pygrid.hpp (100%) rename components/eamxx/src/share/{pyshare => python}/pyscream.cpp (100%) rename components/eamxx/src/share/{pyshare => python}/pyunits.hpp (100%) diff --git a/components/eamxx/src/physics/p3/python/CMakeLists.txt b/components/eamxx/src/physics/p3/python/CMakeLists.txt index 2080d0d7eb01..0b1b75e1b6ed 100644 --- a/components/eamxx/src/physics/p3/python/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/python/CMakeLists.txt @@ -3,4 +3,4 @@ find_package(pybind11 REQUIRED) pybind11_add_module(pyp3 pyp3.cpp) target_link_libraries(pyp3 PUBLIC p3) -target_include_directories(pyp3 PRIVATE ${SCREAM_BASE_DIR}/src/share/pyshare) +target_include_directories(pyp3 PRIVATE ${SCREAM_BASE_DIR}/src/share/python) diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index 6a5620fde5a1..a2a2e2bc9fb7 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -105,7 +105,7 @@ endif() add_subdirectory(io) if (EAMXX_ENABLE_PYBIND) - add_subdirectory(pyshare) + add_subdirectory(python) endif() if (NOT SCREAM_LIB_ONLY) diff --git a/components/eamxx/src/share/pyshare/CMakeLists.txt b/components/eamxx/src/share/python/CMakeLists.txt similarity index 100% rename from components/eamxx/src/share/pyshare/CMakeLists.txt rename to components/eamxx/src/share/python/CMakeLists.txt diff --git a/components/eamxx/src/share/pyshare/pyfield.hpp b/components/eamxx/src/share/python/pyfield.hpp similarity index 100% rename from components/eamxx/src/share/pyshare/pyfield.hpp rename to components/eamxx/src/share/python/pyfield.hpp diff --git a/components/eamxx/src/share/pyshare/pygrid.hpp b/components/eamxx/src/share/python/pygrid.hpp similarity index 100% rename from components/eamxx/src/share/pyshare/pygrid.hpp rename to components/eamxx/src/share/python/pygrid.hpp diff --git a/components/eamxx/src/share/pyshare/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp similarity index 100% rename from components/eamxx/src/share/pyshare/pyscream.cpp rename to components/eamxx/src/share/python/pyscream.cpp diff --git a/components/eamxx/src/share/pyshare/pyunits.hpp b/components/eamxx/src/share/python/pyunits.hpp similarity index 100% rename from components/eamxx/src/share/pyshare/pyunits.hpp rename to components/eamxx/src/share/python/pyunits.hpp From 2aac70d00e7c7ccdd9eb707cd6b0295d910964e4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 00:21:08 -0600 Subject: [PATCH 390/476] EAMxx: no need to expose a GridsManager concept in pyscream When needed, build a GridsManager on the fly, using the ad-hoc type SingleGridGM --- .../eamxx/src/physics/p3/python/pyp3.cpp | 10 ++- .../eamxx/src/share/python/CMakeLists.txt | 2 +- components/eamxx/src/share/python/pygrid.hpp | 66 +++++++++---------- .../eamxx/src/share/python/pyscream.cpp | 8 +-- 4 files changed, 40 insertions(+), 46 deletions(-) diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/physics/p3/python/pyp3.cpp index 69a7da3d4ba6..bbd5c484cbc0 100644 --- a/components/eamxx/src/physics/p3/python/pyp3.cpp +++ b/components/eamxx/src/physics/p3/python/pyp3.cpp @@ -1,4 +1,5 @@ #include "physics/p3/eamxx_p3_process_interface.hpp" +#include "share/grid/point_grid.hpp" #include "pygrid.hpp" #include "pyfield.hpp" @@ -11,13 +12,16 @@ namespace py = pybind11; struct PyP3 { std::shared_ptr p3; - PyP3 (const PyGridsManager& pygm) { + PyP3 (const PyGrid& pygrid) { ekat::Comm comm(MPI_COMM_WORLD); ekat::ParameterList pl; pl.set("max_total_ni",10.0); p3 = create_atmosphere_process(comm,pl); - p3->set_grids(pygm.gm); + + // Create a grids manager on the fly + auto gm = std::make_shared(pygrid.grid); + p3->set_grids(gm); } void set_fields(const std::vector& pyfields) { @@ -47,7 +51,7 @@ PYBIND11_MODULE (pyp3,m) { m.doc() = "Python interfaces to P3Microphysics"; py::class_(m,"P3") - .def(py::init()) + .def(py::init()) .def("set_fields",&PyP3::set_fields) .def("initialize",&PyP3::initialize) .def("cleanup",&PyP3::cleanup); diff --git a/components/eamxx/src/share/python/CMakeLists.txt b/components/eamxx/src/share/python/CMakeLists.txt index 8c1cc6bc9835..1284106421c0 100644 --- a/components/eamxx/src/share/python/CMakeLists.txt +++ b/components/eamxx/src/share/python/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC scream_share scream_test_support scream_io) +target_link_libraries(pyscream PUBLIC scream_share scream_io) diff --git a/components/eamxx/src/share/python/pygrid.hpp b/components/eamxx/src/share/python/pygrid.hpp index 292f27ad4aa2..12cee80ba9da 100644 --- a/components/eamxx/src/share/python/pygrid.hpp +++ b/components/eamxx/src/share/python/pygrid.hpp @@ -1,7 +1,8 @@ #ifndef PYGRID_HPP #define PYGRID_HPP -#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/grid/grids_manager.hpp" +#include "share/grid/point_grid.hpp" #include "pyunits.hpp" #include "pyfield.hpp" @@ -9,8 +10,36 @@ namespace scream { +// Small grids manager class, to hold a pre-built grid +class SingleGridGM : public GridsManager +{ +public: + SingleGridGM (const std::shared_ptr& grid) + { + add_grid(grid); + } + + std::string name () const override { return "SingleGridGM"; } + + void build_grids () override {} + +protected: + remapper_ptr_type + do_create_remapper (const grid_ptr_type /* from_grid */, + const grid_ptr_type /* to_grid */) const + { + EKAT_ERROR_MSG ("Error! do_create_remapper not implemented for SingleGridGM.\n"); + } +}; + struct PyGrid { - std::shared_ptr grid; + std::shared_ptr grid; + + PyGrid(const std::string& name, int ncols, int nlevs) + { + ekat::Comm comm(MPI_COMM_WORLD); + grid = create_point_grid(name,ncols,nlevs,comm); + } // 2d scalar, managed/unmanaged PyField scalar2d (const std::string& name, @@ -104,39 +133,6 @@ struct PyGrid { } }; -struct PyGridsManager { - std::shared_ptr gm; - PyGrid grid; - - // Create field and allocate memory - PyGridsManager(const std::string& name, int ncols, int nlevs) - { - ekat::ParameterList gm_pl; - auto& pg_pl = gm_pl.sublist(name); - pg_pl.set("type",std::string("point_grid")); - pg_pl.set("number_of_global_columns",ncols); - pg_pl.set("number_of_vertical_levels",nlevs); - pg_pl.set>("aliases",{"Physics"}); - - gm_pl.set("grids_names",std::vector{name}); - - ekat::Comm comm(MPI_COMM_WORLD); - gm = std::make_shared(comm,gm_pl); - gm->build_grids(); - - grid.grid = gm->get_grid(name); - } - - const PyGrid& get_grid () const { - return grid; - } - - void cleanup () { - gm = nullptr; - grid = {}; - } -}; - } // namespace scream #endif // PYGRID_HPP diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp index ee22f3b2ee19..6ff3dc0cb40c 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/share/python/pyscream.cpp @@ -59,7 +59,7 @@ PYBIND11_MODULE (pyscream,m) { // Grid py::class_(m,"Grid") - .def(py::init<>()) + .def(py::init()) .def("scalar2d",py::overload_cast(&PyGrid::scalar2d)) .def("scalar2d",py::overload_cast>(&PyGrid::scalar2d)) .def("vector2d",py::overload_cast(&PyGrid::vector2d)) @@ -68,12 +68,6 @@ PYBIND11_MODULE (pyscream,m) { .def("scalar3d_mid",py::overload_cast>(&PyGrid::scalar3d_mid)) .def("scalar3d_int",py::overload_cast(&PyGrid::scalar3d_int)) .def("scalar3d_int",py::overload_cast>(&PyGrid::scalar3d_int)); - - // GridsManager - py::class_(m,"GridsManager") - .def(py::init()) - .def("get_grid",&PyGridsManager::get_grid) - .def("cleanup",&PyGridsManager::cleanup); } } // namespace scream From 3028f3aad2e4af1afc6db440f937c3a206dfc937 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 16:48:29 -0600 Subject: [PATCH 391/476] EAMxx: minor mod to power-user method in Field class --- components/eamxx/src/share/field/field.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index 4a4b20525a3e..c7bc6cbf6df8 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -181,8 +181,9 @@ class Field { ST* get_internal_view_data_unsafe () const { // Check that the scalar type is correct using nonconst_ST = typename std::remove_const::type; - EKAT_REQUIRE_MSG ((field_valid_data_types().at()==m_header->get_identifier().data_type() - or std::is_same::value), + EKAT_REQUIRE_MSG ((std::is_same::value or std::is_same::value or + (field_valid_data_types().has_t() and + get_data_type()==m_header->get_identifier().data_type())), "Error! Attempt to access raw field pointere with the wrong scalar type.\n"); return reinterpret_cast(get_view_impl().data()); From c4d58e20f5981d06c462792bff8fb0250e2018f4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 16:51:19 -0600 Subject: [PATCH 392/476] EAMxx: simplify pyscream interfaces * Fields are created by pyscream, no possibility to pass pre-built numpy array * Add generic impl of atm proc wrapper in PyAtmProc * Users can grab fields out of a py AP based on field name * Removed pointless stuff from py grid * Removed no longer needed wrappers for units --- .../eamxx/src/physics/p3/python/pyp3.cpp | 39 ++------ .../eamxx/src/share/python/pyatmproc.hpp | 84 +++++++++++++++++ components/eamxx/src/share/python/pyfield.hpp | 91 ++++++------------- components/eamxx/src/share/python/pygrid.hpp | 88 +----------------- .../eamxx/src/share/python/pyscream.cpp | 49 ++-------- components/eamxx/src/share/python/pyunits.hpp | 50 ---------- 6 files changed, 127 insertions(+), 274 deletions(-) create mode 100644 components/eamxx/src/share/python/pyatmproc.hpp delete mode 100644 components/eamxx/src/share/python/pyunits.hpp diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/physics/p3/python/pyp3.cpp index bbd5c484cbc0..c777a9585973 100644 --- a/components/eamxx/src/physics/p3/python/pyp3.cpp +++ b/components/eamxx/src/physics/p3/python/pyp3.cpp @@ -1,7 +1,7 @@ #include "physics/p3/eamxx_p3_process_interface.hpp" -#include "share/grid/point_grid.hpp" #include "pygrid.hpp" #include "pyfield.hpp" +#include "pyatmproc.hpp" #include @@ -9,40 +9,17 @@ namespace scream { namespace py = pybind11; -struct PyP3 { - std::shared_ptr p3; - +struct PyP3 : public PyAtmProc { PyP3 (const PyGrid& pygrid) { ekat::Comm comm(MPI_COMM_WORLD); ekat::ParameterList pl; pl.set("max_total_ni",10.0); - p3 = create_atmosphere_process(comm,pl); + ap = create_atmosphere_process(comm,pl); // Create a grids manager on the fly auto gm = std::make_shared(pygrid.grid); - p3->set_grids(gm); - } - - void set_fields(const std::vector& pyfields) { - for (const auto& pyf : pyfields) { - const auto& fid = pyf.f.get_header().get_identifier(); - if (p3->has_required_field(fid)) { - p3->set_required_field(pyf.f.get_const()); - } - if (p3->has_computed_field(fid)) { - p3->set_computed_field(pyf.f); - } - } - } - - void initialize () { - util::TimeStamp ts; - p3->initialize(ts,RunType::Initial); - } - - void cleanup () { - p3 = nullptr; + ap->set_grids(gm); } }; @@ -50,13 +27,9 @@ PYBIND11_MODULE (pyp3,m) { m.doc() = "Python interfaces to P3Microphysics"; - py::class_(m,"P3") - .def(py::init()) - .def("set_fields",&PyP3::set_fields) - .def("initialize",&PyP3::initialize) - .def("cleanup",&PyP3::cleanup); + py::class_(m,"P3") + .def(py::init()); } } // namespace scream - diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp new file mode 100644 index 000000000000..5302772444e8 --- /dev/null +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -0,0 +1,84 @@ +#ifndef PYATMPROC_HPP +#define PYATMPROC_HPP + +#include "share/atm_process/atmosphere_process.hpp" +#include "pygrid.hpp" +#include "pyfield.hpp" + +#include + +namespace scream { + +struct PyAtmProc { + std::shared_ptr ap; + std::map fields; + + std::map create_fields () { + std::map pyfields; + for (const auto& req : ap->get_required_field_requests()) { + if (pyfields.count(req.fid.name())==0) { + pyfields.emplace(req.fid.name(),PyField(req.fid,req.pack_size)); + } + } + for (const auto& req : ap->get_computed_field_requests()) { + if (pyfields.count(req.fid.name())==0) { + pyfields.emplace(req.fid.name(),PyField(req.fid,req.pack_size)); + } + } + + return pyfields; + } + + void set_fields(const std::map& pyfields) { + for (const auto& it : pyfields) { + const auto& f = it.second.f; + const auto& fid = f.get_header().get_identifier(); + if (ap->has_required_field(fid)) { + ap->set_required_field(f.get_const()); + fields[fid.name()] = it.second; + } + if (ap->has_computed_field(fid)) { + ap->set_computed_field(f); + fields[fid.name()] = it.second; + } + } + } + + pybind11::array get_arr(const std::string& name) { + auto it = fields.find(name); + + auto fnames = [&]() { + std::string s; + for (auto it : fields) { + if (s=="") { + s += it.first; + } else { + s += ", " + it.first; + } + } + return s; + }; + EKAT_REQUIRE_MSG (it!=fields.end(), + "Error! Field not found in list of P3 fields.\n" + " - field name: " + name + "\n" + " - p3 fields: " + fnames() + "\n"); + + return it->second.get(); + } + + // If running as part of a process group, call the second function, after + // manually creating/setting the fields + void initialize () { + set_fields(create_fields()); + util::TimeStamp ts; + ap->initialize(ts,RunType::Initial); + } + void initialize_without_creating_fields() { + util::TimeStamp ts; + ap->initialize(ts,RunType::Initial); + } +}; + +} // namespace scream + +#endif // PYATMPROC_HPP diff --git a/components/eamxx/src/share/python/pyfield.hpp b/components/eamxx/src/share/python/pyfield.hpp index 2891998e9d3a..ace027fda4c4 100644 --- a/components/eamxx/src/share/python/pyfield.hpp +++ b/components/eamxx/src/share/python/pyfield.hpp @@ -16,82 +16,43 @@ struct PyField { // Create empty field PyField () = default; - PyField(const FieldIdentifier& fid) + PyField(const FieldIdentifier& fid, + const int pack_size = 1) { - create(fid); + f = Field(fid); + f.get_header().get_alloc_properties().request_allocation(pack_size); + f.allocate_view(); } - // Create field and allocate memory - PyField(const std::string& n, - const std::vector& d) - { - std::vector t (d.size(),FieldTag::Component); - FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); - create(fid); - } + pybind11::array get () const { - // Create field from pre-existing python nd array - PyField(const std::string& n, - pybind11::array_t arr) - { - int rank = arr.ndim(); - std::vector d(rank,-1); - for (int n=0; nget_identifier().name() + "\n"); + pybind11::dtype dt; + + switch (fid.data_type()) { + case DataType::IntType: dt = pybind11::dtype::of(); break; + case DataType::FloatType: dt = pybind11::dtype::of(); break; + case DataType::DoubleType: dt = pybind11::dtype::of(); break; + default: EKAT_ERROR_MSG ("Unrecognized/unsupported data type.\n"); } - std::vector t (rank,FieldTag::Component); - FieldIdentifier fid(n,FieldLayout(t,d),ekat::units::Units::invalid(),""); - create(fid,arr); - } - PyField(const FieldIdentifier& fid, - pybind11::array_t arr) - { - create(fid,arr); + pybind11::array::ShapeContainer shape (fid.get_layout().dims()); + auto data = f.get_internal_view_data_unsafe(); + // auto data = reinterpret_cast(f.get_internal_view_data_unsafe()); + return pybind11::array(dt,shape,data); } void print() const { print_field_hyperslab(f); } - - void cleanup () { - f = Field(); - } - -private: - void create (const FieldIdentifier& fid) - { - f = Field(fid); - f.allocate_view(); - } - - void create (const FieldIdentifier& fid, - pybind11::array_t arr) - { - int rank = arr.ndim(); - EKAT_REQUIRE_MSG (rank==fid.get_layout().rank(), - "Error! Rank mismatch between input FieldIdentifier and pybind11::array_t.\n" - " - field name: " + fid.name() + "\n" - " - identifier rank: " + std::to_string(fid.get_layout().rank()) + "\n" - " - array_t rank : " + std::to_string(rank) + "\n"); - switch (rank) { - case 1: - { - Field::view_dev_t v(arr.mutable_data(0),arr.shape(0)); - f = Field(fid,v); - break; - } - case 2: - { - Field::view_dev_t v(arr.mutable_data(0,0),arr.shape(0),arr.shape(1)); - f = Field(fid,v); - v(0,0) = -1; - break; - } - default: - EKAT_ERROR_MSG ("AAARGH!\n"); - } - } }; } // namespace scream diff --git a/components/eamxx/src/share/python/pygrid.hpp b/components/eamxx/src/share/python/pygrid.hpp index 12cee80ba9da..9134c3e473ef 100644 --- a/components/eamxx/src/share/python/pygrid.hpp +++ b/components/eamxx/src/share/python/pygrid.hpp @@ -3,7 +3,6 @@ #include "share/grid/grids_manager.hpp" #include "share/grid/point_grid.hpp" -#include "pyunits.hpp" #include "pyfield.hpp" #include @@ -11,6 +10,7 @@ namespace scream { // Small grids manager class, to hold a pre-built grid +// We will use this to build a GM on the fly from a single grid class SingleGridGM : public GridsManager { public: @@ -42,95 +42,9 @@ struct PyGrid { } // 2d scalar, managed/unmanaged - PyField scalar2d (const std::string& name, - const PyUnits& u) - { - FieldIdentifier fid(name,grid->get_2d_scalar_layout(),u.units,grid->name()); - return PyField(fid); - } - PyField scalar2d (const std::string& name, - const PyUnits& u, - pybind11::array_t arr) - { - FieldIdentifier fid(name,grid->get_2d_scalar_layout(),u.units,grid->name()); - return PyField(fid,arr); - } - // 2d vector, managed/unmanaged - PyField vector2d (const std::string& name, - const PyUnits& u, - int vdim) - { - FieldIdentifier fid(name,grid->get_2d_vector_layout(vdim),u.units,grid->name()); - return PyField(fid); - } - PyField vector2d (const std::string& name, - const PyUnits& u, - int vdim, - pybind11::array_t arr) - { - FieldIdentifier fid(name,grid->get_2d_vector_layout(vdim),u.units,grid->name()); - return PyField(fid,arr); - } - // 3d midpoints scalar, managed/unmanaged - PyField scalar3d_mid (const std::string& name, - const PyUnits& u) - { - FieldIdentifier fid(name,grid->get_3d_scalar_layout(true),u.units,grid->name()); - return PyField(fid); - } - PyField scalar3d_mid (const std::string& name, - const PyUnits& u, - pybind11::array_t arr) - { - FieldIdentifier fid(name,grid->get_3d_scalar_layout(true),u.units,grid->name()); - return PyField(fid,arr); - } - // 3d interfaces scalar, managed/unmanaged - PyField scalar3d_int (const std::string& name, - const PyUnits& u) - { - FieldIdentifier fid(name,grid->get_3d_scalar_layout(false),u.units,grid->name()); - return PyField(fid); - } - PyField scalar3d_int (const std::string& name, - const PyUnits& u, - pybind11::array_t arr) - { - FieldIdentifier fid(name,grid->get_3d_scalar_layout(false),u.units,grid->name()); - return PyField(fid,arr); - } -private: - PyField create (const FieldIdentifier& fid, - pybind11::array_t arr) - { - const auto rank = fid.get_layout().rank(); - PyField pyf; - EKAT_REQUIRE_MSG (rank==arr.ndim(), - "Error! Rank mismatch between input FieldIdentifier and pybind11::array_t.\n" - " - field name: " + fid.name() + "\n" - " - identifier rank: " + std::to_string(rank) + "\n" - " - array_t rank : " + std::to_string(arr.ndim()) + "\n"); - switch (rank) { - case 1: - { - Field::view_dev_t v(arr.mutable_data(0),arr.shape(0)); - pyf.f = Field(fid,v); - break; - } - case 2: - { - Field::view_dev_t v(arr.mutable_data(0,0),arr.shape(0),arr.shape(1)); - pyf.f = Field(fid,v); - break; - } - default: - EKAT_ERROR_MSG ("AAARGH!\n"); - } - return pyf; - } }; } // namespace scream diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp index 6ff3dc0cb40c..213c470b353b 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/share/python/pyscream.cpp @@ -1,7 +1,7 @@ #include "scream_session.hpp" #include "pyfield.hpp" #include "pygrid.hpp" -#include "pyunits.hpp" +#include "pyatmproc.hpp" #include #include @@ -25,49 +25,20 @@ PYBIND11_MODULE (pyscream,m) { m.def("init",&initialize); m.def("finalize",&finalize); - // Units - py::module units = m.def_submodule("units"); - py::class_(units,"Units") - .def(py::init<>()) - .def(py::init()) - .def("str",&PyUnits::str) - .def("__mul__",[](const PyUnits& x, const PyUnits& y) { return PyUnits(x.units*y.units); }) - .def("__truediv__",[](const PyUnits& x, const PyUnits& y) { return PyUnits(x.units/y.units); }); - - units.def("pow",[](const PyUnits& x, const int n) { return PyUnits(pow(x.units,ekat::RationalConstant(n))); }); - units.def("root",[](const PyUnits& x, const int n) { return PyUnits(pow(x.units,ekat::RationalConstant(1,n))); }); - units.attr("one") = &short_units::one; - units.attr("m") = &short_units::m; - units.attr("s") = &short_units::s; - units.attr("kg") = &short_units::kg; - units.attr("K") = &short_units::K; - units.attr("N") = &short_units::N; - units.attr("J") = &short_units::J; - units.attr("W") = &short_units::W; - units.attr("Pa") = &short_units::Pa; - // Field class py::class_(m,"Field") - .def(py::init<>()) - .def(py::init&>()) - .def(py::init - >()) - .def("print",&PyField::print) - .def("cleanup",&PyField::cleanup); + .def("get",&PyField::get) + .def("print",&PyField::print); // Grid py::class_(m,"Grid") - .def(py::init()) - .def("scalar2d",py::overload_cast(&PyGrid::scalar2d)) - .def("scalar2d",py::overload_cast>(&PyGrid::scalar2d)) - .def("vector2d",py::overload_cast(&PyGrid::vector2d)) - .def("vector2d",py::overload_cast>(&PyGrid::vector2d)) - .def("scalar3d_mid",py::overload_cast(&PyGrid::scalar3d_mid)) - .def("scalar3d_mid",py::overload_cast>(&PyGrid::scalar3d_mid)) - .def("scalar3d_int",py::overload_cast(&PyGrid::scalar3d_int)) - .def("scalar3d_int",py::overload_cast>(&PyGrid::scalar3d_int)); + .def(py::init()); + + // Atm process + py::class_(m,"AtmProc") + .def(py::init<>()) + .def("get_arr",&PyAtmProc::get_arr) + .def("initialize",&PyAtmProc::initialize); } } // namespace scream diff --git a/components/eamxx/src/share/python/pyunits.hpp b/components/eamxx/src/share/python/pyunits.hpp deleted file mode 100644 index 4b4e238fd10e..000000000000 --- a/components/eamxx/src/share/python/pyunits.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef PYUNITS_HPP -#define PYUNITS_HPP - -#include - -#include - -namespace scream { - -struct PyUnits { - ekat::units::Units units; - - PyUnits () - : units(ekat::RationalConstant(0)) - { - // Nothing to do here - } - PyUnits (const ekat::units::Units& src) - : units(src) - { - // Nothing to do here - } - - std::string str() { - return units.to_string(); - } - - PyUnits (const PyUnits&) = default; -}; - -namespace short_units { - -PyUnits one(ekat::units::Units::nondimensional()); -PyUnits m(ekat::units::m); -PyUnits s(ekat::units::s); -PyUnits kg(ekat::units::kg); -PyUnits A(ekat::units::A); -PyUnits mol(ekat::units::mol); -PyUnits cd(ekat::units::cd); -PyUnits K(ekat::units::K); - -PyUnits N (ekat::units::N); -PyUnits J (ekat::units::J); -PyUnits W (ekat::units::W); -PyUnits Pa (ekat::units::Pa); -} - -} // namespace scream - -#endif // PYUNITS_HPP From 81093945cd1da848eb934289d410c8d6c21a9471 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 17:06:39 -0600 Subject: [PATCH 393/476] EAMxx: add possibility to read in IC file in pyscream --- .../eamxx/src/physics/p3/python/pyp3.cpp | 4 +- .../eamxx/src/share/python/pyatmproc.hpp | 43 +++++++++++++++++-- .../eamxx/src/share/python/pyscream.cpp | 5 ++- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/physics/p3/python/pyp3.cpp index c777a9585973..831eb5cb5406 100644 --- a/components/eamxx/src/physics/p3/python/pyp3.cpp +++ b/components/eamxx/src/physics/p3/python/pyp3.cpp @@ -10,7 +10,9 @@ namespace scream { namespace py = pybind11; struct PyP3 : public PyAtmProc { - PyP3 (const PyGrid& pygrid) { + PyP3 (const PyGrid& pygrid) + : PyAtmProc(pygrid) + { ekat::Comm comm(MPI_COMM_WORLD); ekat::ParameterList pl; pl.set("max_total_ni",10.0); diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp index 5302772444e8..7a4bb47095ae 100644 --- a/components/eamxx/src/share/python/pyatmproc.hpp +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -2,6 +2,7 @@ #define PYATMPROC_HPP #include "share/atm_process/atmosphere_process.hpp" +#include "share/io/scorpio_input.hpp" #include "pygrid.hpp" #include "pyfield.hpp" @@ -11,8 +12,15 @@ namespace scream { struct PyAtmProc { std::shared_ptr ap; + PyGrid pygrid; std::map fields; + PyAtmProc (const PyGrid& pyg) + : pygrid(pyg) + { + // Nothing to do here + } + std::map create_fields () { std::map pyfields; for (const auto& req : ap->get_required_field_requests()) { @@ -69,13 +77,40 @@ struct PyAtmProc { // If running as part of a process group, call the second function, after // manually creating/setting the fields void initialize () { + // Create fields set_fields(create_fields()); - util::TimeStamp ts; - ap->initialize(ts,RunType::Initial); + + // TODO: should we allow setting this? + util::TimeStamp t0({2000,1,1},{0,0,0}); + ap->initialize(t0,RunType::Initial); } void initialize_without_creating_fields() { - util::TimeStamp ts; - ap->initialize(ts,RunType::Initial); + // TODO: should we allow setting this? + util::TimeStamp t0({2000,1,1},{0,0,0}); + ap->initialize(t0,RunType::Initial); + } + void initialize (const std::string& ic_filename) { + // Create fields + set_fields(create_fields()); + + // Get input fields, and read them from file + std::vector ic_fields; + for (auto it : fields) { + const auto& f = it.second.f; + if (ap->has_required_field(f.get_header().get_identifier())) { + ic_fields.push_back(f); + } + } + AtmosphereInput reader (ic_filename,pygrid.grid,ic_fields,true); + reader.read_variables(); + + // TODO: should we allow setting this? + util::TimeStamp t0({2000,1,1},{0,0,0}); + for (auto& f : ic_fields) { + f.get_header().get_tracking().update_time_stamp(t0); + } + + ap->initialize(t0,RunType::Initial); } }; diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp index 213c470b353b..cc9198347d56 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/share/python/pyscream.cpp @@ -36,9 +36,10 @@ PYBIND11_MODULE (pyscream,m) { // Atm process py::class_(m,"AtmProc") - .def(py::init<>()) + .def(py::init()) .def("get_arr",&PyAtmProc::get_arr) - .def("initialize",&PyAtmProc::initialize); + .def("initialize",py::overload_cast<>(&PyAtmProc::initialize)) + .def("initialize",py::overload_cast(&PyAtmProc::initialize)); } } // namespace scream From d2d1f80e4edf1b0beb4b3ca92b2624bf9cbabf79 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 18:42:34 -0600 Subject: [PATCH 394/476] EAMxx: add possibility of reading field from an IC file in pyscream --- .../eamxx/src/physics/p3/python/pyp3.cpp | 2 + .../eamxx/src/share/python/pyatmproc.hpp | 47 +++++++++---------- components/eamxx/src/share/python/pyfield.hpp | 3 ++ .../eamxx/src/share/python/pyscream.cpp | 8 +++- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/physics/p3/python/pyp3.cpp index 831eb5cb5406..863c62a66845 100644 --- a/components/eamxx/src/physics/p3/python/pyp3.cpp +++ b/components/eamxx/src/physics/p3/python/pyp3.cpp @@ -22,6 +22,8 @@ struct PyP3 : public PyAtmProc { // Create a grids manager on the fly auto gm = std::make_shared(pygrid.grid); ap->set_grids(gm); + + set_fields(create_fields()); } }; diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp index 7a4bb47095ae..b0a333738875 100644 --- a/components/eamxx/src/share/python/pyatmproc.hpp +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -14,11 +14,14 @@ struct PyAtmProc { std::shared_ptr ap; PyGrid pygrid; std::map fields; + util::TimeStamp t0; + ATMBufferManager buffer; PyAtmProc (const PyGrid& pyg) : pygrid(pyg) + , t0({2000,1,1},{0,0,0}) { - // Nothing to do here + // TODO: make t0 configurable? } std::map create_fields () { @@ -77,40 +80,34 @@ struct PyAtmProc { // If running as part of a process group, call the second function, after // manually creating/setting the fields void initialize () { - // Create fields - set_fields(create_fields()); - - // TODO: should we allow setting this? - util::TimeStamp t0({2000,1,1},{0,0,0}); - ap->initialize(t0,RunType::Initial); - } - void initialize_without_creating_fields() { - // TODO: should we allow setting this? - util::TimeStamp t0({2000,1,1},{0,0,0}); + int nbytes = ap->requested_buffer_size_in_bytes (); + buffer.request_bytes(nbytes); + buffer.allocate(); + ap->init_buffers(buffer); ap->initialize(t0,RunType::Initial); } - void initialize (const std::string& ic_filename) { - // Create fields - set_fields(create_fields()); - // Get input fields, and read them from file + void read_ic (const std::string& ic_filename, bool default_init) { + // Get input fields, and read them from file (if present). + // If field is not in the IC, user is responsible for setting + // it to an initial value std::vector ic_fields; + scorpio::register_file(ic_filename,scorpio::Read); for (auto it : fields) { - const auto& f = it.second.f; + auto& f = it.second.f; if (ap->has_required_field(f.get_header().get_identifier())) { - ic_fields.push_back(f); + if (scorpio::has_var(ic_filename,f.name())) { + ic_fields.push_back(f); + f.get_header().get_tracking().update_time_stamp(t0); + } else if (default_init) { + f.deep_copy(0); + f.get_header().get_tracking().update_time_stamp(t0); + } } } AtmosphereInput reader (ic_filename,pygrid.grid,ic_fields,true); reader.read_variables(); - - // TODO: should we allow setting this? - util::TimeStamp t0({2000,1,1},{0,0,0}); - for (auto& f : ic_fields) { - f.get_header().get_tracking().update_time_stamp(t0); - } - - ap->initialize(t0,RunType::Initial); + scorpio::release_file(ic_filename); } }; diff --git a/components/eamxx/src/share/python/pyfield.hpp b/components/eamxx/src/share/python/pyfield.hpp index ace027fda4c4..552b42d7e98a 100644 --- a/components/eamxx/src/share/python/pyfield.hpp +++ b/components/eamxx/src/share/python/pyfield.hpp @@ -50,6 +50,9 @@ struct PyField { return pybind11::array(dt,shape,data); } + void sync_to_host () { + f.sync_to_host(); + } void print() const { print_field_hyperslab(f); } diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp index cc9198347d56..8ec51b0bf075 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/share/python/pyscream.cpp @@ -12,8 +12,11 @@ namespace scream { void initialize () { initialize_scream_session(true); + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); } void finalize () { + scorpio::finalize_subsystem(); finalize_scream_session(); } @@ -28,6 +31,7 @@ PYBIND11_MODULE (pyscream,m) { // Field class py::class_(m,"Field") .def("get",&PyField::get) + .def("sync_to_host",&PyField::sync_to_host) .def("print",&PyField::print); // Grid @@ -38,8 +42,8 @@ PYBIND11_MODULE (pyscream,m) { py::class_(m,"AtmProc") .def(py::init()) .def("get_arr",&PyAtmProc::get_arr) - .def("initialize",py::overload_cast<>(&PyAtmProc::initialize)) - .def("initialize",py::overload_cast(&PyAtmProc::initialize)); + .def("initialize",&PyAtmProc::initialize) + .def("read_ic",&PyAtmProc::read_ic,py::arg("ic_filename") = "",py::arg("default_init") = true); } } // namespace scream From 5a4742806abf39763353ec94021dbf9f53b430fb Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 30 May 2024 17:56:28 -0500 Subject: [PATCH 395/476] change name to pyeamxx --- .../eamxx/src/python/pyeamxx/__init__.py | 3 +++ .../eamxx/src/python/screaminpy/__init__.py | 7 ------ components/eamxx/src/python/setup.py | 22 +++++++------------ .../eamxx/src/python/tests/basic_test.py | 2 +- 4 files changed, 12 insertions(+), 22 deletions(-) create mode 100644 components/eamxx/src/python/pyeamxx/__init__.py delete mode 100644 components/eamxx/src/python/screaminpy/__init__.py diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py new file mode 100644 index 000000000000..4b4ee19cc170 --- /dev/null +++ b/components/eamxx/src/python/pyeamxx/__init__.py @@ -0,0 +1,3 @@ +""" to get going """ + +import pyscream diff --git a/components/eamxx/src/python/screaminpy/__init__.py b/components/eamxx/src/python/screaminpy/__init__.py deleted file mode 100644 index 1817c6642c4f..000000000000 --- a/components/eamxx/src/python/screaminpy/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" to get going """ - -__version__ = "1.0.0" - -# expose the executable we just packaged -# so that we can do `from screaminpy import pyscream` -import pyscream diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py index f9cc91a1e4a0..76371840008c 100644 --- a/components/eamxx/src/python/setup.py +++ b/components/eamxx/src/python/setup.py @@ -5,23 +5,17 @@ class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -# For all of this to work, we either need to: -# 1. build the extension as we build the python package; or -# 2. simply package the extension as "data". -# The latter is actually quite annoying, but we may have to do it anyway. -# So, bring the extension to the dir of this file, before triggering - setup( - name='screaminpy', - version='1.0.0', - author="screamers", - description='screaminpy wrapper', - packages=['', 'screaminpy'], + name='pyeamxx', + version='0.0.1', + author='E3SM SCREAM', + description='EAMxx wrapper', + packages=['','pyeamxx'], package_data={ - 'screaminpy': ['*.py'], - '': ['pyscream.*so'], + '': ['*.*so'], }, - distclass=BinaryDistribution + distclass=BinaryDistribution, + zip_safe=False ) # TODOs: diff --git a/components/eamxx/src/python/tests/basic_test.py b/components/eamxx/src/python/tests/basic_test.py index 256870e53de3..26e43cad0d2b 100644 --- a/components/eamxx/src/python/tests/basic_test.py +++ b/components/eamxx/src/python/tests/basic_test.py @@ -1,5 +1,5 @@ # luca's verbatim example works from any dir now! -from screaminpy import pyscream as ps +from pyeamxx import pyscream as ps import numpy as np ps.init() From 8bc5feb66e4616e8e7d8f55304044fbeeb9f04d7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 29 May 2024 18:42:34 -0600 Subject: [PATCH 396/476] EAMxx: add possibility of reading field from an IC file in pyeamxx --- .../eamxx/src/share/python/pyatmproc.hpp | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp index b0a333738875..ce771b49d616 100644 --- a/components/eamxx/src/share/python/pyatmproc.hpp +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -79,18 +79,28 @@ struct PyAtmProc { // If running as part of a process group, call the second function, after // manually creating/setting the fields - void initialize () { + void initialize (const std::string& t0_str) { int nbytes = ap->requested_buffer_size_in_bytes (); buffer.request_bytes(nbytes); buffer.allocate(); ap->init_buffers(buffer); + + t0 = util::str_to_time_stamp(t0_str); + for (auto it : fields) { + auto& f = it.second.f; + if (ap->has_required_field(f.get_header().get_identifier())) { + f.get_header().get_tracking().update_time_stamp(t0); + } + } + ap->initialize(t0,RunType::Initial); } - void read_ic (const std::string& ic_filename, bool default_init) { + pybind11::list read_ic (const std::string& ic_filename) { // Get input fields, and read them from file (if present). // If field is not in the IC, user is responsible for setting // it to an initial value + std::vector missing; std::vector ic_fields; scorpio::register_file(ic_filename,scorpio::Read); for (auto it : fields) { @@ -98,16 +108,16 @@ struct PyAtmProc { if (ap->has_required_field(f.get_header().get_identifier())) { if (scorpio::has_var(ic_filename,f.name())) { ic_fields.push_back(f); - f.get_header().get_tracking().update_time_stamp(t0); - } else if (default_init) { - f.deep_copy(0); - f.get_header().get_tracking().update_time_stamp(t0); + } else { + missing.push_back(f.name()); } } } AtmosphereInput reader (ic_filename,pygrid.grid,ic_fields,true); reader.read_variables(); scorpio::release_file(ic_filename); + + return pybind11::cast(missing); } }; From adf01c02e55df4fd0483a37fbe59be862b3651f1 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 19:48:02 -0600 Subject: [PATCH 397/476] EAMxx: fix wrong error msg in cmake macro --- components/eamxx/cmake/CompareNCFiles.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/eamxx/cmake/CompareNCFiles.cmake b/components/eamxx/cmake/CompareNCFiles.cmake index e17bc76b3a96..aa9fbc0a7064 100644 --- a/components/eamxx/cmake/CompareNCFiles.cmake +++ b/components/eamxx/cmake/CompareNCFiles.cmake @@ -13,19 +13,19 @@ function(CompareNCFiles) set (argsMv LABELS FIXTURES_REQUIRED) cmake_parse_arguments(PARSE "${options}" "${args1v}" "${argsMv}" ${ARGN}) - CheckMacroArgs(CompareNCFilesFamily PARSE "${options}" "${args1v}" "${argsMv}") + CheckMacroArgs(CompareNCFiles PARSE "${options}" "${args1v}" "${argsMv}") # Sanity checks if (NOT PARSE_TEST_NAME) - message ("Error! CompareNCFilesPair requires the keyword argument TEST_NAME") + message ("Error! CompareNCFiles requires the keyword argument TEST_NAME") message (FATAL_ERROR "Aborting...") endif() if (NOT PARSE_SRC_FILE) - message ("Error! CompareNCFilesPair requires the keyword argument SRC_FILE") + message ("Error! CompareNCFiles requires the keyword argument SRC_FILE") message (FATAL_ERROR "Aborting...") endif() if (NOT PARSE_TGT_FILE) - message ("Error! CompareNCFilesPair requires the keyword argument TGT_FILE") + message ("Error! CompareNCFiles requires the keyword argument TGT_FILE") message (FATAL_ERROR "Aborting...") endif() From 5130448e7a70e803bad8ec476a0dfc7f5d7dbb63 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 21:53:29 -0600 Subject: [PATCH 398/476] EAMxx: implement OutputManager destructor to call finalize Also, change finalize impl, to avoid infinite recursion --- .../src/share/io/scream_output_manager.cpp | 28 +++++++++++++++++-- .../src/share/io/scream_output_manager.hpp | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 9e25c0aa7483..bc09b86cf4b7 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -17,6 +17,12 @@ namespace scream { +OutputManager:: +~OutputManager () +{ + finalize(); +} + void OutputManager:: setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const std::shared_ptr& field_mgr, @@ -558,9 +564,25 @@ void OutputManager::finalize() scorpio::release_file (m_checkpoint_file_specs.filename); } - // Swapping with an empty mgr is the easiest way to cleanup. - OutputManager other; - std::swap(*this,other); + // Reset everything to a default constructed object. + // NOTE: it's themptying to std::swap(*this,OutputManager()), + // but that calls ~OutputManager() on the destructor, + // which in turns calls finalize, causing endless recursion. + m_output_streams = {}; + m_geo_data_streams = {}; + m_globals.clear(); + m_io_comm = {}; + m_params = {}; + m_filename_prefix = {}; + m_time_bnds = {}; + m_avg_type = {}; + m_output_control = {}; + m_checkpoint_control = {}; + m_output_file_specs = {}; + m_checkpoint_file_specs = {}; + m_case_t0 = {}; + m_run_t0 = {}; + m_atm_logger = {}; } long long OutputManager::res_dep_memory_footprint () const { diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index e1f58d93fede..d25dd037c67f 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -68,7 +68,7 @@ class OutputManager // Constructor(s) & Destructor OutputManager () = default; - virtual ~OutputManager () = default; + virtual ~OutputManager (); // Set up the manager, creating all output streams. Inputs: // - params: the parameter list with file/fields info, as well as method of output options From ef6823285338064dd3a651e23bb1ee5b94068bbf Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:19:22 -0600 Subject: [PATCH 399/476] EAMxx: add getter to atm proc class --- components/eamxx/src/share/atm_process/atmosphere_process.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.hpp b/components/eamxx/src/share/atm_process/atmosphere_process.hpp index a25879b80218..3150f56c9f9a 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.hpp @@ -283,6 +283,10 @@ class AtmosphereProcess : public ekat::enable_shared_from_this get_logger () const { + return m_atm_logger; + } + protected: // Sends a message to the atm log From 0feb46c3d130a6d1d8698479b80cc633ac3dbfdd Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 21:54:42 -0600 Subject: [PATCH 400/476] EAMxx: fix nano bug in atm driver Avoid local temp copy when iterating over output managers. With copy, we get a bug due to double call to scorpio cleanup methods --- components/eamxx/src/control/atmosphere_driver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 60722664630d..ad1cd2b5a387 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -1608,7 +1608,7 @@ void AtmosphereDriver::run (const int dt) { // that quantity at the beginning of the timestep. Or they may need to store // the timestamp at the beginning of the timestep, so that we can compute // dt at the end. - for (auto it : m_output_managers) { + for (auto& it : m_output_managers) { it.init_timestep(m_current_ts,dt); } From 90e0365f13edbdab86ca4ff5e15b11bf73e76e72 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:02:34 -0600 Subject: [PATCH 401/476] EAMxx: add run method to PyAtmProc in pyeamxx --- components/eamxx/src/share/python/pyatmproc.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp index ce771b49d616..748bd0be7aed 100644 --- a/components/eamxx/src/share/python/pyatmproc.hpp +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -15,6 +15,7 @@ struct PyAtmProc { PyGrid pygrid; std::map fields; util::TimeStamp t0; + util::TimeStamp time; ATMBufferManager buffer; PyAtmProc (const PyGrid& pyg) @@ -85,7 +86,7 @@ struct PyAtmProc { buffer.allocate(); ap->init_buffers(buffer); - t0 = util::str_to_time_stamp(t0_str); + time = t0 = util::str_to_time_stamp(t0_str); for (auto it : fields) { auto& f = it.second.f; if (ap->has_required_field(f.get_header().get_identifier())) { @@ -119,6 +120,11 @@ struct PyAtmProc { return pybind11::cast(missing); } + + void run (double dt) { + ap->run(dt); + time += dt; + } }; } // namespace scream From 49db86d21287a5dc03754a8a28ae8d0b55118251 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 30 May 2024 21:31:36 -0700 Subject: [PATCH 402/476] Changes TeamThreadRange to TeamVectorRange; [=] to [&]; removes aitken dia func with deep_copy --- .../physics/mam/eamxx_mam_aci_functions.hpp | 51 +++++-------------- .../mam/eamxx_mam_aci_process_interface.cpp | 32 ++++-------- 2 files changed, 23 insertions(+), 60 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 96e31f4d6c79..a11d9e066d7c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -23,12 +23,12 @@ void compute_w0_and_rho(const haero::ThreadTeam &team, // Gas constant for dry air [J/(kg*K) or J/Kg/K] static constexpr auto rair = C::Rair; Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { w0(icol, kk) = 0; rho(icol, kk) = -999.0; }); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); }); @@ -80,7 +80,7 @@ void compute_tke_using_w_sec(const haero::ThreadTeam &team, // output MAMAci::view_2d tke) { Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev + 1), + Kokkos::TeamVectorRange(team, 0u, nlev + 1), KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); } void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, @@ -104,13 +104,13 @@ void compute_subgrid_scale_velocities( // More refined computation of sub-grid vertical velocity // Set to be zero at the surface by initialization. Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { wsub(icol, kk) = wsubmin; wsubice(icol, kk) = 0.001; wsig(icol, kk) = 0.001; }); Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * (2.0 / 3.0)); wsig(icol, kk) = @@ -135,33 +135,6 @@ void compute_subgrid_scale_velocities( }); } -KOKKOS_INLINE_FUNCTION -void compute_aitken_dry_diameter(const haero::ThreadTeam &team, - const MAMAci::const_view_3d dgnum, - const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - const int aitken_idx = static_cast(mam4::ModeIndex::Aitken); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - aitken_dry_dia(icol, kk) = dgnum(icol, aitken_idx, kk); - }); -} -void compute_aitken_dry_diameter(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_3d dgnum, - const int top_lev, const int nlev, - // output - MAMAci::view_2d aitken_dry_dia) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_aitken_dry_diameter(team, dgnum, icol, top_lev, nlev, - // output - aitken_dry_dia); - }); -} - void compute_nucleate_ice_tendencies( const mam4::NucleateIce &nucleate_ice, haero::ThreadTeamPolicy team_policy, const mam_coupling::DryAtmosphere &dry_atmosphere, @@ -246,7 +219,7 @@ void store_liquid_cloud_fraction( // cut-off for cloud amount (ice or liquid) static constexpr auto qsmall = 1e-18; Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { if((qc(icol, kk) + qi(icol, kk)) > qsmall) { cloud_frac(icol, kk) = liqcldf(icol, kk); cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); @@ -281,7 +254,7 @@ void compute_recipical_pseudo_density(const haero::ThreadTeam &team, // output MAMAci::view_2d rpdel) { Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { + Kokkos::TeamVectorRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), "Error: pdel should be > 0.\n"); rpdel(icol, kk) = 1 / pdel(icol, kk); @@ -457,8 +430,8 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { + Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -553,7 +526,7 @@ void update_interstitial_aerosols_levs( const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], // output MAMAci::view_2d aero_mr) { - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [=](int kk) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; }); } @@ -700,8 +673,8 @@ void call_hetfrz_compute_tendencies( // assign cloud fraction Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, 0u, mam4::ndrop::pver), - [=](int klev) { + Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + [&](int klev) { diags.stratiform_cloud_fraction(klev) = haero_atm.cloud_fraction(klev); }); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 063535ccdab9..c5d6f5eb2043 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -26,13 +26,10 @@ FUTURE WORK: 7. Use std::string rather than c-strings 8. Remove a Kokkos:fence and combine two kernels while computing w_sec_int_ 9. Fix double counting of tracer advection by modifying SHOC. -10. A git issue for computing top_lev, moving liq_cldfrac in ACI and using TKE directly -11.Replace [=] with [&] -12. Remove all set_string and use pow -13. use "aitken_dry_dia = ekat::subview_1(dgnum,mam4::ModeIndex::Aitken);" +10. A git issue for computing top_lev, moving liq_cldfrac in ACI and using TKE +directly 14. Merge kernels so that we are not calling one from another -15. Use "TeamVectorRange" instead of thread -16. improve the way qqcw is populated +16. improve the way qqcw is populated 17.delete fence mentioned by Luca ----------------------------------------------------------------- */ @@ -83,10 +80,7 @@ void MAMAci::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers - //q_unit.set_string("kg/kg"); - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - //n_unit.set_string("#/kg"); + auto n_unit = 1 / kg; // units of number mixing ratios of tracers auto nondim = ekat::units::Units::nondimensional(); @@ -127,10 +121,8 @@ void MAMAci::set_grids( // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); - auto m2 = m * m; - //m2.set_string("m^2"); - auto s2 = s * s; - //s2.set_string("s^2"); + auto m2 = pow(m, 2); + auto s2 = pow(s, 2); // NOTE: w_variance im microp_aero.F90 in EAM is at "itim_old" dynamics time // step. Since, we are using SE dycore, itim_old is 1 which is equivalent to @@ -252,8 +244,7 @@ void MAMAci::set_grids( // units of number mixing ratios of tracers auto frz_unit = 1 / (cm * cm * cm * s); - //n_unit.set_string("1(cm^-3 s^-1)"); - // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] + // heterogeneous freezing by immersion nucleation [cm^-3 s^-1] add_field("hetfrz_immersion_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); @@ -586,11 +577,10 @@ void MAMAci::run_impl(const double dt) { // output wsub_, wsubice_, wsig_); - // We need dry diameter for only aitken mode - compute_aitken_dry_diameter(team_policy, dgnum_, top_lev_, nlev_, - // output - aitken_dry_dia_); + Kokkos::deep_copy( + aitken_dry_dia_, + ekat::subview_1(dgnum_, static_cast(mam4::ModeIndex::Aitken))); Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. @@ -622,7 +612,7 @@ void MAMAci::run_impl(const double dt) { // output kvh_int_); - Kokkos::fence(); + Kokkos::fence(); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies From 546c082d3f71699710419dadf7bc30d6ef364cd3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 21:56:00 -0600 Subject: [PATCH 403/476] EAMxx: add output capability to PyAtmProc class in pyeamxx --- .../eamxx/src/share/python/pyatmproc.hpp | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp index 748bd0be7aed..037ba273ec2c 100644 --- a/components/eamxx/src/share/python/pyatmproc.hpp +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -3,10 +3,14 @@ #include "share/atm_process/atmosphere_process.hpp" #include "share/io/scorpio_input.hpp" +#include "share/io/scream_output_manager.hpp" #include "pygrid.hpp" #include "pyfield.hpp" +#include + #include +#include namespace scream { @@ -18,6 +22,8 @@ struct PyAtmProc { util::TimeStamp time; ATMBufferManager buffer; + std::shared_ptr output_mgr; + PyAtmProc (const PyGrid& pyg) : pygrid(pyg) , t0({2000,1,1},{0,0,0}) @@ -25,6 +31,9 @@ struct PyAtmProc { // TODO: make t0 configurable? } + // I don't think virtual is needed, but just in case + virtual ~PyAtmProc () = default; + std::map create_fields () { std::map pyfields; for (const auto& req : ap->get_required_field_requests()) { @@ -121,9 +130,33 @@ struct PyAtmProc { return pybind11::cast(missing); } + void setup_output (const std::string& yaml_file) { + + // Load output params + auto params = ekat::parse_yaml_file(yaml_file); + + // Stuff all fields in a field manager + auto fm = std::make_shared(pygrid.grid); + fm->registration_begins(); + fm->registration_ends(); + for (auto it : fields) { + fm->add_field(it.second.f); + } + + // Create a grids manager on the fly + auto gm = std::make_shared(pygrid.grid); + + output_mgr = std::make_shared(); + output_mgr->setup(pygrid.grid->get_comm(),params,fm,gm,t0,t0,false); + output_mgr->set_logger(ap->get_logger()); + } + void run (double dt) { ap->run(dt); time += dt; + if (output_mgr) { + output_mgr->run(time); + } } }; From 6ec365b6c40ae83a27b68f35713f9f9a06764c77 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:05:06 -0600 Subject: [PATCH 404/476] EAMxx: cleanup a bit pyscream.cpp --- .../eamxx/src/share/python/pyatmproc.hpp | 15 ++++++++++-- components/eamxx/src/share/python/pyfield.hpp | 9 ++++++++ components/eamxx/src/share/python/pygrid.hpp | 12 +++++----- .../eamxx/src/share/python/pyscream.cpp | 23 +++++-------------- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/share/python/pyatmproc.hpp index 037ba273ec2c..a78ef5624834 100644 --- a/components/eamxx/src/share/python/pyatmproc.hpp +++ b/components/eamxx/src/share/python/pyatmproc.hpp @@ -65,7 +65,7 @@ struct PyAtmProc { } } - pybind11::array get_arr(const std::string& name) { + PyField get_field(const std::string& name) { auto it = fields.find(name); auto fnames = [&]() { @@ -84,7 +84,7 @@ struct PyAtmProc { " - field name: " + name + "\n" " - p3 fields: " + fnames() + "\n"); - return it->second.get(); + return it->second; } // If running as part of a process group, call the second function, after @@ -160,6 +160,17 @@ struct PyAtmProc { } }; +// Register type in the py module +inline void pybind_pyatmproc(pybind11::module& m) +{ + pybind11::class_(m,"AtmProc") + .def(pybind11::init()) + .def("get_field",&PyAtmProc::get_field) + .def("initialize",&PyAtmProc::initialize) + .def("setup_output",&PyAtmProc::setup_output) + .def("run",&PyAtmProc::run) + .def("read_ic",&PyAtmProc::read_ic); +} } // namespace scream #endif // PYATMPROC_HPP diff --git a/components/eamxx/src/share/python/pyfield.hpp b/components/eamxx/src/share/python/pyfield.hpp index 552b42d7e98a..df6ed5012f31 100644 --- a/components/eamxx/src/share/python/pyfield.hpp +++ b/components/eamxx/src/share/python/pyfield.hpp @@ -58,6 +58,15 @@ struct PyField { } }; +inline void pybind_pyfield (pybind11::module& m) { + // Field class + pybind11::class_(m,"Field") + .def(pybind11::init<>()) + .def("get",&PyField::get) + .def("sync_to_host",&PyField::sync_to_host) + .def("print",&PyField::print); +} + } // namespace scream #endif // PYFIELD_HPP diff --git a/components/eamxx/src/share/python/pygrid.hpp b/components/eamxx/src/share/python/pygrid.hpp index 9134c3e473ef..3ffb413a8052 100644 --- a/components/eamxx/src/share/python/pygrid.hpp +++ b/components/eamxx/src/share/python/pygrid.hpp @@ -26,7 +26,7 @@ class SingleGridGM : public GridsManager protected: remapper_ptr_type do_create_remapper (const grid_ptr_type /* from_grid */, - const grid_ptr_type /* to_grid */) const + const grid_ptr_type /* to_grid */) const override { EKAT_ERROR_MSG ("Error! do_create_remapper not implemented for SingleGridGM.\n"); } @@ -40,13 +40,13 @@ struct PyGrid { ekat::Comm comm(MPI_COMM_WORLD); grid = create_point_grid(name,ncols,nlevs,comm); } - - // 2d scalar, managed/unmanaged - - - }; +inline void pybind_pygrid (pybind11::module& m) { + pybind11::class_(m,"Grid") + .def(pybind11::init()); +} + } // namespace scream #endif // PYGRID_HPP diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp index 8ec51b0bf075..dbf1c111a386 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/share/python/pyscream.cpp @@ -11,8 +11,8 @@ namespace py = pybind11; namespace scream { void initialize () { - initialize_scream_session(true); ekat::Comm comm(MPI_COMM_WORLD); + initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); } void finalize () { @@ -28,22 +28,11 @@ PYBIND11_MODULE (pyscream,m) { m.def("init",&initialize); m.def("finalize",&finalize); - // Field class - py::class_(m,"Field") - .def("get",&PyField::get) - .def("sync_to_host",&PyField::sync_to_host) - .def("print",&PyField::print); - - // Grid - py::class_(m,"Grid") - .def(py::init()); - - // Atm process - py::class_(m,"AtmProc") - .def(py::init()) - .def("get_arr",&PyAtmProc::get_arr) - .def("initialize",&PyAtmProc::initialize) - .def("read_ic",&PyAtmProc::read_ic,py::arg("ic_filename") = "",py::arg("default_init") = true); + // Call all other headers' registration routines + pybind_pyparamlist(m); + pybind_pyfield(m); + pybind_pygrid(m); + pybind_pyatmproc(m); } } // namespace scream From 9d3183fc9e3f7594f84b10b036afff0d86e7a1e7 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:05:59 -0600 Subject: [PATCH 405/476] EAMxx: add PyParamList to pyeamxx --- .../eamxx/src/share/python/pyparamlist.hpp | 86 +++++++++++++++++++ .../eamxx/src/share/python/pyscream.cpp | 1 + 2 files changed, 87 insertions(+) create mode 100644 components/eamxx/src/share/python/pyparamlist.hpp diff --git a/components/eamxx/src/share/python/pyparamlist.hpp b/components/eamxx/src/share/python/pyparamlist.hpp new file mode 100644 index 000000000000..30e4f7af48f1 --- /dev/null +++ b/components/eamxx/src/share/python/pyparamlist.hpp @@ -0,0 +1,86 @@ +#ifndef PYPARAMLIST_HPP +#define PYPARAMLIST_HPP + +#include + +#include +#include +#include + +namespace scream { + +struct PyParamList { + ekat::ParameterList pl; + + PyParamList(const pybind11::dict& d) { + parse_dict(d,pl); + } + + void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { + for (auto item : d) { + const std::string key = pybind11::str(item.first); + if (pybind11::isinstance(item.second)) { + auto pystr = pybind11::str(item.second); + p.set(key,pystr.cast()); + } else if (pybind11::isinstance(item.second)) { + auto pyint = pybind11::cast(item.second); + p.set(key,pyint.cast()); + } else if (pybind11::isinstance(item.second)) { + auto pyint = pybind11::cast(item.second); + p.set(key,pyint.cast()); + } else if (pybind11::isinstance(item.second)) { + auto pydouble = pybind11::cast(item.second); + p.set(key,pydouble.cast()); + } else if (pybind11::isinstance(item.second)) { + auto pylist = pybind11::cast(item.second); + parse_list(pylist,p,key); + } else if (pybind11::isinstance(item.second)) { + auto pydict = pybind11::cast(item.second); + parse_dict(pydict,p.sublist(key)); + } else { + EKAT_ERROR_MSG ("Unsupported/unrecognized dict entry type.\n"); + } + } + } + + void parse_list (const pybind11::list& l, ekat::ParameterList&p, const std::string& key) { + EKAT_REQUIRE_MSG (pybind11::len(l)>0, + "Error! Cannot deduce type for dictionary list entry '" + key + "'\n"); + auto first = l[0]; + bool are_ints = pybind11::isinstance(first); + bool are_floats = pybind11::isinstance(first); + bool are_strings = pybind11::isinstance(first); + if (are_ints) { + parse_list_impl(l,p,key); + } else if (are_floats) { + parse_list_impl(l,p,key); + } else if (are_strings) { + parse_list_impl(l,p,key); + } else { + EKAT_ERROR_MSG ("Unrecognized/unsupported list entry type.\n"); + } + } + + template + void parse_list_impl(const pybind11::list& l, ekat::ParameterList& p, const std::string& key) { + std::vector vals; + for (auto item : l) { + EKAT_REQUIRE_MSG (pybind11::isinstance(item), + "Error! Inconsistent types in list entries.\n"); + auto item_py = pybind11::cast(item); + vals.push_back(item_py.template cast()); + } + p.set(key,vals); + } +}; + +inline void pybind_pyparamlist (pybind11::module& m) +{ + // Param list + pybind11::class_(m,"ParameterList") + .def(pybind11::init()); +} + +} // namespace scream + +#endif // PYPARAMLIST_HPP diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/share/python/pyscream.cpp index dbf1c111a386..50326ebfeb48 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/share/python/pyscream.cpp @@ -2,6 +2,7 @@ #include "pyfield.hpp" #include "pygrid.hpp" #include "pyatmproc.hpp" +#include "pyparamlist.hpp" #include #include From ae886f7ccfb0ade274ca52f771f656b80206136b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:10:46 -0600 Subject: [PATCH 406/476] EAMxx: fix PyP3 and add unit test to compare against AD driven test --- .../eamxx/src/physics/p3/python/pyp3.cpp | 20 ++++- components/eamxx/tests/CMakeLists.txt | 4 + components/eamxx/tests/python/CMakeLists.txt | 1 + .../eamxx/tests/python/pyp3/CMakeLists.txt | 58 +++++++++++++ components/eamxx/tests/python/pyp3/input.yaml | 36 ++++++++ .../eamxx/tests/python/pyp3/output.yaml | 18 ++++ .../eamxx/tests/python/pyp3/p3_standalone_py | 86 +++++++++++++++++++ 7 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 components/eamxx/tests/python/CMakeLists.txt create mode 100644 components/eamxx/tests/python/pyp3/CMakeLists.txt create mode 100644 components/eamxx/tests/python/pyp3/input.yaml create mode 100644 components/eamxx/tests/python/pyp3/output.yaml create mode 100755 components/eamxx/tests/python/pyp3/p3_standalone_py diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/physics/p3/python/pyp3.cpp index 863c62a66845..3d5d7b50f267 100644 --- a/components/eamxx/src/physics/p3/python/pyp3.cpp +++ b/components/eamxx/src/physics/p3/python/pyp3.cpp @@ -2,6 +2,7 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyatmproc.hpp" +#include "pyparamlist.hpp" #include @@ -10,14 +11,24 @@ namespace scream { namespace py = pybind11; struct PyP3 : public PyAtmProc { + PyP3 (const PyGrid& pygrid, const PyParamList& params) + : PyAtmProc(pygrid) + { + init (params.pl); + } PyP3 (const PyGrid& pygrid) : PyAtmProc(pygrid) { - ekat::Comm comm(MPI_COMM_WORLD); + // P3 does not have a default (in the C++ code) for this parameter, + // so an entry MUST be set ekat::ParameterList pl; - pl.set("max_total_ni",10.0); + pl.set("max_total_ni",740.0e3); + init(pl); + } - ap = create_atmosphere_process(comm,pl); + void init(const ekat::ParameterList& params) { + auto comm = pygrid.grid->get_comm(); + ap = create_atmosphere_process(comm,params); // Create a grids manager on the fly auto gm = std::make_shared(pygrid.grid); @@ -32,7 +43,8 @@ PYBIND11_MODULE (pyp3,m) { m.doc() = "Python interfaces to P3Microphysics"; py::class_(m,"P3") - .def(py::init()); + .def(py::init()) + .def(py::init()); } diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 7a7a9e821493..ff6df0df55b5 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -70,4 +70,8 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) # Testing multiple atm processes coupled together add_subdirectory(multi-process) + + if (EAMXX_ENABLE_PYBIND) + add_subdirectory(python) + endif() endif() diff --git a/components/eamxx/tests/python/CMakeLists.txt b/components/eamxx/tests/python/CMakeLists.txt new file mode 100644 index 000000000000..aaa74512806a --- /dev/null +++ b/components/eamxx/tests/python/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(pyp3) diff --git a/components/eamxx/tests/python/pyp3/CMakeLists.txt b/components/eamxx/tests/python/pyp3/CMakeLists.txt new file mode 100644 index 000000000000..7142be8dd8f8 --- /dev/null +++ b/components/eamxx/tests/python/pyp3/CMakeLists.txt @@ -0,0 +1,58 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME p3_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) + +########################################## +# Pure CXX run # +########################################## + +set (EXEC_TYPE "cxx") +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_cxx.yaml) +CreateADUnitTest(${TEST_BASE_NAME}_cxx + LABELS p3 physics + LIBS p3 + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_cxx +) + +########################################## +# Run from python # +########################################## + +set (EXEC_TYPE "py") +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_py.yaml) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/p3_standalone_py + ${CMAKE_CURRENT_BINARY_DIR}/p3_standalone_py + @ONLY) + +# This test will a) run pyp3, and b) compare the generated nc files +CreateUnitTestFromExec( + ${TEST_BASE_NAME}_py "p3_standalone_py" + LABELS p3 physics + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_py +) + + +CreateRange(MpiRanks ${TEST_RANK_START} ${TEST_RANK_END} 1) +foreach (rank IN LISTS MpiRanks) + set(suffix "_np${rank}_omp1") + CompareNCFiles( + TEST_NAME pyp3_cxx_vs_py_np${rank} + SRC_FILE p3_standalone_cxx.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc + TGT_FILE p3_standalone_py.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_cxx${suffix} ${FIXTURES_BASE_NAME}_py${suffix} + LABELS pyeamxx + ) +endforeach() diff --git a/components/eamxx/tests/python/pyp3/input.yaml b/components/eamxx/tests/python/pyp3/input.yaml new file mode 100644 index 000000000000..2ad4696bfe40 --- /dev/null +++ b/components/eamxx/tests/python/pyp3/input.yaml @@ -0,0 +1,36 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [p3] + p3: + log_level: info + max_total_ni: 740.0e+3 + do_prescribed_ccn: false + +grids_manager: + Type: Mesh Free + grids_names: [Physics] + Physics: + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 + +# The parameters for I/O control +# Note: only cxx executable uses this, so hard-code the name of the output yaml file +Scorpio: + output_yaml_files: ["output_cxx.yaml"] +... diff --git a/components/eamxx/tests/python/pyp3/output.yaml b/components/eamxx/tests/python/pyp3/output.yaml new file mode 100644 index 000000000000..edd88a744552 --- /dev/null +++ b/components/eamxx/tests/python/pyp3/output.yaml @@ -0,0 +1,18 @@ +%YAML 1.1 +--- +filename_prefix: p3_standalone_${EXEC_TYPE} +Averaging Type: Instant +Field Names: + - qi + - qc + - qr + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_liq_exchange + - micro_vap_ice_exchange +output_control: + Frequency: 1 + frequency_units: nsteps +... diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py new file mode 100755 index 000000000000..d5d41ab1130f --- /dev/null +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +import sys + +# Add path to scream libs +sys.path.append('@SCREAM_BASE_DIR@/scripts') + +# Add path to pyeamxx libs +sys.path.append('@CMAKE_BINARY_DIR@/src/share/python') +sys.path.append('@CMAKE_BINARY_DIR@/src/physics/p3/python') + +from mpi4py import MPI +import pyscream as ps +import pyp3 +from pathlib import Path +import numpy as np + +from utils import ensure_yaml +ensure_yaml() +import yaml + +from netCDF4 import Dataset + +# ######################################### +# def check_output (self): +# ######################################### + +# comm = MPI.COMM_WORLD + +# varnames = ['qi','qc','qr', +# 'eff_radius_qc','eff_radius_qi','eff_radius_qr', +# 'micro_liq_ice_exchange','micro_vap_liq_exchange','micro_vap_ice_exchange'] + +# nc_cxx_filename = f"p3_standalone_cxx.INSTANT.nsteps_x1.np{comm.Get_size()}.@RUN_T0@.nc" +# nc_py_filename = f"p3_standalone_py.INSTANT.nsteps_x1.np{comm.Get_size()}.@RUN_T0@.nc" +# cxx_ds = Dataset(nc_cxx_filename,'r') +# py_ds = Dataset(nc_py_filename,'r') + +# nsnaps = cxx_ds.dimensions["time"].size +# self.assertEqual(nsnaps,py_ds.dimensions['time'].size) +# for n in range(0,nsnaps): +# for vn in varnames: +# # Get timeslice +# cxx_var = cxx_ds.variables[vn][:].take(n,axis=0) +# py_var = py_ds.variables[vn][:].take(n,axis=0) + +# # self.assertTrue(np.isclose(py_var.get(),cxx_var).all()) +# self.assertTrue(np.array_equal(py_var,cxx_var)) + +######################################### +def main (): +######################################### + + # Get timestepping params + with open('input.yaml','r') as fd: + yaml_input = yaml.load(fd,Loader=yaml.SafeLoader) + nsteps = yaml_input['time_stepping']['number_of_steps'] + dt = yaml_input['time_stepping']['time_step'] + t0_str = yaml_input['time_stepping']['run_t0'] + + # Create the grid + ncols = 218 + nlevs = 72 + grid = ps.Grid("Physics",ncols,nlevs) + + ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + + p3params = ps.ParameterList(yaml_input['atmosphere_processes']['p3']) + print (f"type of max_total_ni is {type(yaml_input['atmosphere_processes']['p3']['max_total_ni'])}") + p3 = pyp3.P3(grid,p3params) + missing = p3.read_ic(str(ic_file)) + print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") + p3.initialize(t0_str) + p3.setup_output("output_py.yaml") + + # Time looop + for n in range(0,nsteps): + p3.run(dt) + +#################################### +if __name__ == "__main__": + # This level of indirection ensures all pybind structs are destroyed + # before we finalize eamxx (and hence kokkos) + ps.init() + main () + ps.finalize() From b2fc733cac2f7333ce6d3ada056e574a3587dc19 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:32:16 -0600 Subject: [PATCH 407/476] EAMxx: cleanup p3_standalone_py and fix MPI issue --- .../eamxx/tests/python/pyp3/p3_standalone_py | 38 ++++--------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index d5d41ab1130f..8ba5d0726599 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -9,44 +9,21 @@ sys.path.append('@SCREAM_BASE_DIR@/scripts') sys.path.append('@CMAKE_BINARY_DIR@/src/share/python') sys.path.append('@CMAKE_BINARY_DIR@/src/physics/p3/python') +# Without these, and manual init/finalize, on my laptop I get +# obscure MPI errors at exit. +import mpi4py +mpi4py.rc.initialize = False # do not initialize MPI automatically +mpi4py.rc.finalize = False # do not finalize MPI automatically + from mpi4py import MPI import pyscream as ps import pyp3 from pathlib import Path -import numpy as np from utils import ensure_yaml ensure_yaml() import yaml -from netCDF4 import Dataset - -# ######################################### -# def check_output (self): -# ######################################### - -# comm = MPI.COMM_WORLD - -# varnames = ['qi','qc','qr', -# 'eff_radius_qc','eff_radius_qi','eff_radius_qr', -# 'micro_liq_ice_exchange','micro_vap_liq_exchange','micro_vap_ice_exchange'] - -# nc_cxx_filename = f"p3_standalone_cxx.INSTANT.nsteps_x1.np{comm.Get_size()}.@RUN_T0@.nc" -# nc_py_filename = f"p3_standalone_py.INSTANT.nsteps_x1.np{comm.Get_size()}.@RUN_T0@.nc" -# cxx_ds = Dataset(nc_cxx_filename,'r') -# py_ds = Dataset(nc_py_filename,'r') - -# nsnaps = cxx_ds.dimensions["time"].size -# self.assertEqual(nsnaps,py_ds.dimensions['time'].size) -# for n in range(0,nsnaps): -# for vn in varnames: -# # Get timeslice -# cxx_var = cxx_ds.variables[vn][:].take(n,axis=0) -# py_var = py_ds.variables[vn][:].take(n,axis=0) - -# # self.assertTrue(np.isclose(py_var.get(),cxx_var).all()) -# self.assertTrue(np.array_equal(py_var,cxx_var)) - ######################################### def main (): ######################################### @@ -66,7 +43,6 @@ def main (): ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') p3params = ps.ParameterList(yaml_input['atmosphere_processes']['p3']) - print (f"type of max_total_ni is {type(yaml_input['atmosphere_processes']['p3']['max_total_ni'])}") p3 = pyp3.P3(grid,p3params) missing = p3.read_ic(str(ic_file)) print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") @@ -81,6 +57,8 @@ def main (): if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) + MPI.Init() ps.init() main () ps.finalize() + MPI.Finalize() From 1fbf747267fc5b36012216e758fc3b82c48391e6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:35:28 -0600 Subject: [PATCH 408/476] EAMxx: remove heavy formatting from logger in atm procs constructor --- components/eamxx/src/share/atm_process/atmosphere_process.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/share/atm_process/atmosphere_process.cpp b/components/eamxx/src/share/atm_process/atmosphere_process.cpp index 189c980c431c..ab6d34310ceb 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process.cpp @@ -45,6 +45,7 @@ AtmosphereProcess (const ekat::Comm& comm, const ekat::ParameterList& params) using logger_impl_t = Logger; auto log_level = m_params.get("log_level","trace"); m_atm_logger = std::make_shared("",str2LogLevel(log_level),m_comm); + m_atm_logger->set_no_format(); } if (m_params.isParameter("number_of_subcycles")) { From 21b3c9bf4ca168417c41f682e08ef4e89be9e125 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 30 May 2024 22:45:34 -0600 Subject: [PATCH 409/476] EAMxx: expose pybind config option, and automatically handle BUILD_SHARED_LIBS --- components/eamxx/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 8567512abf2b..580ce26e7f50 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -272,6 +272,12 @@ set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) +option (EAMXX_ENABLE_PYBIND "Whether to enable python interface to eamxx, via pybind11" OFF) +if (EAMXX_ENABLE_PYBIND) + # Pybind11 requires shared libraries + set (BUILD_SHARED_LIBS ON) +endif() + #################################################################### # Packs-related settings # #################################################################### From 8a57eaf703d08c938a48411fa21b1776519cd40c Mon Sep 17 00:00:00 2001 From: Benjamin Hillman Date: Thu, 30 May 2024 23:59:04 -0600 Subject: [PATCH 410/476] Fix number of levels in z_int --- components/eamxx/src/physics/cosp/eamxx_cosp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 2395aaad3dc7..4aa8d53fea8a 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -179,7 +179,7 @@ void Cosp::run_impl (const double dt) // Compute heights const auto z_mid = CospFunc::view_2d("z_mid", m_num_cols, m_num_levs); - const auto z_int = CospFunc::view_2d("z_int", m_num_cols, m_num_levs); + const auto z_int = CospFunc::view_2d("z_int", m_num_cols, m_num_levs+1); const auto dz = z_mid; // reuse tmp memory for dz const auto ncol = m_num_cols; const auto nlev = m_num_levs; From b41796866033260290d95f561e2741180dfe4d9b Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Fri, 31 May 2024 08:20:40 -0600 Subject: [PATCH 411/476] address fieldAllocProp::subview() comments, and add deep_copy test for subfield --- .../src/share/field/field_alloc_prop.cpp | 12 +- .../eamxx/src/share/field/field_impl.hpp | 118 ++++++++++++------ .../eamxx/src/share/tests/subfield_tests.cpp | 36 ++++++ 3 files changed, 120 insertions(+), 46 deletions(-) diff --git a/components/eamxx/src/share/field/field_alloc_prop.cpp b/components/eamxx/src/share/field/field_alloc_prop.cpp index 266fcb88779f..be07af014451 100644 --- a/components/eamxx/src/share/field/field_alloc_prop.cpp +++ b/components/eamxx/src/share/field/field_alloc_prop.cpp @@ -67,13 +67,13 @@ subview (const int idim, const int k, const bool dynamic) const { // and there is no packing props.m_last_extent = m_layout.dim(idim); props.m_pack_size_max = 1; - props.m_alloc_size = m_alloc_size / m_last_extent; } else { // We are keeping the last dim, so same last extent and max pack size props.m_last_extent = m_last_extent; props.m_pack_size_max = m_pack_size_max; - props.m_alloc_size = m_alloc_size / m_layout.dim(idim); } + // give it an invalid value because subviews don't get allocated + props.m_alloc_size = -1; return props; } @@ -103,21 +103,19 @@ FieldAllocProp FieldAllocProp::subview(const int idim, props.m_subview_info = SubviewInfo(idim, index_beg, index_end, m_layout.dim(idim)); - // Figure out strides/packs if (idim == (m_layout.rank() - 1)) { // We're slicing the possibly padded dim, so everything else is as in the // layout, and there is no packing - // props.m_last_extent = m_layout.dim(idim); props.m_last_extent = index_end - index_beg; props.m_pack_size_max = 1; - props.m_alloc_size = m_alloc_size / m_last_extent; } else { // We are keeping the last dim, so same last extent and max pack size props.m_last_extent = m_last_extent; props.m_pack_size_max = m_pack_size_max; - props.m_alloc_size = m_alloc_size / m_layout.dim(idim); } + // give it an invalid value because subviews don't get allocated + props.m_alloc_size = -1; return props; } @@ -125,7 +123,7 @@ void FieldAllocProp::request_allocation (const int pack_size) { using ekat::ScalarTraits; EKAT_REQUIRE_MSG(!m_committed, - "Error! Cannot change allocation properties after they have been commited.\n"); + "Error! Cannot change allocation properties after they have been committed.\n"); const int vts = m_scalar_type_size*pack_size; diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 420d90492e72..42a0a6cd61e9 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -305,8 +305,6 @@ deep_copy_impl (const Field& src) { // different pack sizes. auto src_alloc_props = src.get_header().get_alloc_properties(); auto tgt_alloc_props = get_header().get_alloc_properties(); - auto src_alloc_size = src_alloc_props.get_alloc_size(); - auto tgt_alloc_size = tgt_alloc_props.get_alloc_size(); // If a manual parallel_for is required (b/c of alloc sizes difference), // we need to create extents (rather than just using the one in layout), @@ -325,33 +323,32 @@ deep_copy_impl (const Field& src) { if (src_alloc_props.contiguous() and tgt_alloc_props.contiguous()) { auto v = get_view< ST*,HD>(); auto v_src = src.get_view(); - if (src_alloc_size==tgt_alloc_size) { - Kokkos::deep_copy(v,v_src); - } else { - Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { v(idx) = v_src(idx); }); - } } else { auto v = get_strided_view< ST*,HD>(); auto v_src = src.get_strided_view(); - if (src_alloc_size==tgt_alloc_size) { - Kokkos::deep_copy(v,v_src); - } else { - Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { v(idx) = v_src(idx); }); - } } } break; case 2: { - auto v = get_view< ST**,HD>(); - auto v_src = src.get_view(); - if (src_alloc_size==tgt_alloc_size) { - Kokkos::deep_copy(v,v_src); - } else { + if (src_alloc_props.contiguous() and tgt_alloc_props.contiguous()) { + auto v = get_view< ST**,HD>(); + auto v_src = src.get_view(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + int i,j; + unflatten_idx(idx,ext,i,j); + v(i,j) = v_src(i,j); + }); + } + else { + auto v = get_strided_view< ST**,HD>(); + auto v_src = src.get_strided_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j; unflatten_idx(idx,ext,i,j); @@ -362,11 +359,17 @@ deep_copy_impl (const Field& src) { break; case 3: { - auto v = get_view< ST***,HD>(); - auto v_src = src.get_view(); - if (src_alloc_size==tgt_alloc_size) { - Kokkos::deep_copy(v,v_src); + if (src_alloc_props.contiguous() and tgt_alloc_props.contiguous()) { + auto v = get_view< ST***,HD>(); + auto v_src = src.get_view(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + int i,j,k; + unflatten_idx(idx,ext,i,j,k); + v(i,j,k) = v_src(i,j,k); + }); } else { + auto v = get_strided_view< ST***,HD>(); + auto v_src = src.get_strided_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k; unflatten_idx(idx,ext,i,j,k); @@ -377,11 +380,17 @@ deep_copy_impl (const Field& src) { break; case 4: { - auto v = get_view< ST****,HD>(); - auto v_src = src.get_view(); - if (src_alloc_size==tgt_alloc_size) { - Kokkos::deep_copy(v,v_src); + if (src_alloc_props.contiguous() and tgt_alloc_props.contiguous()) { + auto v = get_view< ST****,HD>(); + auto v_src = src.get_view(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + int i,j,k,l; + unflatten_idx(idx,ext,i,j,k,l); + v(i,j,k,l) = v_src(i,j,k,l); + }); } else { + auto v = get_strided_view< ST****,HD>(); + auto v_src = src.get_strided_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k,l; unflatten_idx(idx,ext,i,j,k,l); @@ -392,11 +401,17 @@ deep_copy_impl (const Field& src) { break; case 5: { - auto v = get_view< ST*****,HD>(); - auto v_src = src.get_view(); - if (src_alloc_size==tgt_alloc_size) { - Kokkos::deep_copy(v,v_src); + if (src_alloc_props.contiguous() and tgt_alloc_props.contiguous()) { + auto v = get_view< ST*****,HD>(); + auto v_src = src.get_view(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { + int i,j,k,l,m; + unflatten_idx(idx,ext,i,j,k,l,m); + v(i,j,k,l,m) = v_src(i,j,k,l,m); + }); } else { + auto v = get_view< ST*****,HD>(); + auto v_src = src.get_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { int i,j,k,l,m; unflatten_idx(idx,ext,i,j,k,l,m); @@ -439,32 +454,57 @@ void Field::deep_copy_impl (const ST value) { break; case 2: { - auto v = get_view(); - Kokkos::deep_copy(v,value); + if (m_header->get_alloc_properties().contiguous()) { + auto v = get_view(); + Kokkos::deep_copy(v,value); + } else { + auto v = get_strided_view(); + Kokkos::deep_copy(v,value); + } } break; case 3: { - auto v = get_view(); - Kokkos::deep_copy(v,value); + if (m_header->get_alloc_properties().contiguous()) { + auto v = get_view(); + Kokkos::deep_copy(v,value); + } else { + auto v = get_strided_view(); + Kokkos::deep_copy(v,value); + } } break; case 4: { - auto v = get_view(); - Kokkos::deep_copy(v,value); + if (m_header->get_alloc_properties().contiguous()) { + auto v = get_view(); + Kokkos::deep_copy(v,value); + } else { + auto v = get_strided_view(); + Kokkos::deep_copy(v,value); + } } break; case 5: { - auto v = get_view(); - Kokkos::deep_copy(v,value); + if (m_header->get_alloc_properties().contiguous()) { + auto v = get_view(); + Kokkos::deep_copy(v,value); + } else { + auto v = get_strided_view(); + Kokkos::deep_copy(v,value); + } } break; case 6: { - auto v = get_view(); - Kokkos::deep_copy(v,value); + if (m_header->get_alloc_properties().contiguous()) { + auto v = get_view(); + Kokkos::deep_copy(v,value); + } else { + auto v = get_strided_view(); + Kokkos::deep_copy(v,value); + } } break; default: diff --git a/components/eamxx/src/share/tests/subfield_tests.cpp b/components/eamxx/src/share/tests/subfield_tests.cpp index f40041a79560..97fe20318c5c 100644 --- a/components/eamxx/src/share/tests/subfield_tests.cpp +++ b/components/eamxx/src/share/tests/subfield_tests.cpp @@ -300,6 +300,42 @@ TEST_CASE("field", "") { } } } + SECTION("Subfield deep_copy") { + // ============== + /* Rank-3 view */ + // ============== + std::vector t3 = {COL, CMP, LEV}; + std::vector d3 = {5, 10, 2}; + FieldIdentifier fid3("3d", {t3, d3}, m / s, "some_grid"); + + Field f3a(fid3); + f3a.allocate_view(); + randomize(f3a, engine, pdf); + + Field f3b(fid3); + f3b.allocate_view(); + randomize(f3b, engine, pdf); + + const int idim = 1; + const int sl_beg = 3; + const int sl_end = 6; + + auto sfa = f3a.subfield(idim, sl_beg, sl_end); + auto sv_a = sfa.get_strided_view(); + + auto sfb = f3b.subfield(idim, sl_beg, sl_end); + auto sv_b = sfb.get_strided_view(); + + sfb.deep_copy(sfa); + + for (int i = 0; i < d3[0]; i++) { + for (int j = sl_beg; j < sl_end; j++) { + for (int k = 0; k < d3[2]; k++) { + REQUIRE(sv_a(i, j - sl_beg, k) == sv_b(i, j - sl_beg, k)); + } + } + } + } } // Dynamic Subfields SECTION("dynamic_subfield") { From 45e462ea5515d47cf7fddd1480a8e4bcae63785d Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Fri, 31 May 2024 11:38:08 -0600 Subject: [PATCH 412/476] Need non-zero eff_radius for COSP standalone --- components/eamxx/tests/single-process/cosp/input.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/tests/single-process/cosp/input.yaml b/components/eamxx/tests/single-process/cosp/input.yaml index 5b3b2787943d..c6b79c019822 100644 --- a/components/eamxx/tests/single-process/cosp/input.yaml +++ b/components/eamxx/tests/single-process/cosp/input.yaml @@ -28,8 +28,8 @@ initial_conditions: dtau067: 1.0 dtau105: 1.0 cldfrac_rad: 0.5 - eff_radius_qc: 0.0 - eff_radius_qi: 0.0 + eff_radius_qc: 10.0 + eff_radius_qi: 10.0 sunlit: 1.0 surf_radiative_T: 288.0 pseudo_density: 1.0 From e53315270f5313d172193bb8eaf4281a92071b20 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 31 May 2024 15:12:49 -0500 Subject: [PATCH 413/476] coalesce libpyeamxx in src/python --- components/eamxx/src/CMakeLists.txt | 4 ++++ components/eamxx/src/physics/p3/CMakeLists.txt | 4 ---- components/eamxx/src/physics/p3/python/CMakeLists.txt | 6 ------ components/eamxx/src/python/CMakeLists.txt | 1 + components/eamxx/src/python/libpyeamxx/CMakeLists.txt | 4 ++++ .../pyscream.cpp => python/libpyeamxx/libpyeamxx.cpp} | 4 +++- .../{share/python => python/libpyeamxx}/pyatmproc.hpp | 0 .../{share/python => python/libpyeamxx}/pyfield.hpp | 0 .../{share/python => python/libpyeamxx}/pygrid.hpp | 0 .../{physics/p3/python => python/libpyeamxx}/pyp3.cpp | 7 ++++--- .../python => python/libpyeamxx}/pyparamlist.hpp | 0 components/eamxx/src/python/readme | 11 +++++++++++ components/eamxx/src/python/setup.py | 4 ++-- components/eamxx/src/share/CMakeLists.txt | 4 ---- components/eamxx/src/share/python/CMakeLists.txt | 4 ---- components/eamxx/tests/python/pyp3/p3_standalone_py | 3 +-- 16 files changed, 30 insertions(+), 26 deletions(-) delete mode 100644 components/eamxx/src/physics/p3/python/CMakeLists.txt create mode 100644 components/eamxx/src/python/CMakeLists.txt create mode 100644 components/eamxx/src/python/libpyeamxx/CMakeLists.txt rename components/eamxx/src/{share/python/pyscream.cpp => python/libpyeamxx/libpyeamxx.cpp} (91%) rename components/eamxx/src/{share/python => python/libpyeamxx}/pyatmproc.hpp (100%) rename components/eamxx/src/{share/python => python/libpyeamxx}/pyfield.hpp (100%) rename components/eamxx/src/{share/python => python/libpyeamxx}/pygrid.hpp (100%) rename components/eamxx/src/{physics/p3/python => python/libpyeamxx}/pyp3.cpp (89%) rename components/eamxx/src/{share/python => python/libpyeamxx}/pyparamlist.hpp (100%) create mode 100644 components/eamxx/src/python/readme delete mode 100644 components/eamxx/src/share/python/CMakeLists.txt diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 59a5d6646443..7672d79f13d6 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -7,3 +7,7 @@ add_subdirectory(control) if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() + +if (EAMXX_ENABLE_PYBIND) + add_subdirectory(python) +endif() diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 287bac452c1b..7ea0aeeb8145 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -129,10 +129,6 @@ if (Kokkos_ENABLE_HIP) set_source_files_properties(p3_functions_f90.cpp PROPERTIES COMPILE_FLAGS -O0) endif() -if (EAMXX_ENABLE_PYBIND) - add_subdirectory (python) -endif() - if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/physics/p3/python/CMakeLists.txt b/components/eamxx/src/physics/p3/python/CMakeLists.txt deleted file mode 100644 index 0b1b75e1b6ed..000000000000 --- a/components/eamxx/src/physics/p3/python/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# By now it should have already been found, but just for completeness -find_package(pybind11 REQUIRED) - -pybind11_add_module(pyp3 pyp3.cpp) -target_link_libraries(pyp3 PUBLIC p3) -target_include_directories(pyp3 PRIVATE ${SCREAM_BASE_DIR}/src/share/python) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt new file mode 100644 index 000000000000..01df409551c2 --- /dev/null +++ b/components/eamxx/src/python/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(libpyeamxx) diff --git a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt new file mode 100644 index 000000000000..53a9e3812702 --- /dev/null +++ b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt @@ -0,0 +1,4 @@ +find_package(pybind11 REQUIRED) + +pybind11_add_module(libpyeamxx libpyeamxx.cpp) +target_link_libraries(libpyeamxx PUBLIC scream_share scream_io p3) diff --git a/components/eamxx/src/share/python/pyscream.cpp b/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp similarity index 91% rename from components/eamxx/src/share/python/pyscream.cpp rename to components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp index 50326ebfeb48..177bedbfb37a 100644 --- a/components/eamxx/src/share/python/pyscream.cpp +++ b/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp @@ -3,6 +3,7 @@ #include "pygrid.hpp" #include "pyatmproc.hpp" #include "pyparamlist.hpp" +#include "pyp3.cpp" #include #include @@ -21,7 +22,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (pyscream,m) { +PYBIND11_MODULE (libpyeamxx,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; @@ -34,6 +35,7 @@ PYBIND11_MODULE (pyscream,m) { pybind_pyfield(m); pybind_pygrid(m); pybind_pyatmproc(m); + pybind_pyp3(m); } } // namespace scream diff --git a/components/eamxx/src/share/python/pyatmproc.hpp b/components/eamxx/src/python/libpyeamxx/pyatmproc.hpp similarity index 100% rename from components/eamxx/src/share/python/pyatmproc.hpp rename to components/eamxx/src/python/libpyeamxx/pyatmproc.hpp diff --git a/components/eamxx/src/share/python/pyfield.hpp b/components/eamxx/src/python/libpyeamxx/pyfield.hpp similarity index 100% rename from components/eamxx/src/share/python/pyfield.hpp rename to components/eamxx/src/python/libpyeamxx/pyfield.hpp diff --git a/components/eamxx/src/share/python/pygrid.hpp b/components/eamxx/src/python/libpyeamxx/pygrid.hpp similarity index 100% rename from components/eamxx/src/share/python/pygrid.hpp rename to components/eamxx/src/python/libpyeamxx/pygrid.hpp diff --git a/components/eamxx/src/physics/p3/python/pyp3.cpp b/components/eamxx/src/python/libpyeamxx/pyp3.cpp similarity index 89% rename from components/eamxx/src/physics/p3/python/pyp3.cpp rename to components/eamxx/src/python/libpyeamxx/pyp3.cpp index 3d5d7b50f267..497cecb135fc 100644 --- a/components/eamxx/src/physics/p3/python/pyp3.cpp +++ b/components/eamxx/src/python/libpyeamxx/pyp3.cpp @@ -38,10 +38,11 @@ struct PyP3 : public PyAtmProc { } }; -PYBIND11_MODULE (pyp3,m) { - - m.doc() = "Python interfaces to P3Microphysics"; +// PYBIND11_MODULE (pyp3,m) { +// m.doc() = "Python interfaces to P3Microphysics"; +inline void pybind_pyp3(py::module& m) { + py::class_(m,"P3") .def(py::init()) .def(py::init()); diff --git a/components/eamxx/src/share/python/pyparamlist.hpp b/components/eamxx/src/python/libpyeamxx/pyparamlist.hpp similarity index 100% rename from components/eamxx/src/share/python/pyparamlist.hpp rename to components/eamxx/src/python/libpyeamxx/pyparamlist.hpp diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme new file mode 100644 index 000000000000..05776cdcd019 --- /dev/null +++ b/components/eamxx/src/python/readme @@ -0,0 +1,11 @@ +INFO: +- EAMxx python bindings +- libpyeamxx is where we will house the extensions +- pyeamxx is where we will house the python code + +TODO: +- formalize the build + - need to build libpyeamxx and get all the shared objs there + - need to write down scripts for building and testing +- add more python code to organize structures +- establish ci diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py index 76371840008c..6009dd48dc66 100644 --- a/components/eamxx/src/python/setup.py +++ b/components/eamxx/src/python/setup.py @@ -12,7 +12,7 @@ def has_ext_modules(foo): description='EAMxx wrapper', packages=['','pyeamxx'], package_data={ - '': ['*.*so'], + '': ['libpyeamxx/*.so'], }, distclass=BinaryDistribution, zip_safe=False @@ -26,4 +26,4 @@ def has_ext_modules(foo): # We simply build manually-ish for support machines and just upload the wheels # to pypi. That way, we can ensure the build works on the machine, # we can set its environment as we wish, etc.. -# The downside is that it is manual... \ No newline at end of file +# The downside is that it is manual... diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index a2a2e2bc9fb7..e83f9d98ab9f 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -104,10 +104,6 @@ endif() # IO library add_subdirectory(io) -if (EAMXX_ENABLE_PYBIND) - add_subdirectory(python) -endif() - if (NOT SCREAM_LIB_ONLY) # Create test_support lib add_library(scream_test_support diff --git a/components/eamxx/src/share/python/CMakeLists.txt b/components/eamxx/src/share/python/CMakeLists.txt deleted file mode 100644 index 1284106421c0..000000000000 --- a/components/eamxx/src/share/python/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -find_package(pybind11 REQUIRED) - -pybind11_add_module(pyscream pyscream.cpp) -target_link_libraries(pyscream PUBLIC scream_share scream_io) diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 8ba5d0726599..9d2b648060bf 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -6,8 +6,7 @@ import sys sys.path.append('@SCREAM_BASE_DIR@/scripts') # Add path to pyeamxx libs -sys.path.append('@CMAKE_BINARY_DIR@/src/share/python') -sys.path.append('@CMAKE_BINARY_DIR@/src/physics/p3/python') +sys.path.append('@CMAKE_BINARY_DIR@/src/python/libpyeamxx') # Without these, and manual init/finalize, on my laptop I get # obscure MPI errors at exit. From 2e9f20338a889fd1f76e64b5507fa05e4ae183e9 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Fri, 31 May 2024 16:53:02 -0500 Subject: [PATCH 414/476] organize package further, start build recipe --- .gitignore | 5 ++-- .../src/python/libpyeamxx/CMakeLists.txt | 4 +-- .../src/python/libpyeamxx/libpyeamxx.cpp | 2 +- .../eamxx/src/python/pyeamxx/__init__.py | 24 ++++++++++++++++-- components/eamxx/src/python/setup.py | 4 +-- components/eamxx/src/python/tools/build.sh | 25 +++++++++++++++++++ components/eamxx/src/python/tools/meta.yaml | 3 +++ 7 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 components/eamxx/src/python/tools/build.sh create mode 100644 components/eamxx/src/python/tools/meta.yaml diff --git a/.gitignore b/.gitignore index f70951bb3578..e3d164277805 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,6 @@ components/eamxx/docs/common/eamxx_params.md # Python packaging *.egg-info -*components/eamxx/src/python/build -*components/eamxx/src/python/dist +components/eamxx/src/python/build +components/eamxx/src/python/build_src +components/eamxx/src/python/dist diff --git a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt index 53a9e3812702..1f3890eb3af4 100644 --- a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt +++ b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) -pybind11_add_module(libpyeamxx libpyeamxx.cpp) -target_link_libraries(libpyeamxx PUBLIC scream_share scream_io p3) +pybind11_add_module(libpyeamxx_ext libpyeamxx.cpp) +target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io p3) diff --git a/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp b/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp index 177bedbfb37a..1713db23d119 100644 --- a/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp +++ b/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp @@ -22,7 +22,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (libpyeamxx,m) { +PYBIND11_MODULE (libpyeamxx_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py index 4b4ee19cc170..75a36e148291 100644 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ b/components/eamxx/src/python/pyeamxx/__init__.py @@ -1,3 +1,23 @@ -""" to get going """ +""" + This file will serve as a way to organize and expose + libpyeamxx internals to the rest of pyeamxx +""" -import pyscream +from libpyeamxx.libpyeamxx_ext import AtmProc +from libpyeamxx.libpyeamxx_ext import Grid +from libpyeamxx.libpyeamxx_ext import ParameterList +from libpyeamxx.libpyeamxx_ext import init +from libpyeamxx.libpyeamxx_ext import Field +from libpyeamxx.libpyeamxx_ext import P3 +from libpyeamxx.libpyeamxx_ext import finalize + + +__all__ = [ + 'init', + 'finalize', + 'AtmProc', + 'Grid', + 'ParameterList', + 'Field', + 'P3', +] diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py index 6009dd48dc66..20081b1b72d2 100644 --- a/components/eamxx/src/python/setup.py +++ b/components/eamxx/src/python/setup.py @@ -7,10 +7,10 @@ def has_ext_modules(foo): setup( name='pyeamxx', - version='0.0.1', + version='0.0.1.dev4', author='E3SM SCREAM', description='EAMxx wrapper', - packages=['','pyeamxx'], + packages=['', 'pyeamxx'], package_data={ '': ['libpyeamxx/*.so'], }, diff --git a/components/eamxx/src/python/tools/build.sh b/components/eamxx/src/python/tools/build.sh new file mode 100644 index 000000000000..6de0e9906e69 --- /dev/null +++ b/components/eamxx/src/python/tools/build.sh @@ -0,0 +1,25 @@ +#!/bin/env bash + +cmake \ + -S ../../../ \ + -B ../build_src \ + -DCMAKE_BUILD_TYPE='Release' \ + -DEAMXX_ENABLE_PYBIND='ON' \ + -DNetCDF_Fortran_PATH=$CONDA_PREFIX \ + -DNetCDF_C_PATH=$CONDA_PREFIX \ + -DCMAKE_CXX_FLAGS='-fvisibility-inlines-hidden -fmessage-length=0' \ + -DCMAKE_C_FLAGS='' \ + -DCMAKE_Fortran_FLAGS='' \ + -DCMAKE_CXX_COMPILER=mpic++ \ + -DCMAKE_C_COMPILER=mpicc \ + -DCMAKE_Fortran_COMPILER=mpif90 \ + -DBUILD_SHARED_LIBS=ON \ + -DSCREAM_INPUT_ROOT="../build_src" \ + -DSCREAM_ENABLE_BASELINE_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX + +cmake --build ../build_src/src/python -j${CPU_COUNT:-128} + +find ../build_src -type f -name "*.so" | xargs cp -t ../libpyeamxx + +python -m pip install -vvv ../ diff --git a/components/eamxx/src/python/tools/meta.yaml b/components/eamxx/src/python/tools/meta.yaml new file mode 100644 index 000000000000..8227be9b010b --- /dev/null +++ b/components/eamxx/src/python/tools/meta.yaml @@ -0,0 +1,3 @@ +# add pyeamxx recipe here +# get yaml-cpp, spdlog, netcdf, libnetcdf, lippnetcdf, etc. from cf +# cannot get kokkos from cf, must build From f8a19928dc24a269ac24c85523f09eeea055cc0a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 31 May 2024 22:32:02 -0700 Subject: [PATCH 415/476] GPU fixes: changes array of views to views in Kokkos loop --- .../physics/mam/eamxx_mam_aci_functions.hpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index a11d9e066d7c..6c9499da5ce1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -429,8 +429,9 @@ void call_function_dropmixnuc( atmosphere_for_column(dry_atmosphere, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays + constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; @@ -522,12 +523,11 @@ void update_cloud_borne_aerosols( KOKKOS_INLINE_FUNCTION void update_interstitial_aerosols_levs( const haero::ThreadTeam &team, const int nlev, const int icol, - const int s_idx, const Real dt, - const MAMAci::view_2d ptend_q[mam4::aero_model::pcnst], + const Real dt, const MAMAci::view_2d ptend_view, // output MAMAci::view_2d aero_mr) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - aero_mr(icol, kk) += ptend_q[s_idx](icol, kk) * dt; + aero_mr(icol, kk) += ptend_view(icol, kk) * dt; }); } @@ -548,12 +548,13 @@ void update_interstitial_aerosols( auto aero_mmr = dry_aero.int_aero_mmr[m][a]; if(aero_mmr.data()) { + const auto ptend_view = ptend_q[s_idx]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, + update_interstitial_aerosols_levs(team, nlev, icol, dt, + ptend_view, // output aero_mmr); }); @@ -563,12 +564,13 @@ void update_interstitial_aerosols( } auto aero_nmr = dry_aero.int_aero_nmr[m]; // number mixing ratio for mode "m" + const auto ptend_view = ptend_q[s_idx]; Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, s_idx, dt, - ptend_q, + update_interstitial_aerosols_levs(team, nlev, icol, dt, + ptend_view, // output aero_nmr); }); @@ -672,8 +674,9 @@ void call_hetfrz_compute_tendencies( ekat::subview(diagnostic_scratch[42], icol); // assign cloud fraction + constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, mam4::ndrop::pver), + Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { diags.stratiform_cloud_fraction(klev) = haero_atm.cloud_fraction(klev); From bb08b46f3df624efaf74ff9870e673387c2e72bd Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 1 Jun 2024 00:34:32 -0500 Subject: [PATCH 416/476] add conda recipe that builds correctly --- components/eamxx/src/python/readme | 9 ++-- components/eamxx/src/python/setup.py | 12 +---- components/eamxx/src/python/tools/build.sh | 27 ++++++---- .../src/python/tools/conda_build_config.yaml | 6 +++ components/eamxx/src/python/tools/meta.yaml | 52 +++++++++++++++++-- 5 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 components/eamxx/src/python/tools/conda_build_config.yaml diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index 05776cdcd019..823ff9c0426b 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -4,8 +4,9 @@ INFO: - pyeamxx is where we will house the python code TODO: -- formalize the build - - need to build libpyeamxx and get all the shared objs there - - need to write down scripts for building and testing - add more python code to organize structures -- establish ci +- establish ci pipeline in scream/e3sm repos +- add to cf (or alternatively sync with it) +- decide how to deal with versioning +- decide archs/pythons/mpis to tgt +- figure out what's tripping cf diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py index 20081b1b72d2..2f04925a8d83 100644 --- a/components/eamxx/src/python/setup.py +++ b/components/eamxx/src/python/setup.py @@ -7,7 +7,7 @@ def has_ext_modules(foo): setup( name='pyeamxx', - version='0.0.1.dev4', + version='0.0.1.dev5', author='E3SM SCREAM', description='EAMxx wrapper', packages=['', 'pyeamxx'], @@ -17,13 +17,3 @@ def has_ext_modules(foo): distclass=BinaryDistribution, zip_safe=False ) - -# TODOs: -# 1. We need to figure out how to handle the complex env needed by scream -# 2. We need to think forward about packaging this neatly - -# for both issues, I think the most straightforward solution for now is the following: -# We simply build manually-ish for support machines and just upload the wheels -# to pypi. That way, we can ensure the build works on the machine, -# we can set its environment as we wish, etc.. -# The downside is that it is manual... diff --git a/components/eamxx/src/python/tools/build.sh b/components/eamxx/src/python/tools/build.sh index 6de0e9906e69..798fa5fe1818 100644 --- a/components/eamxx/src/python/tools/build.sh +++ b/components/eamxx/src/python/tools/build.sh @@ -1,25 +1,30 @@ #!/bin/env bash +cd components/eamxx/src/python + cmake \ - -S ../../../ \ - -B ../build_src \ + -S ../../ \ + -B build_src \ -DCMAKE_BUILD_TYPE='Release' \ -DEAMXX_ENABLE_PYBIND='ON' \ - -DNetCDF_Fortran_PATH=$CONDA_PREFIX \ - -DNetCDF_C_PATH=$CONDA_PREFIX \ - -DCMAKE_CXX_FLAGS='-fvisibility-inlines-hidden -fmessage-length=0' \ + -DNETCDF_FORTRAN_PATH=$PREFIX \ + -DNETCDF_C_PATH=$PREFIX \ + -DCMAKE_CXX_FLAGS='-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized' \ -DCMAKE_C_FLAGS='' \ - -DCMAKE_Fortran_FLAGS='' \ + -DCMAKE_Fortran_FLAGS='-Wno-maybe-uninitialized -Wno-unused-dummy-argument' \ -DCMAKE_CXX_COMPILER=mpic++ \ -DCMAKE_C_COMPILER=mpicc \ -DCMAKE_Fortran_COMPILER=mpif90 \ -DBUILD_SHARED_LIBS=ON \ - -DSCREAM_INPUT_ROOT="../build_src" \ + -DSCREAM_INPUT_ROOT="build_src" \ -DSCREAM_ENABLE_BASELINE_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DPYTHON_EXECUTABLE=$PYTHON + +cmake --build build_src/src/python -j${CPU_COUNT:-128} -cmake --build ../build_src/src/python -j${CPU_COUNT:-128} +find build_src -type f -name "*.so" | xargs cp -t libpyeamxx/ -find ../build_src -type f -name "*.so" | xargs cp -t ../libpyeamxx +$PYTHON -m build -w -n -x -python -m pip install -vvv ../ +pip install dist/*.whl -vv diff --git a/components/eamxx/src/python/tools/conda_build_config.yaml b/components/eamxx/src/python/tools/conda_build_config.yaml new file mode 100644 index 000000000000..f5f68d8d781f --- /dev/null +++ b/components/eamxx/src/python/tools/conda_build_config.yaml @@ -0,0 +1,6 @@ + +# eamxx fails with gcc 13 +c_compiler_version: # [linux] + - 12 # [linux] +cxx_compiler_version: # [linux] + - 12 # [linux] diff --git a/components/eamxx/src/python/tools/meta.yaml b/components/eamxx/src/python/tools/meta.yaml index 8227be9b010b..354a62c61dce 100644 --- a/components/eamxx/src/python/tools/meta.yaml +++ b/components/eamxx/src/python/tools/meta.yaml @@ -1,3 +1,49 @@ -# add pyeamxx recipe here -# get yaml-cpp, spdlog, netcdf, libnetcdf, lippnetcdf, etc. from cf -# cannot get kokkos from cf, must build +package: + name: pyeamxx + version: 0.0.1.dev5 + +source: + path: ../../../../../ + +build: + number: 0 + +requirements: + build: + - cmake + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('fortran') }} + host: + - python + - pybind11 + - setuptools + - build + - spdlog + - yaml-cpp + - openmpi + - hdf5 *=*openmpi* + - libnetcdf + - netcdf-cxx4 + - netcdf-fortran + - python + - numpy + run: + - spdlog + - yaml-cpp + - openmpi + - hdf5 *=*openmpi* + - libnetcdf + - netcdf-cxx4 + - netcdf-fortran + - python + - numpy + - mpi4py + +about: + home: https://github.com/E3SM-Project/scream + summary: Python wrapper for EAMxx + +extra: + recipe-maintainers: + - scream-team From 9a6dfdab8e281bd920a783b45d9617e77e5635c4 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 1 Jun 2024 00:41:44 -0500 Subject: [PATCH 417/476] remove pybind11 submodule --- .gitmodules | 3 --- externals/pybind11 | 1 - 2 files changed, 4 deletions(-) delete mode 160000 externals/pybind11 diff --git a/.gitmodules b/.gitmodules index 2db4c370ec78..69d56e140361 100644 --- a/.gitmodules +++ b/.gitmodules @@ -83,6 +83,3 @@ path = externals/mam4xx url = git@github.com:eagles-project/mam4xx.git -[submodule "externals/pybind11"] - path = externals/pybind11 - url = git@github.com:pybind/pybind11.git diff --git a/externals/pybind11 b/externals/pybind11 deleted file mode 160000 index ae6432b8172e..000000000000 --- a/externals/pybind11 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ae6432b8172e24f8da60375302a2e6e9e5f714d1 From 45acb00ab9b94a7c02b80c6f19201459c15ae34d Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 1 Jun 2024 10:50:15 -0500 Subject: [PATCH 418/476] improve recipe further and use patchelf --- .gitignore | 1 + components/eamxx/src/python/setup.py | 2 +- components/eamxx/src/python/tools/build.sh | 7 +++--- components/eamxx/src/python/tools/meta.yaml | 24 ++++++++++++--------- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index e3d164277805..f2db25207114 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ # Shared libraries *.so +*.so.* # Executables *.exe diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py index 2f04925a8d83..40b62d3c17cb 100644 --- a/components/eamxx/src/python/setup.py +++ b/components/eamxx/src/python/setup.py @@ -12,7 +12,7 @@ def has_ext_modules(foo): description='EAMxx wrapper', packages=['', 'pyeamxx'], package_data={ - '': ['libpyeamxx/*.so'], + '': ['libpyeamxx/*.so*'], }, distclass=BinaryDistribution, zip_safe=False diff --git a/components/eamxx/src/python/tools/build.sh b/components/eamxx/src/python/tools/build.sh index 798fa5fe1818..21757e8206b2 100644 --- a/components/eamxx/src/python/tools/build.sh +++ b/components/eamxx/src/python/tools/build.sh @@ -7,8 +7,8 @@ cmake \ -B build_src \ -DCMAKE_BUILD_TYPE='Release' \ -DEAMXX_ENABLE_PYBIND='ON' \ - -DNETCDF_FORTRAN_PATH=$PREFIX \ - -DNETCDF_C_PATH=$PREFIX \ + -DNetcdf_Fortran_PATH=$PREFIX \ + -DNetcdf_C_PATH=$PREFIX \ -DCMAKE_CXX_FLAGS='-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized' \ -DCMAKE_C_FLAGS='' \ -DCMAKE_Fortran_FLAGS='-Wno-maybe-uninitialized -Wno-unused-dummy-argument' \ @@ -23,7 +23,8 @@ cmake \ cmake --build build_src/src/python -j${CPU_COUNT:-128} -find build_src -type f -name "*.so" | xargs cp -t libpyeamxx/ +find build_src -name "*.so*" | xargs cp -t libpyeamxx/ +for f in libpyeamxx/*.so*; do patchelf --set-rpath '$ORIGIN' --force-rpath $f; done $PYTHON -m build -w -n -x diff --git a/components/eamxx/src/python/tools/meta.yaml b/components/eamxx/src/python/tools/meta.yaml index 354a62c61dce..0ee370bbbb55 100644 --- a/components/eamxx/src/python/tools/meta.yaml +++ b/components/eamxx/src/python/tools/meta.yaml @@ -11,6 +11,7 @@ build: requirements: build: - cmake + # - {{ stdlib('c') }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('fortran') }} @@ -20,24 +21,27 @@ requirements: - setuptools - build - spdlog + - fmt - yaml-cpp - - openmpi - - hdf5 *=*openmpi* + - mpich + - hdf5 *=*mpich* - libnetcdf - netcdf-cxx4 - netcdf-fortran - python - numpy + - mpi4py run: - - spdlog - - yaml-cpp - - openmpi - - hdf5 *=*openmpi* - - libnetcdf - - netcdf-cxx4 - - netcdf-fortran + - {{ pin_compatible('spdlog', max_pin='x.x') }} + - {{ pin_compatible('fmt', max_pin='x.x') }} + - {{ pin_compatible('yaml-cpp', max_pin='x.x') }} + - {{ pin_compatible('mpich', max_pin='x.x') }} + - hdf5 *=*mpich* + - {{ pin_compatible('libnetcdf', max_pin='x.x') }} + - {{ pin_compatible('netcdf-cxx4', max_pin='x.x') }} + - {{ pin_compatible('netcdf-fortran', max_pin='x.x') }} - python - - numpy + - {{ pin_compatible('numpy', max_pin='x.x') }} - mpi4py about: From 2e4832c2a05d939a2e5c50b5915f2a9b2b05ed98 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 1 Jun 2024 11:25:14 -0700 Subject: [PATCH 419/476] Fixes warnings and changes [=] to [&] --- components/eamxx/src/physics/mam/mam_coupling.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index a80b41f6c08b..ed156a9de893 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -735,7 +735,7 @@ void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, dim), [=](int kk) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, dim), [&](int kk) { out_view(icol, kk) = inp_view(icol, kk); }); }); @@ -851,7 +851,6 @@ void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; @@ -884,7 +883,6 @@ void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); } else { - int m = static_cast(mode_index); if (aero_id != NoAero) { // constituent is an aerosol species int a = aerosol_index_for_mode(mode_index, aero_id); const Real mw = mam4::aero_species(a).molecular_weight; From b8e86ba51e57a342903ab088feeb4563decb64d1 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Sat, 1 Jun 2024 15:06:20 -0500 Subject: [PATCH 420/476] use declarative build, move tools away --- components/eamxx/src/python/pyproject.toml | 17 ++++++ components/eamxx/src/python/readme | 5 +- components/eamxx/src/python/setup.py | 19 ------- components/eamxx/src/python/tools/build.sh | 31 ----------- .../src/python/tools/conda_build_config.yaml | 6 --- components/eamxx/src/python/tools/meta.yaml | 53 ------------------- 6 files changed, 19 insertions(+), 112 deletions(-) create mode 100644 components/eamxx/src/python/pyproject.toml delete mode 100644 components/eamxx/src/python/setup.py delete mode 100644 components/eamxx/src/python/tools/build.sh delete mode 100644 components/eamxx/src/python/tools/conda_build_config.yaml delete mode 100644 components/eamxx/src/python/tools/meta.yaml diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml new file mode 100644 index 000000000000..69a155439833 --- /dev/null +++ b/components/eamxx/src/python/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "pyeamxx" +version = "0.0.1.dev6" +dependencies = ["numpy", "mpi4py"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["pyeamxx", "libpyeamxx"] +exclude = ["build_src", "tests"] +namespaces = true + +[tool.setuptools.package-data] +"libpyeamxx" = ["*.so*"] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index 823ff9c0426b..8acf0f604298 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -1,12 +1,11 @@ INFO: - EAMxx python bindings -- libpyeamxx is where we will house the extensions - pyeamxx is where we will house the python code +- libpyeamxx is where we will house the extensions +- packaging moved to https://github.com/mahf708/experimental-scream-feedstock TODO: - add more python code to organize structures - establish ci pipeline in scream/e3sm repos -- add to cf (or alternatively sync with it) - decide how to deal with versioning - decide archs/pythons/mpis to tgt -- figure out what's tripping cf diff --git a/components/eamxx/src/python/setup.py b/components/eamxx/src/python/setup.py deleted file mode 100644 index 40b62d3c17cb..000000000000 --- a/components/eamxx/src/python/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from setuptools import setup, Distribution - - -class BinaryDistribution(Distribution): - def has_ext_modules(foo): - return True - -setup( - name='pyeamxx', - version='0.0.1.dev5', - author='E3SM SCREAM', - description='EAMxx wrapper', - packages=['', 'pyeamxx'], - package_data={ - '': ['libpyeamxx/*.so*'], - }, - distclass=BinaryDistribution, - zip_safe=False -) diff --git a/components/eamxx/src/python/tools/build.sh b/components/eamxx/src/python/tools/build.sh deleted file mode 100644 index 21757e8206b2..000000000000 --- a/components/eamxx/src/python/tools/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/env bash - -cd components/eamxx/src/python - -cmake \ - -S ../../ \ - -B build_src \ - -DCMAKE_BUILD_TYPE='Release' \ - -DEAMXX_ENABLE_PYBIND='ON' \ - -DNetcdf_Fortran_PATH=$PREFIX \ - -DNetcdf_C_PATH=$PREFIX \ - -DCMAKE_CXX_FLAGS='-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized' \ - -DCMAKE_C_FLAGS='' \ - -DCMAKE_Fortran_FLAGS='-Wno-maybe-uninitialized -Wno-unused-dummy-argument' \ - -DCMAKE_CXX_COMPILER=mpic++ \ - -DCMAKE_C_COMPILER=mpicc \ - -DCMAKE_Fortran_COMPILER=mpif90 \ - -DBUILD_SHARED_LIBS=ON \ - -DSCREAM_INPUT_ROOT="build_src" \ - -DSCREAM_ENABLE_BASELINE_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DPYTHON_EXECUTABLE=$PYTHON - -cmake --build build_src/src/python -j${CPU_COUNT:-128} - -find build_src -name "*.so*" | xargs cp -t libpyeamxx/ -for f in libpyeamxx/*.so*; do patchelf --set-rpath '$ORIGIN' --force-rpath $f; done - -$PYTHON -m build -w -n -x - -pip install dist/*.whl -vv diff --git a/components/eamxx/src/python/tools/conda_build_config.yaml b/components/eamxx/src/python/tools/conda_build_config.yaml deleted file mode 100644 index f5f68d8d781f..000000000000 --- a/components/eamxx/src/python/tools/conda_build_config.yaml +++ /dev/null @@ -1,6 +0,0 @@ - -# eamxx fails with gcc 13 -c_compiler_version: # [linux] - - 12 # [linux] -cxx_compiler_version: # [linux] - - 12 # [linux] diff --git a/components/eamxx/src/python/tools/meta.yaml b/components/eamxx/src/python/tools/meta.yaml deleted file mode 100644 index 0ee370bbbb55..000000000000 --- a/components/eamxx/src/python/tools/meta.yaml +++ /dev/null @@ -1,53 +0,0 @@ -package: - name: pyeamxx - version: 0.0.1.dev5 - -source: - path: ../../../../../ - -build: - number: 0 - -requirements: - build: - - cmake - # - {{ stdlib('c') }} - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler('fortran') }} - host: - - python - - pybind11 - - setuptools - - build - - spdlog - - fmt - - yaml-cpp - - mpich - - hdf5 *=*mpich* - - libnetcdf - - netcdf-cxx4 - - netcdf-fortran - - python - - numpy - - mpi4py - run: - - {{ pin_compatible('spdlog', max_pin='x.x') }} - - {{ pin_compatible('fmt', max_pin='x.x') }} - - {{ pin_compatible('yaml-cpp', max_pin='x.x') }} - - {{ pin_compatible('mpich', max_pin='x.x') }} - - hdf5 *=*mpich* - - {{ pin_compatible('libnetcdf', max_pin='x.x') }} - - {{ pin_compatible('netcdf-cxx4', max_pin='x.x') }} - - {{ pin_compatible('netcdf-fortran', max_pin='x.x') }} - - python - - {{ pin_compatible('numpy', max_pin='x.x') }} - - mpi4py - -about: - home: https://github.com/E3SM-Project/scream - summary: Python wrapper for EAMxx - -extra: - recipe-maintainers: - - scream-team From 54ab3665681edeb5e5d99ff44e2988fb8b4dcf1f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 4 Jun 2024 12:55:31 -0600 Subject: [PATCH 421/476] Fix valgrind errors in eamxx_homme_iop.cpp We were running off the end of a couple views. --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 66f73b5ac9a8..eaf65a69cda5 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -594,9 +594,9 @@ apply_iop_forcing(const Real dt) // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high // is in units of [hPa], thus convert iop_nudge_tq_low/high - Mask nudge_level; - for (int p=0; p Date: Tue, 4 Jun 2024 12:25:18 -0700 Subject: [PATCH 422/476] Fixes LABLES of the tests to change mam to mam4_aci for time reporting --- .../physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt | 4 ++-- .../CMakeLists.txt | 4 ++-- .../mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt | 4 ++-- .../physics_only/mam/shoc_mam4_aci/CMakeLists.txt | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt index 946d68cb7501..856ef3ca1622 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc cld_fraction p3 mam - LABELS shoc cld p3 physics mam + LABELS shoc cld p3 physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc cld p3 physics PEM mam + LABELS shoc cld p3 physics PEM mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt index c956df2b8fd8..3f9aeae73fda 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc cld_fraction p3 scream_rrtmgp mam - LABELS shoc cld p3 rrtmgp physics mam + LABELS shoc cld p3 rrtmgp physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc cld p3 rrtmgp physics PEM mam + LABELS shoc cld p3 rrtmgp physics PEM mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt index bb30c120d38b..cdd564c73a86 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_mam4_aci_p3_rrtmgp/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc cld_fraction p3 scream_rrtmgp mam - LABELS shoc cld p3 rrtmgp physics mam + LABELS shoc cld p3 rrtmgp physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc cld p3 rrtmgp physics PEM mam + LABELS shoc cld p3 rrtmgp physics PEM mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt index af6fff36b0a8..69a154223fd9 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_aci/CMakeLists.txt @@ -6,7 +6,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LIBS shoc mam - LABELS shoc physics mam + LABELS shoc physics mam4_aci MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -36,6 +36,6 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS shoc physics mam + LABELS shoc physics mam4_aci META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) From 699a486523b78d60753be78c2d9bef9f908af0d4 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 4 Jun 2024 21:35:04 -0700 Subject: [PATCH 423/476] Fixed a layout and Clang format --- .../physics/mam/eamxx_mam_aci_functions.hpp | 28 +++++++++---------- .../mam/eamxx_mam_aci_process_interface.cpp | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 6c9499da5ce1..9fedb64555ef 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -431,8 +431,7 @@ void call_function_dropmixnuc( // Construct state_q (interstitial) and qqcw (cloud borne) arrays constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, pver), - [&](int klev) { + Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -521,11 +520,12 @@ void update_cloud_borne_aerosols( // Update interstitial aerosols using tendencies - levels KOKKOS_INLINE_FUNCTION -void update_interstitial_aerosols_levs( - const haero::ThreadTeam &team, const int nlev, const int icol, - const Real dt, const MAMAci::view_2d ptend_view, - // output - MAMAci::view_2d aero_mr) { +void update_interstitial_aerosols_levs(const haero::ThreadTeam &team, + const int nlev, const int icol, + const Real dt, + const MAMAci::view_2d ptend_view, + // output + MAMAci::view_2d aero_mr) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { aero_mr(icol, kk) += ptend_view(icol, kk) * dt; }); @@ -569,8 +569,7 @@ void update_interstitial_aerosols( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, dt, - ptend_view, + update_interstitial_aerosols_levs(team, nlev, icol, dt, ptend_view, // output aero_nmr); }); @@ -675,12 +674,11 @@ void call_hetfrz_compute_tendencies( // assign cloud fraction constexpr auto pver = mam4::ndrop::pver; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, pver), - [&](int klev) { - diags.stratiform_cloud_fraction(klev) = - haero_atm.cloud_fraction(klev); - }); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, 0u, pver), + [&](int klev) { + diags.stratiform_cloud_fraction(klev) = + haero_atm.cloud_fraction(klev); + }); //------------------------------------------------------------- // Heterogeneous freezing // frzimm, frzcnt, frzdep are the outputs of diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index c5d6f5eb2043..252745679029 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -149,7 +149,7 @@ void MAMAci::set_grids( // Layout for 4D (2d horiz X 1d vertical x number of modes) variables const int num_aero_modes = mam_coupling::num_aero_modes(); FieldLayout scalar4d_layout_mid = - make_layout({ncol_, num_aero_modes, nlev_}, {"ncol", "nmodes", "lev"}); + make_layout({ncol_, num_aero_modes, nlev_}, {"COL", "NMODES", "LEV"}); // dry diameter of aerosols [m] add_field("dgnum", scalar4d_layout_mid, m, grid_name); From 3f01e52f8237dfd673603c23a51e099180833553 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 5 Jun 2024 07:14:40 -0700 Subject: [PATCH 424/476] Adds a cime test for aci --- cime_config/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 3223ae10804d..87249990a59c 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -690,6 +690,7 @@ "time" : "01:00:00", "tests" : ( "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", ) }, From b7a4aa0100d1a905305125ad455e0fe24a924047 Mon Sep 17 00:00:00 2001 From: mahf708 Date: Thu, 6 Jun 2024 13:41:40 -0500 Subject: [PATCH 425/476] set versions at 0.0.1 --- components/eamxx/src/python/pyproject.toml | 2 +- components/eamxx/src/python/readme | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index 69a155439833..4d1861a242e3 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyeamxx" -version = "0.0.1.dev6" +version = "0.0.1" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index 8acf0f604298..f5ed48f058ee 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -9,3 +9,7 @@ TODO: - establish ci pipeline in scream/e3sm repos - decide how to deal with versioning - decide archs/pythons/mpis to tgt + +USER: +- conda install pyeamxx -c mahf708/label/$mac (mac is chrysalis or pm-cpu) +- see example in components/eamxx/tests/python/pyp3 From da58272d4a232b0fc620a07f7d25606cd881db60 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 15:45:49 -0600 Subject: [PATCH 426/476] EAMxx: avoid nullptr when reading horiz remap sparse matrix entries A std vector of length 0 is not allocated, resulting in invalid memory --- .../src/share/grid/remap/horiz_interp_remapper_data.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index b26501a499b9..2f08e5a2be57 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -49,9 +49,11 @@ get_my_triplets (const std::string& map_file) const int nlweights = scorpio::get_dimlen_local(map_file,"n_s"); // 1.1 Read a chunk of triplets col indices - std::vector cols(nlweights); - std::vector rows(nlweights); - std::vector S(nlweights); + // NOTE: add 1 so that we don't pass nullptr to scorpio read routines (which would trigger + // a runtime error). Don't worry though: we never access the last entry of these vectors + std::vector cols(nlweights+1,-1); + std::vector rows(nlweights+1,-1); + std::vector S(nlweights+1,0); // Figure out if we are reading the right map, that is: // - n_a or n_b matches the fine grid ncols From 8a0159512803c81b6de1eff7306227ba835713c6 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 16:44:09 -0600 Subject: [PATCH 427/476] EAMxx: fix the previous fix --- .../src/share/grid/remap/horiz_interp_remapper_data.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index 2f08e5a2be57..6df6b14daf92 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -80,6 +80,12 @@ get_my_triplets (const std::string& map_file) const scorpio::read_var(map_file,"row",rows.data()); scorpio::read_var(map_file,"S" ,S.data()); + // Previously, we added 1 to their length, to avoid nullptr in scorpio::read. + // However, we later do range loops on these vectors, so resize them back to nlweights + cols.resize(nlweights); + rows.resize(nlweights); + S.resize(nlweights); + scorpio::release_file(map_file); // 1.2 Dofs in grid are likely 0-based, while row/col ids in map file From 6da60ae405103fed3408fa036978651408671ce9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 14:26:49 -0600 Subject: [PATCH 428/476] EAMxx: in pyeamxx, allow handling any eamxx physics parametrization --- .../src/python/libpyeamxx/CMakeLists.txt | 2 +- .../src/python/libpyeamxx/libpyeamxx.cpp | 2 - .../eamxx/src/python/libpyeamxx/pyatmproc.hpp | 87 +++++++++---------- .../eamxx/src/python/libpyeamxx/pygrid.hpp | 2 + .../eamxx/src/python/libpyeamxx/pyp3.cpp | 52 ----------- 5 files changed, 43 insertions(+), 102 deletions(-) delete mode 100644 components/eamxx/src/python/libpyeamxx/pyp3.cpp diff --git a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt index 1f3890eb3af4..9140e8f28c87 100644 --- a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt +++ b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) pybind11_add_module(libpyeamxx_ext libpyeamxx.cpp) -target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io p3) +target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io p3 scream_rrtmgp shoc spa cld_fraction) diff --git a/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp b/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp index 1713db23d119..f78eeb5a41bf 100644 --- a/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp +++ b/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp @@ -3,7 +3,6 @@ #include "pygrid.hpp" #include "pyatmproc.hpp" #include "pyparamlist.hpp" -#include "pyp3.cpp" #include #include @@ -35,7 +34,6 @@ PYBIND11_MODULE (libpyeamxx_ext,m) { pybind_pyfield(m); pybind_pygrid(m); pybind_pyatmproc(m); - pybind_pyp3(m); } } // namespace scream diff --git a/components/eamxx/src/python/libpyeamxx/pyatmproc.hpp b/components/eamxx/src/python/libpyeamxx/pyatmproc.hpp index a78ef5624834..623292af3e1d 100644 --- a/components/eamxx/src/python/libpyeamxx/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyeamxx/pyatmproc.hpp @@ -4,8 +4,10 @@ #include "share/atm_process/atmosphere_process.hpp" #include "share/io/scorpio_input.hpp" #include "share/io/scream_output_manager.hpp" +#include "physics/register_physics.hpp" #include "pygrid.hpp" #include "pyfield.hpp" +#include "pyparamlist.hpp" #include @@ -16,7 +18,7 @@ namespace scream { struct PyAtmProc { std::shared_ptr ap; - PyGrid pygrid; + PyGrid phys_grid; std::map fields; util::TimeStamp t0; util::TimeStamp time; @@ -24,65 +26,56 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const PyGrid& pyg) - : pygrid(pyg) - , t0({2000,1,1},{0,0,0}) + PyAtmProc (const PyParamList& params, const PyGrid& phys_grid_in) + : phys_grid(phys_grid_in) { - // TODO: make t0 configurable? + // Get the comm + const auto& comm = phys_grid.grid->get_comm(); + + // Create a grids manager on the fly + auto gm = std::make_shared(phys_grid.grid); + + // Create the atm proc + register_physics(); + auto& apf = AtmosphereProcessFactory::instance(); + const auto& ap_type = params.pl.isParameter("Type") + ? params.pl.get("Type") + : params.pl.name(); + ap = apf.create(ap_type,comm,params.pl); + + // Create the fields + ap->set_grids(gm); + create_fields(); } // I don't think virtual is needed, but just in case virtual ~PyAtmProc () = default; - std::map create_fields () { - std::map pyfields; + void create_fields () { + // Create fields that are input/output to the atm proc for (const auto& req : ap->get_required_field_requests()) { - if (pyfields.count(req.fid.name())==0) { - pyfields.emplace(req.fid.name(),PyField(req.fid,req.pack_size)); - } + const auto& fn = req.fid.name(); + auto it_bool = fields.emplace(fn,PyField(req.fid,req.pack_size)); + ap->set_required_field(it_bool.first->second.f.get_const()); } for (const auto& req : ap->get_computed_field_requests()) { - if (pyfields.count(req.fid.name())==0) { - pyfields.emplace(req.fid.name(),PyField(req.fid,req.pack_size)); - } - } - - return pyfields; - } - - void set_fields(const std::map& pyfields) { - for (const auto& it : pyfields) { - const auto& f = it.second.f; - const auto& fid = f.get_header().get_identifier(); - if (ap->has_required_field(fid)) { - ap->set_required_field(f.get_const()); - fields[fid.name()] = it.second; - } - if (ap->has_computed_field(fid)) { - ap->set_computed_field(f); - fields[fid.name()] = it.second; - } + const auto& fn = req.fid.name(); + auto it_bool = fields.emplace(fn,PyField(req.fid,req.pack_size)); + ap->set_computed_field(it_bool.first->second.f); } } PyField get_field(const std::string& name) { auto it = fields.find(name); - auto fnames = [&]() { - std::string s; - for (auto it : fields) { - if (s=="") { - s += it.first; - } else { - s += ", " + it.first; - } - } - return s; + auto print_key = [](auto it) { + return it.first; }; EKAT_REQUIRE_MSG (it!=fields.end(), - "Error! Field not found in list of P3 fields.\n" + "Error! Field not found among this atm proc fields.\n" + " - atm proc name: " + ap->name() + "\n" " - field name: " + name + "\n" - " - p3 fields: " + fnames() + "\n"); + " - atm proc fields: " + ekat::join(fields,print_key,",") + "\n"); return it->second; } @@ -123,7 +116,7 @@ struct PyAtmProc { } } } - AtmosphereInput reader (ic_filename,pygrid.grid,ic_fields,true); + AtmosphereInput reader (ic_filename,phys_grid.grid,ic_fields,true); reader.read_variables(); scorpio::release_file(ic_filename); @@ -136,7 +129,7 @@ struct PyAtmProc { auto params = ekat::parse_yaml_file(yaml_file); // Stuff all fields in a field manager - auto fm = std::make_shared(pygrid.grid); + auto fm = std::make_shared(phys_grid.grid); fm->registration_begins(); fm->registration_ends(); for (auto it : fields) { @@ -144,10 +137,10 @@ struct PyAtmProc { } // Create a grids manager on the fly - auto gm = std::make_shared(pygrid.grid); + auto gm = std::make_shared(phys_grid.grid); output_mgr = std::make_shared(); - output_mgr->setup(pygrid.grid->get_comm(),params,fm,gm,t0,t0,false); + output_mgr->setup(phys_grid.grid->get_comm(),params,fm,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); } @@ -164,7 +157,7 @@ struct PyAtmProc { inline void pybind_pyatmproc(pybind11::module& m) { pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + .def(pybind11::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) .def("setup_output",&PyAtmProc::setup_output) diff --git a/components/eamxx/src/python/libpyeamxx/pygrid.hpp b/components/eamxx/src/python/libpyeamxx/pygrid.hpp index 3ffb413a8052..aed51ead8894 100644 --- a/components/eamxx/src/python/libpyeamxx/pygrid.hpp +++ b/components/eamxx/src/python/libpyeamxx/pygrid.hpp @@ -35,6 +35,8 @@ class SingleGridGM : public GridsManager struct PyGrid { std::shared_ptr grid; + PyGrid () = default; + PyGrid(const std::string& name, int ncols, int nlevs) { ekat::Comm comm(MPI_COMM_WORLD); diff --git a/components/eamxx/src/python/libpyeamxx/pyp3.cpp b/components/eamxx/src/python/libpyeamxx/pyp3.cpp deleted file mode 100644 index 497cecb135fc..000000000000 --- a/components/eamxx/src/python/libpyeamxx/pyp3.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "physics/p3/eamxx_p3_process_interface.hpp" -#include "pygrid.hpp" -#include "pyfield.hpp" -#include "pyatmproc.hpp" -#include "pyparamlist.hpp" - -#include - -namespace scream { - -namespace py = pybind11; - -struct PyP3 : public PyAtmProc { - PyP3 (const PyGrid& pygrid, const PyParamList& params) - : PyAtmProc(pygrid) - { - init (params.pl); - } - PyP3 (const PyGrid& pygrid) - : PyAtmProc(pygrid) - { - // P3 does not have a default (in the C++ code) for this parameter, - // so an entry MUST be set - ekat::ParameterList pl; - pl.set("max_total_ni",740.0e3); - init(pl); - } - - void init(const ekat::ParameterList& params) { - auto comm = pygrid.grid->get_comm(); - ap = create_atmosphere_process(comm,params); - - // Create a grids manager on the fly - auto gm = std::make_shared(pygrid.grid); - ap->set_grids(gm); - - set_fields(create_fields()); - } -}; - -// PYBIND11_MODULE (pyp3,m) { - -// m.doc() = "Python interfaces to P3Microphysics"; -inline void pybind_pyp3(py::module& m) { - - py::class_(m,"P3") - .def(py::init()) - .def(py::init()); - -} - -} // namespace scream From 3deb588efeb009e06128ce62fa776fc054a0141a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 14:31:30 -0600 Subject: [PATCH 429/476] EAMxx: create interface library eamxx_physics and use it in pyeamxx Allows to link to all supported physics libs wihtout explicitly list them --- components/eamxx/src/physics/CMakeLists.txt | 2 ++ components/eamxx/src/physics/cld_fraction/CMakeLists.txt | 3 +++ components/eamxx/src/physics/cosp/CMakeLists.txt | 3 +++ components/eamxx/src/physics/mam/CMakeLists.txt | 3 +++ components/eamxx/src/physics/ml_correction/CMakeLists.txt | 3 +++ components/eamxx/src/physics/nudging/CMakeLists.txt | 3 +++ components/eamxx/src/physics/p3/CMakeLists.txt | 3 +++ components/eamxx/src/physics/rrtmgp/CMakeLists.txt | 3 +++ components/eamxx/src/physics/share/CMakeLists.txt | 3 +++ components/eamxx/src/physics/shoc/CMakeLists.txt | 3 +++ components/eamxx/src/physics/spa/CMakeLists.txt | 3 +++ components/eamxx/src/physics/tms/CMakeLists.txt | 3 +++ components/eamxx/src/python/libpyeamxx/CMakeLists.txt | 2 +- 13 files changed, 36 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/CMakeLists.txt b/components/eamxx/src/physics/CMakeLists.txt index 475fc588cd4a..032edfdf0c38 100644 --- a/components/eamxx/src/physics/CMakeLists.txt +++ b/components/eamxx/src/physics/CMakeLists.txt @@ -1,3 +1,5 @@ +add_library(eamxx_physics INTERFACE) + # Build shared physics functions first, so other parametrizations can use them add_subdirectory(share) diff --git a/components/eamxx/src/physics/cld_fraction/CMakeLists.txt b/components/eamxx/src/physics/cld_fraction/CMakeLists.txt index 45bbfa00f3df..ab1b249059de 100644 --- a/components/eamxx/src/physics/cld_fraction/CMakeLists.txt +++ b/components/eamxx/src/physics/cld_fraction/CMakeLists.txt @@ -14,6 +14,9 @@ target_compile_definitions(cld_fraction PUBLIC EAMXX_HAS_CLD_FRACTION) target_link_libraries(cld_fraction physics_share scream_share) target_compile_options(cld_fraction PUBLIC) +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE cld_fraction) + # Cloud fraction does not yet have a set of unit tests or a BFB test comparing with the F90 # code. # The cloud fraction stand alone test, in the /tests/ directory covers a range of property diff --git a/components/eamxx/src/physics/cosp/CMakeLists.txt b/components/eamxx/src/physics/cosp/CMakeLists.txt index 4428ee504cdc..e6ba4c363794 100644 --- a/components/eamxx/src/physics/cosp/CMakeLists.txt +++ b/components/eamxx/src/physics/cosp/CMakeLists.txt @@ -53,3 +53,6 @@ add_library(eamxx_cosp ${COSP_SRCS}) target_link_libraries(eamxx_cosp physics_share scream_share cosp) target_compile_options(eamxx_cosp PUBLIC) target_compile_definitions(eamxx_cosp PUBLIC EAMXX_HAS_COSP) + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE eamxx_cosp) diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 53eb4049f11b..969e2071aeb7 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -54,3 +54,6 @@ target_link_libraries(mam PUBLIC physics_share scream_share mam4xx haero) #if (NOT SCREAM_LIB_ONLY) # add_subdirectory(tests) #endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE mam) diff --git a/components/eamxx/src/physics/ml_correction/CMakeLists.txt b/components/eamxx/src/physics/ml_correction/CMakeLists.txt index d5c94a58d3cf..ac836018e12f 100644 --- a/components/eamxx/src/physics/ml_correction/CMakeLists.txt +++ b/components/eamxx/src/physics/ml_correction/CMakeLists.txt @@ -22,3 +22,6 @@ target_compile_definitions(ml_correction PUBLIC EAMXX_HAS_ML_CORRECTION) target_compile_definitions(ml_correction PRIVATE -DML_CORRECTION_CUSTOM_PATH="${CMAKE_CURRENT_SOURCE_DIR}") target_include_directories(ml_correction SYSTEM PUBLIC ${PYTHON_INCLUDE_DIRS}) target_link_libraries(ml_correction physics_share scream_share pybind11::pybind11 Python::Python) + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE ml_correction) diff --git a/components/eamxx/src/physics/nudging/CMakeLists.txt b/components/eamxx/src/physics/nudging/CMakeLists.txt index faf345d09012..7c209a1e0f20 100644 --- a/components/eamxx/src/physics/nudging/CMakeLists.txt +++ b/components/eamxx/src/physics/nudging/CMakeLists.txt @@ -5,3 +5,6 @@ target_link_libraries(nudging physics_share scream_share) if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE nudging) diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 7ea0aeeb8145..1a01f607c630 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -132,3 +132,6 @@ endif() if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE p3) diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index c098a0829d52..fcb819f599a8 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -226,3 +226,6 @@ if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE scream_rrtmgp) diff --git a/components/eamxx/src/physics/share/CMakeLists.txt b/components/eamxx/src/physics/share/CMakeLists.txt index 98e1336c8cb4..d3c64a4a0b9c 100644 --- a/components/eamxx/src/physics/share/CMakeLists.txt +++ b/components/eamxx/src/physics/share/CMakeLists.txt @@ -29,3 +29,6 @@ target_link_libraries(physics_share scream_share) if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE physics_share) diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index e37379095d00..891626b9d3df 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -132,3 +132,6 @@ endforeach() if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE shoc) diff --git a/components/eamxx/src/physics/spa/CMakeLists.txt b/components/eamxx/src/physics/spa/CMakeLists.txt index 26d2ba86e203..0a9477f70e11 100644 --- a/components/eamxx/src/physics/spa/CMakeLists.txt +++ b/components/eamxx/src/physics/spa/CMakeLists.txt @@ -5,3 +5,6 @@ target_link_libraries(spa physics_share scream_share) if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE spa) diff --git a/components/eamxx/src/physics/tms/CMakeLists.txt b/components/eamxx/src/physics/tms/CMakeLists.txt index b6e1715e5f32..c346ce8f2415 100644 --- a/components/eamxx/src/physics/tms/CMakeLists.txt +++ b/components/eamxx/src/physics/tms/CMakeLists.txt @@ -28,3 +28,6 @@ if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() + +# Add this library to eamxx_physics +target_link_libraries(eamxx_physics INTERFACE tms) diff --git a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt index 9140e8f28c87..3c91874ea4dd 100644 --- a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt +++ b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) pybind11_add_module(libpyeamxx_ext libpyeamxx.cpp) -target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io p3 scream_rrtmgp shoc spa cld_fraction) +target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io eamxx_physics) From 42405c7f319cebdc9c66b310f03764796a9b96b2 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 16:52:58 -0600 Subject: [PATCH 430/476] EAMxx: allow setting param list name in PyParamList --- components/eamxx/src/python/libpyeamxx/pyparamlist.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/python/libpyeamxx/pyparamlist.hpp b/components/eamxx/src/python/libpyeamxx/pyparamlist.hpp index 30e4f7af48f1..00ee1c894f84 100644 --- a/components/eamxx/src/python/libpyeamxx/pyparamlist.hpp +++ b/components/eamxx/src/python/libpyeamxx/pyparamlist.hpp @@ -16,6 +16,11 @@ struct PyParamList { parse_dict(d,pl); } + PyParamList(const pybind11::dict& d, const std::string& name) { + pl.rename(name); + parse_dict(d,pl); + } + void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { for (auto item : d) { const std::string key = pybind11::str(item.first); @@ -78,7 +83,8 @@ inline void pybind_pyparamlist (pybind11::module& m) { // Param list pybind11::class_(m,"ParameterList") - .def(pybind11::init()); + .def(pybind11::init()) + .def(pybind11::init()); } } // namespace scream From 651cba657ee9c6d3d901b5a518d9d1944e3c377f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 16:54:12 -0600 Subject: [PATCH 431/476] EAMxx: moved pyeamxx out of libpyeamxx folder --- components/eamxx/src/python/CMakeLists.txt | 5 ++++- components/eamxx/src/python/{libpyeamxx => }/libpyeamxx.cpp | 0 components/eamxx/src/python/libpyeamxx/CMakeLists.txt | 4 ---- components/eamxx/src/python/{libpyeamxx => }/pyatmproc.hpp | 0 components/eamxx/src/python/{libpyeamxx => }/pyfield.hpp | 0 components/eamxx/src/python/{libpyeamxx => }/pygrid.hpp | 0 components/eamxx/src/python/{libpyeamxx => }/pyparamlist.hpp | 0 7 files changed, 4 insertions(+), 5 deletions(-) rename components/eamxx/src/python/{libpyeamxx => }/libpyeamxx.cpp (100%) delete mode 100644 components/eamxx/src/python/libpyeamxx/CMakeLists.txt rename components/eamxx/src/python/{libpyeamxx => }/pyatmproc.hpp (100%) rename components/eamxx/src/python/{libpyeamxx => }/pyfield.hpp (100%) rename components/eamxx/src/python/{libpyeamxx => }/pygrid.hpp (100%) rename components/eamxx/src/python/{libpyeamxx => }/pyparamlist.hpp (100%) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 01df409551c2..3c91874ea4dd 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1 +1,4 @@ -add_subdirectory(libpyeamxx) +find_package(pybind11 REQUIRED) + +pybind11_add_module(libpyeamxx_ext libpyeamxx.cpp) +target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io eamxx_physics) diff --git a/components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp b/components/eamxx/src/python/libpyeamxx.cpp similarity index 100% rename from components/eamxx/src/python/libpyeamxx/libpyeamxx.cpp rename to components/eamxx/src/python/libpyeamxx.cpp diff --git a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt b/components/eamxx/src/python/libpyeamxx/CMakeLists.txt deleted file mode 100644 index 3c91874ea4dd..000000000000 --- a/components/eamxx/src/python/libpyeamxx/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -find_package(pybind11 REQUIRED) - -pybind11_add_module(libpyeamxx_ext libpyeamxx.cpp) -target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io eamxx_physics) diff --git a/components/eamxx/src/python/libpyeamxx/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp similarity index 100% rename from components/eamxx/src/python/libpyeamxx/pyatmproc.hpp rename to components/eamxx/src/python/pyatmproc.hpp diff --git a/components/eamxx/src/python/libpyeamxx/pyfield.hpp b/components/eamxx/src/python/pyfield.hpp similarity index 100% rename from components/eamxx/src/python/libpyeamxx/pyfield.hpp rename to components/eamxx/src/python/pyfield.hpp diff --git a/components/eamxx/src/python/libpyeamxx/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp similarity index 100% rename from components/eamxx/src/python/libpyeamxx/pygrid.hpp rename to components/eamxx/src/python/pygrid.hpp diff --git a/components/eamxx/src/python/libpyeamxx/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp similarity index 100% rename from components/eamxx/src/python/libpyeamxx/pyparamlist.hpp rename to components/eamxx/src/python/pyparamlist.hpp From 253774dd24b77435972057e5f16470b6243e380c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 16:55:56 -0600 Subject: [PATCH 432/476] EAMxx: renamed libpyeamxx_ext to pyeamxx --- components/eamxx/src/python/CMakeLists.txt | 4 ++-- components/eamxx/src/python/{libpyeamxx.cpp => pyeamxx.cpp} | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) rename components/eamxx/src/python/{libpyeamxx.cpp => pyeamxx.cpp} (89%) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 3c91874ea4dd..bb368e9c63e5 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) -pybind11_add_module(libpyeamxx_ext libpyeamxx.cpp) -target_link_libraries(libpyeamxx_ext PUBLIC scream_share scream_io eamxx_physics) +pybind11_add_module(pyeamxx pyeamxx.cpp) +target_link_libraries(pyeamxx PUBLIC scream_share scream_io eamxx_physics) diff --git a/components/eamxx/src/python/libpyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp similarity index 89% rename from components/eamxx/src/python/libpyeamxx.cpp rename to components/eamxx/src/python/pyeamxx.cpp index f78eeb5a41bf..b6fc908eb5f4 100644 --- a/components/eamxx/src/python/libpyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -5,8 +5,6 @@ #include "pyparamlist.hpp" #include -#include -#include namespace py = pybind11; namespace scream { @@ -21,7 +19,7 @@ void finalize () { finalize_scream_session(); } -PYBIND11_MODULE (libpyeamxx_ext,m) { +PYBIND11_MODULE (pyeamxx,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; From a27cd1c03087d5ca100852755ff791597fa757bf Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 17:21:19 -0600 Subject: [PATCH 433/476] EAMxx: make some tms sources PRIVATE With PUBLIC, the compiled mod files magically appeared in the build folder of targets that link against tms. I'm not sure if this is the right fix, but it seems to work --- components/eamxx/src/physics/tms/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/tms/CMakeLists.txt b/components/eamxx/src/physics/tms/CMakeLists.txt index c346ce8f2415..c8f4efa47384 100644 --- a/components/eamxx/src/physics/tms/CMakeLists.txt +++ b/components/eamxx/src/physics/tms/CMakeLists.txt @@ -16,7 +16,7 @@ endif() if (NOT SCREAM_LIB_ONLY) # For testing, add some more sources and modules directory - target_sources (tms PUBLIC + target_sources (tms PRIVATE ${SCREAM_BASE_DIR}/../eam/src/physics/cam/trb_mtn_stress.F90 ${CMAKE_CURRENT_SOURCE_DIR}/tms_iso_c.f90 ${CMAKE_CURRENT_SOURCE_DIR}/tms_functions_f90.cpp From b745a81c29502b0b86e7e1fc3ba71e107a84327f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 17:27:00 -0600 Subject: [PATCH 434/476] EAMxx: make diagnostics available in pyeamxx --- components/eamxx/src/python/CMakeLists.txt | 2 +- components/eamxx/src/python/pyatmproc.hpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index bb368e9c63e5..3e1dd22072fc 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,4 +1,4 @@ find_package(pybind11 REQUIRED) pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC scream_share scream_io eamxx_physics) +target_link_libraries(pyeamxx PUBLIC scream_share scream_io diagnostics eamxx_physics) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/pyatmproc.hpp index 623292af3e1d..4e7a88b21e67 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/pyatmproc.hpp @@ -1,10 +1,12 @@ #ifndef PYATMPROC_HPP #define PYATMPROC_HPP +#include "physics/register_physics.hpp" +#include "diagnostics/register_diagnostics.hpp" #include "share/atm_process/atmosphere_process.hpp" #include "share/io/scorpio_input.hpp" #include "share/io/scream_output_manager.hpp" -#include "physics/register_physics.hpp" + #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" @@ -139,6 +141,8 @@ struct PyAtmProc { // Create a grids manager on the fly auto gm = std::make_shared(phys_grid.grid); + // Make sure diagnostics are available, then create the output mgr + register_diagnostics(); output_mgr = std::make_shared(); output_mgr->setup(phys_grid.grid->get_comm(),params,fm,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); From de48b9fdec1e2c953867c59c6649c860c15925d5 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Tue, 11 Jun 2024 07:10:36 -0700 Subject: [PATCH 435/476] fixes to cmake to get standalone ml test working --- components/eamxx/cmake/machine-files/ruby-intel.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake index 9c6318da4952..70d8750df7ed 100644 --- a/components/eamxx/cmake/machine-files/ruby-intel.cmake +++ b/components/eamxx/cmake/machine-files/ruby-intel.cmake @@ -1,7 +1,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/ruby.cmake) set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE) +set(Python_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE) set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) From aa9b4ffedf2213e2b280f5e49ac7da31697b1102 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Tue, 11 Jun 2024 07:31:04 -0700 Subject: [PATCH 436/476] Fix bug in precip adjustment, add property check, to corrective-ML --- .../eamxx_ml_correction_process_interface.cpp | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 067d675b3c44..91f23a884fa7 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -3,6 +3,8 @@ #include "ekat/util/ekat_units.hpp" #include "share/field/field_utils.hpp" #include "physics/share/physics_constants.hpp" +#include "share/property_checks/field_lower_bound_check.hpp" +#include "share/property_checks/field_within_interval_check.hpp" namespace scream { // ========================================================================================= @@ -68,6 +70,7 @@ void MLCorrection::set_grids( add_group("tracers", grid_name, 1, Bundling::Required); } +// ========================================================================================= void MLCorrection::initialize_impl(const RunType /* run_type */) { fpe_mask = ekat::get_enabled_fpes(); ekat::disable_all_fpes(); // required for importing numpy @@ -81,6 +84,16 @@ void MLCorrection::initialize_impl(const RunType /* run_type */) { ML_model_uv = py_correction.attr("get_ML_model")(m_ML_model_path_uv); ML_model_sfc_fluxes = py_correction.attr("get_ML_model")(m_ML_model_path_sfc_fluxes); ekat::enable_fpes(fpe_mask); + + // Enforce bounds on quantities adjusted by ML using Field Property Checks + using LowerBound = FieldLowerBoundCheck; + add_postcondition_check(get_field_out("precip_liq_surf_mass"),m_grid,0,true); + add_postcondition_check(get_field_out("precip_ice_surf_mass"),m_grid,0,true); + // Add sanity property check for other variables. + using Interval = FieldWithinIntervalCheck; + add_postcondition_check(get_field_out("qv"),m_grid,0,0.2,true); + add_postcondition_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); + add_postcondition_check(get_field_out("horiz_winds"),m_grid,-400.0,400.0,false); } // ========================================================================================= @@ -101,8 +114,11 @@ void MLCorrection::run_impl(const double dt) { const auto &v = get_field_out("horiz_winds").get_component(1).get_view(); // For precipitation adjustment we need to track the change in column integrated 'qv' - decltype(qv) qv_told("", qv.extent(0), qv.extent(1)); - Kokkos::deep_copy(qv_told,qv); + // So we clone the original qv before ML changes the state so we can back out a qv_tend + // to use with precip adjustment. + auto qv_src = get_field_in("qv"); + auto qv_in = qv_src.clone(); + qv_in.deep_copy(qv_src); auto h_lat = m_lat.get_view(); auto h_lon = m_lon.get_view(); @@ -161,15 +177,25 @@ void MLCorrection::run_impl(const double dt) { Kokkos::parallel_for("Compute WVP diff", policy, KOKKOS_LAMBDA(const MT& team) { const int icol = team.league_rank(); - auto qold_icol = ekat::subview(qv_told,icol); + auto qold_icol = ekat::subview(qv_in,icol); auto qnew_icol = ekat::subview(qv_tnew,icol); auto rho_icol = ekat::subview(pseudo_density,icol); Real net_column_moistening = 0; // Compute WaterVaporPath Difference + // The water vapor path (WVP) is calculated as the integral of d_qv over the vertical column + // which is converted to the units of precipitation which are kg/m*m + // WVP = sum( d_qv * pseudo_density / gravity ), + // where d_qv = qv_new - qv_old + // units sanity check + // d_qv = kg/kg + // pseudo_density = Pa = kg/m/s2 + // gravity = m/s2 + // d_qv * pseduo_density / gravity = kg/kg * kg/m/s2 * s2/m = kg/m2 + // Compute WaterVaporPath Difference Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const int& ilev, Real& lsum) { lsum += (qnew_icol(ilev)-qold_icol(ilev)) * rho_icol(ilev) / g; - },net_column_moistening); + },,Kokkos::Sum(net_column_moistening)); team.team_barrier(); // Adjust Precipitation // - Note, we subtract the water vapor path because positive precip represents @@ -187,7 +213,7 @@ void MLCorrection::run_impl(const double dt) { // Apply all the adjustment to a single phase based on surface temperature Kokkos::single(Kokkos::PerTeam(team), [&] { auto T_icol = ekat::subview(T_mid,icol); - if (T_icol(m_num_levs-1)>273.15) { + if (T_icol(num_levs-1)>273.15) { precip_liq_surf_mass(icol) -= net_column_moistening; } else { precip_ice_surf_mass(icol) -= net_column_moistening; From 82fc3a69c7c850465e0b81a8500940839bd8bac4 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 11 Jun 2024 09:22:34 -0600 Subject: [PATCH 437/476] EAMxx: fix PyField::get to reinstate view semantic --- components/eamxx/src/python/pyfield.hpp | 87 ++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/pyfield.hpp index df6ed5012f31..8566f19a694c 100644 --- a/components/eamxx/src/python/pyfield.hpp +++ b/components/eamxx/src/python/pyfield.hpp @@ -25,7 +25,6 @@ struct PyField { } pybind11::array get () const { - const auto& fh = f.get_header(); const auto& fid = fh.get_identifier(); @@ -35,19 +34,33 @@ struct PyField { "Error! Cannot get the array for a field that is a subfield of another. Please, get array of parent field.\n" " - field name : " + fid.name() + "\n" " - parent name: " + fh.get_parent().lock()->get_identifier().name() + "\n"); - pybind11::dtype dt; + // Get array shape and strides. + // NOTE: since the field may be padded, the strides do not necessarily + // match the dims. Also, the strides must be grabbed from the + // actual view, since the layout doesn't know them. + pybind11::array::ShapeContainer shape (fid.get_layout().dims()); + std::vector strides; + + pybind11::dtype dt; switch (fid.data_type()) { - case DataType::IntType: dt = pybind11::dtype::of(); break; - case DataType::FloatType: dt = pybind11::dtype::of(); break; - case DataType::DoubleType: dt = pybind11::dtype::of(); break; - default: EKAT_ERROR_MSG ("Unrecognized/unsupported data type.\n"); + case DataType::IntType: + dt = get_dt_and_set_strides(strides); + break; + case DataType::FloatType: + dt = get_dt_and_set_strides(strides); + break; + case DataType::DoubleType: + dt = get_dt_and_set_strides(strides); + break; + default: + EKAT_ERROR_MSG ("Unrecognized/unsupported data type.\n"); } - pybind11::array::ShapeContainer shape (fid.get_layout().dims()); + // NOTE: you MUST set the parent handle, or else you won't have view semantic auto data = f.get_internal_view_data_unsafe(); - // auto data = reinterpret_cast(f.get_internal_view_data_unsafe()); - return pybind11::array(dt,shape,data); + auto this_obj = pybind11::cast(this); + return pybind11::array(dt,shape,strides,data,pybind11::handle(this_obj)); } void sync_to_host () { @@ -56,6 +69,62 @@ struct PyField { void print() const { print_field_hyperslab(f); } +private: + + template + pybind11::dtype get_dt_and_set_strides (std::vector& strides) const + { + strides.resize(f.rank()); + switch (f.rank()) { + case 1: + { + auto v = f.get_view(); + strides[0] = v.stride(0)*sizeof(T); + break; + } + case 2: + { + auto v = f.get_view(); + strides[0] = v.stride(0)*sizeof(T); + strides[1] = v.stride(1)*sizeof(T); + break; + } + case 3: + { + auto v = f.get_view(); + strides[0] = v.stride(0)*sizeof(T); + strides[1] = v.stride(1)*sizeof(T); + strides[2] = v.stride(2)*sizeof(T); + break; + } + case 4: + { + auto v = f.get_view(); + strides[0] = v.stride(0)*sizeof(T); + strides[1] = v.stride(1)*sizeof(T); + strides[2] = v.stride(2)*sizeof(T); + strides[3] = v.stride(3)*sizeof(T); + break; + } + case 5: + { + auto v = f.get_view(); + strides[0] = v.stride(0)*sizeof(T); + strides[1] = v.stride(1)*sizeof(T); + strides[2] = v.stride(2)*sizeof(T); + strides[3] = v.stride(3)*sizeof(T); + strides[4] = v.stride(4)*sizeof(T); + break; + } + default: + EKAT_ERROR_MSG ( + "Unsupported field rank in PyField.\n" + " - field name: " + f.name() + "\n" + " - field rnak: " + std::to_string(f.rank()) + "\n"); + } + + return pybind11::dtype::of(); + } }; inline void pybind_pyfield (pybind11::module& m) { From 8d8e2af9d8ebaf27caa4beb6b455ff56b5c80ccc Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 11 Jun 2024 09:22:55 -0600 Subject: [PATCH 438/476] EAMxx: add PyField::sync_to_dev interface --- components/eamxx/src/python/pyfield.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/pyfield.hpp index 8566f19a694c..eeb4125cecc2 100644 --- a/components/eamxx/src/python/pyfield.hpp +++ b/components/eamxx/src/python/pyfield.hpp @@ -66,6 +66,9 @@ struct PyField { void sync_to_host () { f.sync_to_host(); } + void sync_to_dev () { + f.sync_to_dev(); + } void print() const { print_field_hyperslab(f); } @@ -133,6 +136,7 @@ inline void pybind_pyfield (pybind11::module& m) { .def(pybind11::init<>()) .def("get",&PyField::get) .def("sync_to_host",&PyField::sync_to_host) + .def("sync_to_dev",&PyField::sync_to_dev) .def("print",&PyField::print); } From 6418837cebe6eb00aa7db1daba956c94b64d8e2d Mon Sep 17 00:00:00 2001 From: Hassan Beydoun Date: Tue, 11 Jun 2024 11:13:10 -0700 Subject: [PATCH 439/476] allows for computing qv_sat_ice when T < T_freeze for RH diagnostic --- components/eamxx/src/diagnostics/relative_humidity.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/diagnostics/relative_humidity.cpp b/components/eamxx/src/diagnostics/relative_humidity.cpp index 909213a95b65..7df76168815c 100644 --- a/components/eamxx/src/diagnostics/relative_humidity.cpp +++ b/components/eamxx/src/diagnostics/relative_humidity.cpp @@ -64,8 +64,12 @@ void RelativeHumidityDiagnostic::compute_diagnostic_impl() const int jpack = idx % npacks; const auto range_pack = ekat::range(jpack*Pack::n); const auto range_mask = range_pack < num_levs; - auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), false, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), - physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl"); + if (T_mid(icol,jpack) >= 273.15) + {auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), false, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), + physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl");} + else + {auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), true, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), + physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl");} RH(icol,jpack) = qv_mid(icol,jpack)/qv_sat_l; }); From 8cf5d386989ebfcba4f124f6fd28e04b26bd911e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 11 Jun 2024 13:20:34 -0600 Subject: [PATCH 440/476] EAMxx: add support for passing MPI comm from py to C in pyeamxx --- components/eamxx/cmake/Findmpi4py.cmake | 36 ++++++++++++++++++++++ components/eamxx/src/python/CMakeLists.txt | 3 +- components/eamxx/src/python/pyeamxx.cpp | 22 ++++++++++--- components/eamxx/src/python/pygrid.hpp | 16 ++++++++-- components/eamxx/src/python/pyutils.hpp | 23 ++++++++++++++ 5 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 components/eamxx/cmake/Findmpi4py.cmake create mode 100644 components/eamxx/src/python/pyutils.hpp diff --git a/components/eamxx/cmake/Findmpi4py.cmake b/components/eamxx/cmake/Findmpi4py.cmake new file mode 100644 index 000000000000..685b46ca5866 --- /dev/null +++ b/components/eamxx/cmake/Findmpi4py.cmake @@ -0,0 +1,36 @@ +# - FindMPI4PY +# Find mpi4py includes +# This module defines: +# MPI4PY_INCLUDE_DIR, where to find mpi4py.h, etc. +# MPI4PY_FOUND + +function (SetMpi4pyIncludeDir) +endfunction() + +if (NOT TARGET mpi4py) + # If user provided an include dir, we will use that, otherwise we'll ask python to find it + if (NOT MPI4PY_INCLUDE_DIR) + execute_process(COMMAND + "${PYTHON_EXECUTABLE}" "-c" "import mpi4py; print (mpi4py.get_include())" + OUTPUT_VARIABLE OUTPUT + RESULT_VARIABLE RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE) + if (RESULT) + set(MPI4PY_FOUND FALSE) + else () + set (MPI4PY_INCLUDE_DIR ${OUTPUT} CACHE PATH "Path to mpi4py include directory" FORCE) + endif() + endif() + + # If we still don't have an include dir, it means we have no mpi4py installed + if (NOT MPI4PY_INCLUDE_DIR) + set(MPI4PY_FOUND FALSE) + else () + add_library(mpi4py INTERFACE) + target_include_directories(mpi4py INTERFACE SYSTEM ${MPI4PY_INCLUDE_DIR}) + set(MPI4PY_FOUND TRUE) + endif() +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(mpi4py DEFAULT_MSG MPI4PY_INCLUDE_DIR) diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 3e1dd22072fc..8c7cf2aaba9c 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(pybind11 REQUIRED) +find_package(mpi4py REQUIRED) pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC scream_share scream_io diagnostics eamxx_physics) +target_link_libraries(pyeamxx PUBLIC mpi4py scream_share scream_io diagnostics eamxx_physics) diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/pyeamxx.cpp index b6fc908eb5f4..5834ca10f74f 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/pyeamxx.cpp @@ -1,19 +1,32 @@ -#include "scream_session.hpp" +#include #include "pyfield.hpp" #include "pygrid.hpp" #include "pyatmproc.hpp" #include "pyparamlist.hpp" +#include "pyutils.hpp" + +#include #include +#include + namespace py = pybind11; + namespace scream { -void initialize () { - ekat::Comm comm(MPI_COMM_WORLD); +void initialize (MPI_Comm mpi_comm) { + ekat::Comm comm(mpi_comm); initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); } + +void initialize () { + initialize(MPI_COMM_WORLD); +} +void initialize (pybind11::object py_comm) { + initialize(get_c_comm(py_comm)); +} void finalize () { scorpio::finalize_subsystem(); finalize_scream_session(); @@ -24,7 +37,8 @@ PYBIND11_MODULE (pyeamxx,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; // Scream Session - m.def("init",&initialize); + m.def("init",py::overload_cast<>(&initialize)); + m.def("init",py::overload_cast(&initialize)); m.def("finalize",&finalize); // Call all other headers' registration routines diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp index aed51ead8894..874b1aef17a0 100644 --- a/components/eamxx/src/python/pygrid.hpp +++ b/components/eamxx/src/python/pygrid.hpp @@ -4,9 +4,12 @@ #include "share/grid/grids_manager.hpp" #include "share/grid/point_grid.hpp" #include "pyfield.hpp" +#include "pyutils.hpp" #include +#include + namespace scream { // Small grids manager class, to hold a pre-built grid @@ -38,15 +41,24 @@ struct PyGrid { PyGrid () = default; PyGrid(const std::string& name, int ncols, int nlevs) + : PyGrid (name,ncols,nlevs,MPI_COMM_WORLD) + {} + + PyGrid(const std::string& name, int ncols, int nlevs, pybind11::object py_comm) + : PyGrid (name,ncols,nlevs,get_c_comm(py_comm)) + {} + + PyGrid(const std::string& name, int ncols, int nlevs, MPI_Comm mpi_comm) { - ekat::Comm comm(MPI_COMM_WORLD); + ekat::Comm comm(mpi_comm); grid = create_point_grid(name,ncols,nlevs,comm); } }; inline void pybind_pygrid (pybind11::module& m) { pybind11::class_(m,"Grid") - .def(pybind11::init()); + .def(pybind11::init()) + .def(pybind11::init()); } } // namespace scream diff --git a/components/eamxx/src/python/pyutils.hpp b/components/eamxx/src/python/pyutils.hpp new file mode 100644 index 000000000000..6467df57eda1 --- /dev/null +++ b/components/eamxx/src/python/pyutils.hpp @@ -0,0 +1,23 @@ +#ifndef PYUTILS_HPP +#define PYUTILS_HPP + +#include +#include +#include + +#include + +MPI_Comm get_c_comm (pybind11::object py_comm) { + if (import_mpi4py() < 0) { + throw pybind11::error_already_set(); + } + auto py_src = py_comm.ptr(); + if (not PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { + throw std::bad_cast(); + } + + auto comm_ptr = PyMPIComm_Get(py_src); + return *comm_ptr; +} + +#endif // PYUTILS_HPP From 0023fc1c56b4139bc3f01979752f2aab21616648 Mon Sep 17 00:00:00 2001 From: Hassan Beydoun Date: Tue, 11 Jun 2024 15:30:16 -0700 Subject: [PATCH 441/476] Turns out that the MurphyKoop function only needs the ice flag to determine whether to take ice into consideration. So the PR reduces to changing that flag to true. --- components/eamxx/src/diagnostics/relative_humidity.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/diagnostics/relative_humidity.cpp b/components/eamxx/src/diagnostics/relative_humidity.cpp index 7df76168815c..10f320dbf684 100644 --- a/components/eamxx/src/diagnostics/relative_humidity.cpp +++ b/components/eamxx/src/diagnostics/relative_humidity.cpp @@ -64,12 +64,8 @@ void RelativeHumidityDiagnostic::compute_diagnostic_impl() const int jpack = idx % npacks; const auto range_pack = ekat::range(jpack*Pack::n); const auto range_mask = range_pack < num_levs; - if (T_mid(icol,jpack) >= 273.15) - {auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), false, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), - physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl");} - else - {auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), true, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), - physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl");} + auto qv_sat_l = physics::qv_sat_wet(T_mid(icol,jpack), p_dry_mid(icol,jpack), true, range_mask, dp_wet(icol,jpack), dp_dry(icol,jpack), + physics::MurphyKoop, "RelativeHumidityDiagnostic::compute_diagnostic_impl"); RH(icol,jpack) = qv_mid(icol,jpack)/qv_sat_l; }); From 9051d7edfe3487ecd9659bd75007a6a41ab8f41e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 16 May 2024 11:26:41 -0600 Subject: [PATCH 442/476] EAMxx: enable kokkos bounds checks in full_debug standalone build --- components/eamxx/scripts/test_factory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index 3c480e3c6d8b..848aa0e31ffa 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -81,7 +81,8 @@ def __str__(self): class DBG(TestProperty): ############################################################################### - CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True")] + CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True"), + ("Kokkos_ENABLE_DEBUG_BOUNDS_CHECK", "True")] def __init__(self, _): TestProperty.__init__( From 64b6ea4cfb8578123e8f033e8fd9b01cbc76476e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 16 May 2024 12:46:39 -0600 Subject: [PATCH 443/476] EAMxx: do not make SP/FPE builds inherit DBG cmake args in test_factory.py --- components/eamxx/scripts/test_factory.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index 848aa0e31ffa..0b3f0c738c88 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -81,15 +81,13 @@ def __str__(self): class DBG(TestProperty): ############################################################################### - CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True"), - ("Kokkos_ENABLE_DEBUG_BOUNDS_CHECK", "True")] - def __init__(self, _): TestProperty.__init__( self, "full_debug", "debug", - self.CMAKE_ARGS, + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True"), + ("Kokkos_ENABLE_DEBUG_BOUNDS_CHECK", "True")] ) ############################################################################### @@ -101,7 +99,8 @@ def __init__(self, _): self, "full_sp_debug", "debug single precision", - DBG.CMAKE_ARGS + [("SCREAM_DOUBLE_PRECISION", "False")], + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True"), + ("SCREAM_DOUBLE_PRECISION", "False")], ) ############################################################################### @@ -113,7 +112,8 @@ def __init__(self, tas): self, "debug_nopack_fpe", "debug pksize=1 floating point exceptions on", - DBG.CMAKE_ARGS + [("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True"), + ("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], uses_baselines=False, on_by_default=(tas is not None and not tas.on_cuda()) ) From 7b2188de064dbb8ac9b4815e5449cdece93aacce Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 16 May 2024 17:02:37 -0600 Subject: [PATCH 444/476] EAMxx: fix OOB access in nudging unit test --- components/eamxx/src/physics/nudging/tests/nudging_tests.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 8870b4cce209..51773cb249df 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -169,6 +169,9 @@ TEST_CASE("nudging_tests") { auto fine_h = fine.get_view(); auto data_h = data.get_view(); const bool is_pmid = data.name()=="p_mid"; + const int nlevs_fine = 2*nlevs_data-1; + const int top = 0; + const int bot = nlevs_fine-1; for (int icol=0; icol Date: Thu, 16 May 2024 17:13:52 -0600 Subject: [PATCH 445/476] EAMxx: fix OOB access in aerocom_cld test --- components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp index 5ffaf5498e60..f2a686ca136c 100644 --- a/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aerocom_cld_test.cpp @@ -233,8 +233,8 @@ TEST_CASE("aerocom_cld") { // Case 5a: test independence of ice and liq fractions cd_v(0, 3) = 1.0; - cd_v(0, 8) = 1.0; - cd_v(0, 9) = 0.2; + cd_v(0, 7) = 1.0; + cd_v(0, 8) = 0.2; qc.deep_copy(1.0); qi.deep_copy(0.0); // zero ice! cd.sync_to_dev(); From de71513a012e838932ecfa815b24bff3881e4abe Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 11 Jun 2024 17:00:41 -0600 Subject: [PATCH 446/476] Update EKAT submodule Gets fix to ensure bounds checking in debug mode in one utility --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index fb4babcf3a24..1c944624fe48 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit fb4babcf3a24c14e9e94d71df152a7f097805c14 +Subproject commit 1c944624fe4888bf4cd7ddf14bec1e4e7e596504 From 967dea7ee16fab97657943ab3d53daad125e0a4e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 10 May 2024 17:40:04 -0600 Subject: [PATCH 447/476] EAMxx: add utility method to FieldLayout Allow to strip any tag from a list, without erroring out if not found --- components/eamxx/src/share/field/field_layout.cpp | 10 ++++++++++ components/eamxx/src/share/field/field_layout.hpp | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 1893bbada849..2ce924cd0a74 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -198,6 +198,16 @@ FieldLayout& FieldLayout::rename_dim (const FieldTag t, const std::string& n, co return *this; } +FieldLayout& FieldLayout::strip_dims (const std::vector& tags) +{ + for (auto t : tags) { + if (has_tag(t)) { + strip_dim(t,false); + } + } + return *this; +} + FieldLayout& FieldLayout::rename_dims (const std::map& new_names) { for (const auto& it : new_names) { diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 7c0eaf9f040f..018831f8818c 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -131,10 +131,13 @@ class FieldLayout { FieldLayout& append_dim (const FieldTag t, const int extent, const std::string& name); FieldLayout& rename_dim (const int idim, const std::string& n); FieldLayout& rename_dim (const FieldTag tag, const std::string& n, const bool throw_if_not_found = true); - FieldLayout& rename_dims (const std::map& new_names); // Does not throw if not found FieldLayout& reset_dim (const int idim, const int extent); FieldLayout& reset_dim (const FieldTag t, const int extent, const bool throw_if_not_found = true); + // These overload allow to remove/rename dims *if found*. They won't throw if layout does not have them + FieldLayout& strip_dims (const std::vector& tags); // Does not throw if not found + FieldLayout& rename_dims (const std::map& new_names); // Does not throw if not found + FieldLayout clone() const; // NOTE: congruent does not check the tags names. It only checks From 5271c0aac143c27828136a2a6940ab83bf6da32f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 10 May 2024 18:23:52 -0600 Subject: [PATCH 448/476] EAMxx: refactor vertical interpolation Do away with vert interp utility. In VerticalRemapper, just use ekat::LinInterp manually, and in FieldAtPressureLevel, we don't even need LinInterp. --- .../diagnostics/field_at_pressure_level.cpp | 123 ++-- .../diagnostics/field_at_pressure_level.hpp | 10 - .../share/grid/remap/vertical_remapper.cpp | 519 +++++++++------ .../share/grid/remap/vertical_remapper.hpp | 46 +- .../iop/intensive_observation_period.cpp | 1 - .../eamxx/src/share/tests/CMakeLists.txt | 3 - .../src/share/tests/vertical_interp_tests.cpp | 357 ---------- .../share/tests/vertical_remapper_tests.cpp | 28 +- .../util/scream_vertical_interpolation.hpp | 321 --------- .../scream_vertical_interpolation_impl.hpp | 627 ------------------ 10 files changed, 439 insertions(+), 1596 deletions(-) delete mode 100644 components/eamxx/src/share/tests/vertical_interp_tests.cpp delete mode 100644 components/eamxx/src/share/util/scream_vertical_interpolation.hpp delete mode 100644 components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index 132de63971fe..ed94e45bc9f2 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -1,7 +1,8 @@ #include "diagnostics/field_at_pressure_level.hpp" -#include "share/util/scream_vertical_interpolation.hpp" +#include "share/util/scream_universal_constants.hpp" #include "ekat/std_meta/ekat_std_utils.hpp" +#include "ekat/util/ekat_upper_bound.hpp" #include "ekat/util/ekat_units.hpp" namespace scream @@ -35,10 +36,7 @@ FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params) m_pressure_level *= 100; } - m_p_tgt = view_1d("",1); - Kokkos::deep_copy(m_p_tgt, m_pressure_level); - - m_mask_val = m_params.get("mask_value",Real(std::numeric_limits::max()/10.0)); + m_mask_val = m_params.get("mask_value",Real(constants::DefaultFillValue::value)); m_diag_name = m_field_name + "_at_" + location; } @@ -102,12 +100,6 @@ initialize_impl (const RunType /*run_type*/) m_diagnostic_output.get_header().set_extra_data("mask_data",diag_mask); m_diagnostic_output.get_header().set_extra_data("mask_value",m_mask_val); - // Allocate helper views - FieldLayout mask_src_layout( {COL, LEV}, {num_cols, m_num_levs}); - FieldIdentifier mask_src_fid ("mask_tmp",mask_src_layout, nondim, gname); - m_mask_field = Field(mask_src_fid); - m_mask_field.allocate_view(); - using stratts_t = std::map; // Propagate any io string attribute from input field to diag field @@ -122,46 +114,93 @@ initialize_impl (const RunType /*run_type*/) // ========================================================================================= void FieldAtPressureLevel::compute_diagnostic_impl() { - using namespace scream::vinterp; + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; //This is 2D source pressure - const Field& pressure_f = get_field_in(m_pressure_name); - const auto pressure = pressure_f.get_view(); - view_Nd pres(pressure.data(),pressure.extent_int(0),pressure.extent_int(1)); - // Kokkos::deep_copy(pres,pressure); - + const Field& p_src = get_field_in(m_pressure_name); + const auto p_src_v = p_src.get_view(); const Field& f = get_field_in(m_field_name); // The setup for interpolation varies depending on the rank of the input field: const int rank = f.rank(); - m_mask_field.deep_copy(1.0); - auto mask_v_tmp = m_mask_field.get_view(); + const auto& pl = p_src.get_header().get_identifier().get_layout(); + const int ncols = pl.dim(0); + const int nlevs = pl.dim(1); + + auto p_tgt = m_pressure_level; + auto mval = m_mask_val; if (rank==2) { - const auto f_data_src = f.get_view(); - //output field on new grid - auto d_data_tgt = m_diagnostic_output.get_view(); - view_Nd data_tgt_tmp(d_data_tgt.data(),d_data_tgt.extent_int(0),1); // Note, vertical interp wants a 2D view, so we create a temporary one - perform_vertical_interpolation(pres,m_p_tgt,f_data_src,data_tgt_tmp,m_num_levs,1,m_mask_val); - - // Track mask - auto mask = m_diagnostic_output.get_header().get_extra_data("mask_data"); - auto d_mask_tgt = mask.get_view(); - view_Nd mask_tgt_tmp(d_mask_tgt.data(),d_mask_tgt.extent_int(0),1); - perform_vertical_interpolation(pres,m_p_tgt,mask_v_tmp,mask_tgt_tmp,m_num_levs,1,0); + auto policy = KT::RangePolicy(0,ncols); + auto diag = m_diagnostic_output.get_view(); + auto mask = m_diagnostic_output.get_header().get_extra_data("mask_data").get_view(); + auto f_v = f.get_view(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int icol) { + auto x1 = ekat::subview(p_src_v,icol); + auto y1 = ekat::subview(f_v,icol); + auto beg = x1.data(); + auto end = beg + nlevs; + auto last = beg + (nlevs-1); + if (p_tgt<*beg or p_tgt>*last) { + diag(icol) = mval; + mask(icol) = 0; + } else { + auto ub = ekat::upper_bound(beg,end,p_tgt); + auto k1 = ub - beg; + if (k1==0) { + // Corner case: p_tgt==y1(0) + diag(icol) = y1(icol); + } else if (k1==nlevs) { + // Corner case: p_tgt==y1(nlevs-1) + diag(icol) = y1(nlevs-1); + } else { + // General case: interpolate between k1 and k1-1 + diag(icol) = y1(k1-1) + (y1(k1)-y1(k1-1))/(x1(k1) - x1(k1-1)) * (p_tgt-x1(k1-1)); + } + mask(icol) = 1; + } + }); } else if (rank==3) { - const auto f_data_src = f.get_view(); - //output field on new grid - auto d_data_tgt = m_diagnostic_output.get_view(); - view_Nd data_tgt_tmp(d_data_tgt.data(),d_data_tgt.extent_int(0),d_data_tgt.extent_int(1),1); - - perform_vertical_interpolation(pres,m_p_tgt,f_data_src,data_tgt_tmp,m_num_levs,1,m_mask_val); - - // Track mask - auto mask = m_diagnostic_output.get_header().get_extra_data("mask_data"); - auto d_mask_tgt = mask.get_view(); - view_Nd mask_tgt_tmp(d_mask_tgt.data(),d_mask_tgt.extent_int(0),1); - perform_vertical_interpolation(pres,m_p_tgt,mask_v_tmp,mask_tgt_tmp,m_num_levs,1,0); + const int ndims = f.get_header().get_identifier().get_layout().get_vector_dim(); + auto policy = KT::TeamPolicy(ncols,ndims); + auto diag = m_diagnostic_output.get_view(); + auto mask = m_diagnostic_output.get_header().get_extra_data("mask_data").get_view(); + auto f_v = f.get_view(); + Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const MemberType& team) { + int icol = team.league_rank(); + auto x1 = ekat::subview(p_src_v,icol); + auto beg = x1.data(); + auto end = beg + nlevs; + auto last = beg + (nlevs-1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,ndims),[&](const int idim) { + if (p_tgt<*beg or p_tgt>*last) { + diag(icol,idim) = mval; + Kokkos::single(Kokkos::PerTeam(team),[&]{ + mask(icol) = 0; + }); + } else { + auto y1 = ekat::subview(f_v,icol,idim); + auto ub = ekat::upper_bound(beg,end,p_tgt); + auto k1 = ub - beg; + if (k1==0) { + // Corner case: p_tgt==y1(0) + diag(icol,idim) = y1(icol); + } else if (k1==nlevs) { + // Corner case: p_tgt==y1(nlevs-1) + diag(icol,idim) = y1(nlevs-1); + } else { + // General case: interpolate between k1 and k1-1 + diag(icol,idim) = y1(k1-1) + (y1(k1)-y1(k1-1))/(x1(k1) - x1(k1-1)) * (p_tgt-x1(k1-1)); + } + Kokkos::single(Kokkos::PerTeam(team),[&]{ + mask(icol) = 1; + }); + } + }); + }); } else { EKAT_ERROR_MSG("Error! field at pressure level only supports fields ranks 2 and 3 \n"); } diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp index bdad35321f99..950c0c5e2ee9 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.hpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.hpp @@ -16,12 +16,6 @@ class FieldAtPressureLevel : public AtmosphereDiagnostic { public: - using KT = KokkosTypes; - template - using view_1d = typename KT::template view_1d; - template - using view_2d = typename KT::template view_2d; - // Constructors FieldAtPressureLevel (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -39,14 +33,10 @@ class FieldAtPressureLevel : public AtmosphereDiagnostic protected: void initialize_impl (const RunType /*run_type*/); - using Pack1 = ekat::Pack; - std::string m_pressure_name; std::string m_field_name; std::string m_diag_name; - view_1d m_p_tgt; - Field m_mask_field; Real m_pressure_level; int m_num_levs; Real m_mask_val; diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index c899bef26dc1..a4c474685733 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -1,7 +1,6 @@ #include "vertical_remapper.hpp" #include "share/grid/point_grid.hpp" -#include "share/util/scream_vertical_interpolation.hpp" #include "share/io/scorpio_input.hpp" #include "share/field/field_tag.hpp" #include "share/field/field_identifier.hpp" @@ -16,12 +15,13 @@ namespace scream { + VerticalRemapper:: VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, - const Field& lev_prof, - const Field& ilev_prof) - : VerticalRemapper(src_grid,map_file,lev_prof,ilev_prof,constants::DefaultFillValue::value) + const Field& pmid_src, + const Field& pint_src) + : VerticalRemapper(src_grid,map_file,pmid_src,pint_src,constants::DefaultFillValue::value) { // Nothing to do here } @@ -29,8 +29,8 @@ VerticalRemapper (const grid_ptr_type& src_grid, VerticalRemapper:: VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, - const Field& lev_prof, - const Field& ilev_prof, + const Field& pmid_src, + const Field& pint_src, const Real mask_val) : AbstractRemapper() , m_comm (src_grid->get_comm()) @@ -55,21 +55,21 @@ VerticalRemapper (const grid_ptr_type& src_grid, // as the source field, but will have a different number of // vertical levels. scorpio::register_file(map_file,scorpio::FileMode::Read); - m_num_remap_levs = scorpio::get_dimlen(map_file,"lev"); + auto nlevs_tgt = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); - tgt_grid->reset_num_vertical_lev(m_num_remap_levs); + tgt_grid->reset_num_vertical_lev(nlevs_tgt); this->set_grids(src_grid,tgt_grid); // Set the LEV and ILEV vertical profiles for interpolation from - register_vertical_source_field(lev_prof); - register_vertical_source_field(ilev_prof); + set_source_pressure_fields(pmid_src,pint_src); // Gather the pressure level data for vertical remapping set_pressure_levels(map_file); // Add tgt pressure levels to the tgt grid - tgt_grid->set_geometry_data(m_remap_pres); + tgt_grid->set_geometry_data(m_tgt_pressure); + scorpio::release_file(map_file); } @@ -144,269 +144,404 @@ set_pressure_levels(const std::string& map_file) } using namespace ShortFieldTagsNames; - std::vector tags = {LEV}; - std::vector dims = {m_num_remap_levs}; - FieldLayout layout(tags,dims); + auto layout = m_tgt_grid->get_vertical_layout(true); FieldIdentifier fid("p_levs",layout,ekat::units::Pa,m_tgt_grid->name()); - m_remap_pres = Field(fid); - m_remap_pres.get_header().get_alloc_properties().request_allocation(mPack::n); - m_remap_pres.allocate_view(); + m_tgt_pressure = Field(fid); + // Just in case input fields are packed + m_tgt_pressure.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + m_tgt_pressure.allocate_view(); - auto remap_pres_scal = m_remap_pres.get_view(); + auto remap_pres_data = m_tgt_pressure.get_view().data(); + scorpio::read_var(map_file,"p_levs",remap_pres_data); - scorpio::read_var(map_file,"p_levs",remap_pres_scal.data()); - - m_remap_pres.sync_to_dev(); + m_tgt_pressure.sync_to_dev(); } void VerticalRemapper:: -register_vertical_source_field(const Field& src) +set_source_pressure_fields(const Field& pmid, const Field& pint) { using namespace ShortFieldTagsNames; - EKAT_REQUIRE_MSG(src.is_allocated(), - "Error! Vertical level source field is not yet allocated.\n" - " - field name: " + src.name() + "\n"); - - const auto& layout = src.get_header().get_identifier().get_layout(); - const auto vert_tag = layout.tags().back(); - EKAT_REQUIRE_MSG (vert_tag==LEV or vert_tag==ILEV, - "Error! Input vertical level field does not have a vertical level tag at the end.\n" - " - field name: " + src.name() + "\n" - " - field layout: " + layout.to_string() + "\n"); - - if (vert_tag==LEV) { - m_src_mid = src; - m_mid_set = true; - } else { - m_src_int = src; - m_int_set = true; - } + EKAT_REQUIRE_MSG(pmid.is_allocated(), + "Error! Source midpoint pressure field is not yet allocated.\n" + " - field name: " + pmid.name() + "\n"); + + EKAT_REQUIRE_MSG(pint.is_allocated(), + "Error! Source interface pressure field is not yet allocated.\n" + " - field name: " + pint.name() + "\n"); + + const auto& pmid_layout = pmid.get_header().get_identifier().get_layout(); + const auto& pint_layout = pint.get_header().get_identifier().get_layout(); + EKAT_REQUIRE_MSG(pmid_layout.congruent(m_src_grid->get_3d_scalar_layout(true)), + "Error! Source midpoint pressure field has the wrong layout.\n" + " - field name: " + pmid.name() + "\n" + " - field layout: " + pmid_layout.to_string() + "\n" + " - expected layout: " + m_src_grid->get_3d_scalar_layout(true).to_string() + "\n"); + EKAT_REQUIRE_MSG(pint_layout.congruent(m_src_grid->get_3d_scalar_layout(false)), + "Error! Source interface pressure field has the wrong layout.\n" + " - field name: " + pint.name() + "\n" + " - field layout: " + pint_layout.to_string() + "\n" + " - expected layout: " + m_src_grid->get_3d_scalar_layout(false).to_string() + "\n"); + + m_src_pmid = pmid; + m_src_pint = pint; } void VerticalRemapper:: do_register_field (const identifier_type& src, const identifier_type& tgt) { - m_src_fields.push_back(field_type(src)); - field_type tgt_f(tgt); - m_tgt_fields.push_back(tgt_f); + using namespace ShortFieldTagsNames; + + // Note, for vertical remapper we set all target fields as having LEV as the vertical dimension. + // So we check that all other tags between source and target match, but skip vert tag (since we + // could have src with ILEV and tgt with LEV) + auto src_layout = src.get_layout().clone(); + auto tgt_layout = tgt.get_layout().clone(); + EKAT_REQUIRE_MSG(src_layout.strip_dims({ILEV,LEV}).congruent(tgt_layout.strip_dims({LEV})), + "[VerticalRemapper] Error! Once vertical level tag is stripped, src/tgt layouts are incompatible.\n" + " - src field name: " + src.name() + "\n" + " - tgt field name: " + tgt.name() + "\n" + " - src field layout: " + src_layout.to_string() + "\n" + " - tgt field layout: " + tgt_layout.to_string() + "\n"); + + m_src_fields.emplace_back(src); + m_tgt_fields.emplace_back(tgt); } void VerticalRemapper:: do_bind_field (const int ifield, const field_type& src, const field_type& tgt) { using namespace ShortFieldTagsNames; - auto name = src.name(); - auto src_layout = src.get_header().get_identifier().get_layout(); - auto tgt_layout = tgt.get_header().get_identifier().get_layout(); - const bool has_ilev = src_layout.has_tag(ILEV); - EKAT_REQUIRE_MSG(src_layout.rank()==tgt_layout.rank(), - "ERROR! vert_remap:do_bind_field:" + name + ", tgt and src do not have the same rank"); - // Note, for vertical remapper we set all target fields as having LEV as the vertical dimension. So we check that all other tags - // between source and target match if source has ILEV - if (has_ilev) { - EKAT_REQUIRE_MSG(src_layout.clone().strip_dim(ILEV).tags()==tgt_layout.clone().strip_dim(LEV).tags(), - "ERROR! vert_remap:do_bind_field:" + name + ", tgt and src do not have the same set of field tags"); - } else { - EKAT_REQUIRE_MSG(src_layout.tags()==tgt_layout.tags(), - "ERROR! vert_remap:do_bind_field:" + name + ", tgt and src do not have the same set of field tags"); - } - - EKAT_REQUIRE_MSG ( - src_layout.rank()>1 || - src.get_header().get_alloc_properties().get_padding()==0, - "Error! vert_remap:do_bind_field:check_src:" + name + ", We don't support 2d scalar fields that are padded.\n"); - EKAT_REQUIRE_MSG ( - tgt_layout.rank()>1 || - tgt.get_header().get_alloc_properties().get_padding()==0, - "Error! vert_remap:do_bind_field:check_tgt:" + name + ", We don't support 2d scalar fields that are padded.\n"); + using PackT = ekat::Pack; m_src_fields[ifield] = src; m_tgt_fields[ifield] = tgt; - // Add mask tracking to the target field - using namespace ShortFieldTagsNames; - using namespace ekat::units; - auto nondim = Units::nondimensional(); - if (src_layout.has_tag(LEV) || src_layout.has_tag(ILEV)) { - auto& f_tgt = m_tgt_fields[ifield]; + // Clone src layout, since we may strip dims later for mask creation + auto src_layout = src.get_header().get_identifier().get_layout().clone(); + + auto& f_tgt = m_tgt_fields[ifield]; // Nonconst, since we need to set extra data in the header + if (src_layout.has_tag(LEV) or src_layout.has_tag(ILEV)) { + // Determine if this field can be handled with packs, and whether it's at midpoints + // Add mask tracking to the target field. The mask tracks location of tgt pressure levs that are outside the + // bounds of the src pressure field, and hence cannot be recovered by interpolation + auto& ft = m_field2type[src.name()]; + ft.midpoints = src.get_header().get_identifier().get_layout().has_tag(LEV); + ft.packed = src.get_header().get_alloc_properties().is_compatible() and + tgt.get_header().get_alloc_properties().is_compatible(); + // NOTE: for now we assume that masking is determined only by the COL,LEV location in space // and that fields with multiple components will have the same masking for each component // at a specific COL,LEV - auto src_lay = src_layout; - auto tags = src_lay.tags(); - for (auto tag : tags) { - if (tag != COL && tag != LEV && tag != ILEV) { - src_lay.strip_dim(tag); - } - } - const auto lname = src.get_header().get_identifier().get_id_string()+"_mask"; - bool found = false; - // Check if a field with lname has already been created: - for (unsigned ii=0; iiname() ); - Field mask_src_fld (mask_src_fid); - mask_src_fld.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); - mask_src_fld.allocate_view(); - const auto& tgt_lay = create_tgt_layout(src_lay); - FieldIdentifier mask_tgt_fid (lname, tgt_lay, nondim, m_tgt_grid->name() ); - Field mask_tgt_fld (mask_tgt_fid); - mask_tgt_fld.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); - mask_tgt_fld.allocate_view(); + src_layout.strip_dims({CMP}); + + // Check if this mask has already been created + const auto mask_name = m_tgt_grid->name() + "_" + ekat::join(src_layout.names(),"_") + "_mask"; + if (m_field2type.count(mask_name)==0) { + auto nondim = ekat::units::Units::nondimensional(); + // Create this src/tgt mask fields, and assign them to these src/tgt fields extra data + + FieldIdentifier src_mask_fid (mask_name, src_layout, nondim, m_src_grid->name() ); + FieldIdentifier tgt_mask_fid = create_tgt_fid(src_mask_fid); + + Field src_mask (src_mask_fid); + Field tgt_mask (tgt_mask_fid); + + src_mask.allocate_view(); + tgt_mask.allocate_view(); + EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_data"), - "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); - f_tgt.get_header().set_extra_data("mask_data",mask_tgt_fld); + "[VerticalRemapper::do_bind_field] Error! Target field already has mask data assigned.\n" + " - tgt field name: " + tgt.name() + "\n"); EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_value"), - "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); + "[VerticalRemapper::do_bind_field] Error! Target field already has mask value assigned.\n" + " - tgt field name: " + tgt.name() + "\n"); + + f_tgt.get_header().set_extra_data("mask_data",tgt_mask); f_tgt.get_header().set_extra_data("mask_value",m_mask_val); - m_src_masks.push_back(mask_src_fld); - m_tgt_masks.push_back(mask_tgt_fld); + m_src_masks.push_back(src_mask); + m_tgt_masks.push_back(tgt_mask); + + auto& mt = m_field2type[src_mask_fid.name()]; + mt.packed = false; + mt.midpoints = src_layout.has_tag(LEV); } } else { // If a field does not have LEV or ILEV it may still have mask tracking assigned from somewhere else. + // For instance, this could be a 2d field computed by FieldAtPressureLevel diagnostic. // In those cases we want to copy that mask tracking to the target field. - // Note, we still make a new field to ensure it is defined on the target grid. if (src.get_header().has_extra_data("mask_data")) { - auto f_src_mask = src.get_header().get_extra_data("mask_data"); - FieldIdentifier mask_tgt_fid (f_src_mask.name(), f_src_mask.get_header().get_identifier().get_layout(), nondim, m_tgt_grid->name() ); - Field mask_tgt_fld (mask_tgt_fid); - mask_tgt_fld.allocate_view(); - mask_tgt_fld.deep_copy(f_src_mask); - - auto& f_tgt = m_tgt_fields[ifield]; EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_data"), - "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); - f_tgt.get_header().set_extra_data("mask_data",mask_tgt_fld); + "[VerticalRemapper::do_bind_field] Error! Target field already has mask data assigned.\n" + " - tgt field name: " + tgt.name() + "\n"); + auto src_mask = src.get_header().get_extra_data("mask_data"); + f_tgt.get_header().set_extra_data("mask_data",src_mask); + } + if (src.get_header().has_extra_data("mask_value")) { EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_value"), - "ERROR VerticalRemapper::do_bind_field " + src.name() + " already has mask_data assigned!"); - f_tgt.get_header().set_extra_data("mask_value",m_mask_val); + "[VerticalRemapper::do_bind_field] Error! Target field already has mask value assigned.\n" + " - tgt field name: " + tgt.name() + "\n"); + auto src_mask_val = src.get_header().get_extra_data("mask_value"); + f_tgt.get_header().set_extra_data("mask_data",src_mask_val); } } + + if (this->m_num_bound_fields==this->m_num_registered_fields) { + create_lin_interp (); + } } void VerticalRemapper::do_registration_ends () { - // Check that the vertical profiles for the source data have been set - EKAT_REQUIRE_MSG(m_mid_set,"Error::VerticalRemapper:registration_ends,\n" - "Field for vertical profile of the source data for layout LEV has not been set.\n"); - EKAT_REQUIRE_MSG(m_int_set,"Error::VerticalRemapper:registration_ends,\n" - "Field for vertical profile of the source data for layout ILEV has not been set.\n"); + if (this->m_num_bound_fields==this->m_num_registered_fields) { + create_lin_interp (); + } +} + +void VerticalRemapper::create_lin_interp() +{ + // Count number fields for each packed-midpoints value + auto beg = m_field2type.begin(); + auto end = m_field2type.end(); + + int num_packed_mid = + std::count_if(beg,end,[](const std::pair& it) + { + return it.second.midpoints and it.second.packed; + }); + int num_packed_int = + std::count_if(beg,end,[](const std::pair& it) + { + return not it.second.midpoints and it.second.packed; + }); + int num_scalar_mid = + std::count_if(beg,end,[](const std::pair& it) + { + return it.second.midpoints and not it.second.packed; + }); + int num_scalar_int = + std::count_if(beg,end,[](const std::pair& it) + { + return not it.second.midpoints and not it.second.packed; + }); + + // Create the linear interpolation object + const auto ncols = m_src_grid->get_num_local_dofs(); + const auto nlevs_src = m_src_grid->get_num_vertical_levels(); + const auto nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); + + if (num_packed_mid>0) { + m_lin_interp_mid_packed = + std::make_shared>(ncols,nlevs_src,nlevs_tgt); + } + if (num_scalar_mid>0) { + m_lin_interp_mid_scalar = + std::make_shared>(ncols,nlevs_src,nlevs_tgt); + } + if (num_packed_int>0) { + m_lin_interp_int_packed = + std::make_shared>(ncols,nlevs_src,nlevs_tgt); + } + if (num_scalar_int>0) { + m_lin_interp_int_scalar = + std::make_shared>(ncols,nlevs_src,nlevs_tgt); + } } void VerticalRemapper::do_remap_fwd () { + // 1. Setup any interp object that was created (if nullptr, no fields need it) + if (m_lin_interp_mid_packed) { + setup_lin_interp(*m_lin_interp_mid_packed,m_src_pmid); + } + if (m_lin_interp_int_packed) { + setup_lin_interp(*m_lin_interp_int_packed,m_src_pint); + } + if (m_lin_interp_mid_scalar) { + setup_lin_interp(*m_lin_interp_mid_scalar,m_src_pmid); + } + if (m_lin_interp_int_scalar) { + setup_lin_interp(*m_lin_interp_int_scalar,m_src_pint); + } + using namespace ShortFieldTagsNames; - // Loop over each field - const auto& tgt_pres_ap = m_remap_pres.get_header().get_alloc_properties(); + + // 2. Interpolate the fields for (int i=0; i{ILEV,LEV},src_tag); - if (do_remap) { - // Dispatch kernel with the largest possible pack size - const auto& src_ap = f_src.get_header().get_alloc_properties(); - const auto& tgt_ap = f_tgt.get_header().get_alloc_properties(); - const auto& src_pres_ap = src_tag == LEV ? m_src_mid.get_header().get_alloc_properties() : m_src_int.get_header().get_alloc_properties(); - if (src_ap.is_compatible>() && - tgt_ap.is_compatible>() && - src_pres_ap.is_compatible>() && - tgt_pres_ap.is_compatible>()) { - apply_vertical_interpolation(f_src,f_tgt); + auto& f_tgt = m_tgt_fields[i]; + const auto& tgt_layout = f_tgt.get_header().get_identifier().get_layout(); + if (tgt_layout.has_tag(LEV)) { + const auto& type = m_field2type.at(f_src.name()); + // Dispatch interpolation to the proper lin interp object + if (type.midpoints) { + if (type.packed) { + apply_vertical_interpolation(*m_lin_interp_mid_packed,f_src,f_tgt,m_src_pmid,m_mask_val); + } else { + apply_vertical_interpolation(*m_lin_interp_mid_scalar,f_src,f_tgt,m_src_pmid,m_mask_val); + } } else { - apply_vertical_interpolation<1>(f_src,f_tgt); + if (type.packed) { + apply_vertical_interpolation(*m_lin_interp_int_packed,f_src,f_tgt,m_src_pint,m_mask_val); + } else { + apply_vertical_interpolation(*m_lin_interp_int_scalar,f_src,f_tgt,m_src_pint,m_mask_val); + } } } else { - // There is nothing to do, this field cannot be vertically interpolated, + // There is nothing to do, this field does not need vertical interpolation, // so just copy it over. Note, if this field has its own mask data make // sure that is copied too. + f_tgt.deep_copy(f_src); if (f_tgt.get_header().has_extra_data("mask_data")) { auto f_tgt_mask = f_tgt.get_header().get_extra_data("mask_data"); auto f_src_mask = f_src.get_header().get_extra_data("mask_data"); f_tgt_mask.deep_copy(f_src_mask); } - f_tgt.deep_copy(f_src); } } + + // 3. Interpolate the mask fields for (unsigned i=0; i{ILEV,LEV},src_tag); - if (do_remap) { - // If we are remapping then we need to initialize the mask source values to 1.0 - f_src.deep_copy(1.0); - // Dispatch kernel with the largest possible pack size - const auto& src_ap = f_src.get_header().get_alloc_properties(); - const auto& tgt_ap = f_tgt.get_header().get_alloc_properties(); - const auto& src_pres_ap = src_tag == LEV ? m_src_mid.get_header().get_alloc_properties() : m_src_int.get_header().get_alloc_properties(); - if (src_ap.is_compatible>() && - tgt_ap.is_compatible>() && - src_pres_ap.is_compatible>() && - tgt_pres_ap.is_compatible>()) { - apply_vertical_interpolation(f_src,f_tgt,true); + auto& f_src = m_src_masks[i]; + auto& f_tgt = m_tgt_masks[i]; + const auto& type = m_field2type.at(f_src.name()); + + // Initialize the mask source values to 1.0 + f_src.deep_copy(1.0); + + // Dispatch interpolation to the proper lin interp object + if (type.midpoints) { + if (type.packed) { + apply_vertical_interpolation(*m_lin_interp_mid_packed,f_src,f_tgt,m_src_pmid,0); } else { - apply_vertical_interpolation<1>(f_src,f_tgt,true); + apply_vertical_interpolation(*m_lin_interp_mid_scalar,f_src,f_tgt,m_src_pmid,0); } } else { - // There is nothing to do, this field cannot be vertically interpolated, - // so just copy it over. - f_tgt.deep_copy(f_src); + if (type.packed) { + apply_vertical_interpolation(*m_lin_interp_int_packed,f_src,f_tgt,m_src_pint,0); + } else { + apply_vertical_interpolation(*m_lin_interp_int_scalar,f_src,f_tgt,m_src_pint,0); + } } } } template void VerticalRemapper:: -apply_vertical_interpolation(const Field& f_src, const Field& f_tgt, const bool mask_interp) const +setup_lin_interp (const ekat::LinInterp& lin_interp, + const Field& p_src) const { - using Pack = ekat::Pack; - using namespace ShortFieldTagsNames; - using namespace scream::vinterp; - const auto& layout = f_src.get_header().get_identifier().get_layout(); - const auto rank = f_src.rank(); - const auto src_tag = layout.tags().back(); - const auto src_num_levs = layout.dims().back(); - // ARG mask_interp checks if this is a vertical interpolation of the mask array that tracks masked 0.0 or not 1.0 - Real mask_val = mask_interp ? 0.0 : m_mask_val; - - Field src_lev_f; - if (src_tag == ILEV) { - src_lev_f = m_src_int; - } else { - src_lev_f = m_src_mid; - } - auto src_lev = src_lev_f.get_view(); - auto remap_pres_view = m_remap_pres.get_view(); - switch(rank) { + using LI_t = ekat::LinInterp; + using ESU = ekat::ExeSpaceUtils; + using PackT = ekat::Pack; + auto p_src_v = p_src.get_view(); + auto p_tgt_v = m_tgt_pressure.get_view(); + + auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { + const int icol = team.league_rank(); + lin_interp.setup(team,ekat::subview(p_src_v,icol), + p_tgt_v); + }; + + const int ncols = m_src_grid->get_num_local_dofs(); + const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); + const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); + auto policy = ESU::get_default_team_policy(ncols,npacks_tgt); + Kokkos::parallel_for("VerticalRemapper::interp_setup",policy,lambda); + Kokkos::fence(); +} + +template +void VerticalRemapper:: +apply_vertical_interpolation(const ekat::LinInterp& lin_interp, + const Field& f_src, const Field& f_tgt, + const Field& p_src, + const Real mask_val) const +{ + // Note: if Packsize==1, we grab packs of size 1, which are for sure + // compatible with the allocation + using LI_t = ekat::LinInterp; + using PackT = ekat::Pack; + using ESU = ekat::ExeSpaceUtils; + + auto p_src_v = p_src.get_view(); + auto x_tgt = m_tgt_pressure.get_view(); + const int ncols = m_src_grid->get_num_local_dofs(); + const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); + const int nlevs_src = m_src_grid->get_num_vertical_levels(); + const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); + + const int last_src_pack_idx = ekat::PackInfo::last_pack_idx(nlevs_src); + const int last_src_pack_end = ekat::PackInfo::last_vec_end(nlevs_src); + + switch(f_src.rank()) { case 2: { - auto src_view = f_src.get_view(); - auto tgt_view = f_tgt.get_view< Pack**>(); - perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); + auto f_src_v = f_src.get_view(); + auto f_tgt_v = f_tgt.get_view< PackT**>(); + auto policy = ESU::get_default_team_policy(ncols,npacks_tgt); + auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) { + + // Interpolate + const int icol = team.league_rank(); + auto x_src = ekat::subview(p_src_v,icol); + auto y_src = ekat::subview(f_src_v,icol); + auto y_tgt = ekat::subview(f_tgt_v,icol); + lin_interp.lin_interp(team,x_src,x_tgt,y_src,y_tgt,icol); + team.team_barrier(); + + // If x_tgt is extrapolated, set to mask_val + auto x_min = x_src[0][0]; + auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; + auto set_mask = [&](const int ipack) { + auto oob = x_tgt[ipack]x_max; + if (oob.any()) { + y_tgt[ipack].set(oob,mask_val); + } + }; + Kokkos::parallel_for (Kokkos::TeamThreadRange(team,npacks_tgt), set_mask); + }; + Kokkos::parallel_for("VerticalRemapper::apply_vertical_interpolation",policy,lambda); break; } case 3: { - auto src_view = f_src.get_view(); - auto tgt_view = f_tgt.get_view< Pack***>(); - perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); + auto f_src_v = f_src.get_view(); + auto f_tgt_v = f_tgt.get_view< PackT***>(); + const auto& layout = f_src.get_header().get_identifier().get_layout(); + const int ncomps = layout.get_vector_dim(); + auto policy = ESU::get_default_team_policy(ncols*ncomps,npacks_tgt); + + auto lambda = KOKKOS_LAMBDA(typename LI_t::MemberType const& team) + { + // Interpolate + const int icol = team.league_rank() / ncomps; + const int icmp = team.league_rank() % ncomps; + auto x_src = ekat::subview(p_src_v,icol); + auto y_src = ekat::subview(f_src_v,icol,icmp); + auto y_tgt = ekat::subview(f_tgt_v,icol,icmp); + lin_interp.lin_interp(team,x_src,x_tgt,y_src,y_tgt,icol); + team.team_barrier(); + + // If x_tgt is extrapolated, set to mask_val + auto x_min = x_src[0][0]; + auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; + auto set_mask = [&](const int ipack) { + auto oob = x_tgt[ipack]x_max; + if (oob.any()) { + y_tgt[ipack].set(oob,mask_val); + } + }; + Kokkos::parallel_for (Kokkos::TeamThreadRange(team,npacks_tgt), set_mask); + }; + Kokkos::parallel_for("VerticalRemapper::apply_vertical_interpolation",policy,lambda); break; } default: - EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by VerticalRemapper.\n"); + EKAT_ERROR_MSG ( + "[VerticalRemapper::apply_vertical_interpolation] Error! Unsupported field rank.\n" + " - src field name: " + f_src.name() + "\n" + " - src field rank: " + std::to_string(f_src.rank()) + "\n"); } } diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp index f9950fad0f3a..214fef8e2b37 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp @@ -1,10 +1,9 @@ #ifndef EAMXX_VERTICAL_REMAPPER_HPP #define EAMXX_VERTICAL_REMAPPER_HPP -#include "share/field/field_tag.hpp" #include "share/grid/remap/abstract_remapper.hpp" -#include "ekat/ekat_pack.hpp" +#include namespace scream { @@ -63,8 +62,6 @@ class VerticalRemapper : public AbstractRemapper FieldLayout create_layout (const FieldLayout& fl_in, const grid_ptr_type& grid_out) const; - void register_vertical_source_field(const Field& src); - const identifier_type& do_get_src_field_id (const int ifield) const override { return m_src_fields[ifield].get_header().get_identifier(); } @@ -99,16 +96,21 @@ class VerticalRemapper : public AbstractRemapper public: #endif template - void apply_vertical_interpolation (const Field& f_src, const Field& f_tgt, const bool mask_interp=false) const; -protected: + void apply_vertical_interpolation (const ekat::LinInterp& lin_interp, + const Field& f_src, const Field& f_tgt, + const Field& p_src, + const Real mask_value) const; - using KT = KokkosTypes; - using gid_type = AbstractGrid::gid_type; template - using RPack = ekat::Pack; + void setup_lin_interp (const ekat::LinInterp& lin_interp, + const Field& p_src) const; +protected: - using mPack = RPack; + void set_source_pressure_fields(const Field& pmid, const Field& pint); + void create_lin_interp (); + + using KT = KokkosTypes; template using view_1d = typename KT::template view_1d; @@ -124,13 +126,25 @@ class VerticalRemapper : public AbstractRemapper std::vector m_src_masks; // Vertical profile fields, both for source and target - int m_num_remap_levs; Real m_mask_val; - Field m_remap_pres; - Field m_src_mid; // Src vertical profile for LEV layouts - Field m_src_int; // Src vertical profile for ILEV layouts - bool m_mid_set = false; - bool m_int_set = false; + Field m_tgt_pressure; + Field m_src_pmid; // Src vertical profile for LEV layouts + Field m_src_pint; // Src vertical profile for ILEV layouts + + // We need to remap mid/int fields separately, and we want to use packs if possible, + // so we need to divide input fields into 4 separate categories + + // Map field id to whether it's packed/scalar and midpoint/interface + struct FType { + bool packed = false; + bool midpoints = true; + }; + std::map m_field2type; + + std::shared_ptr> m_lin_interp_mid_packed; + std::shared_ptr> m_lin_interp_int_packed; + std::shared_ptr> m_lin_interp_mid_scalar; + std::shared_ptr> m_lin_interp_int_scalar; }; } // namespace scream diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 061e7022649b..bbe3306a415e 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -1,7 +1,6 @@ #include "share/grid/point_grid.hpp" #include "share/io/scorpio_input.hpp" #include "share/iop/intensive_observation_period.hpp" -#include "share/util/scream_vertical_interpolation.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_lin_interp.hpp" diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index 4aab3ef95da6..3b4b60cb283b 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -2,9 +2,6 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) - # Test vertical interpolation - CreateUnitTest(vertical_interp "vertical_interp_tests.cpp") - # Test utils CreateUnitTest(utils "utils_tests.cpp") diff --git a/components/eamxx/src/share/tests/vertical_interp_tests.cpp b/components/eamxx/src/share/tests/vertical_interp_tests.cpp deleted file mode 100644 index c3e5abbfa0b6..000000000000 --- a/components/eamxx/src/share/tests/vertical_interp_tests.cpp +++ /dev/null @@ -1,357 +0,0 @@ -#include - -#include "share/util/scream_vertical_interpolation.hpp" - -using namespace scream; -using namespace vinterp; - -template -using SmallPack = Pack; -using Spack = SmallPack; - -using Smask = ekat::Mask; - -template -void run(){ - //This function first tests cases where the output levels are the same - //as the input levels to make sure you get the same output as input - //In this case only two cases are investigated: - //1) n_layers_src = n_layers_tgt = 2*P - //2) n_layers_src = n_layers_tgt = 2*P+1 - //where P is the pack size - //Then the function tests 4 different scenarios where the - //the output levels are different from the input levels: - //1) n_layers_src= 2*P, n_layers_tgt= 2*P - //2) n_layers_src= 2*P+1, n_layers_tgt= 2*P - //3) n_layers_src= 2*P, n_layers_tgt= 2*P-1 - //4) n_layers_src= 2*P+1, n_layers_tgt= 2*P-1 - //For each scenario target levels are at the midpoint and so should be the average - //of the source layers. - // TODO: ASD - Add a test for when the source pressure is a single column and target is multiple columns - - int n_layers_src[4] = {2*P,2*P+1,2*P,2*P+1}; - int n_layers_tgt[4] = {2*P,2*P,2*P-1,2*P-1}; - - printf (" -- Testing vertical interpolation with Pack size %d --\n",P); - - for (double perturb : {0, 1}){ - printf (" -> Target pressure levels: p_tgt = p_src + %f\n",perturb); - for (int i=0; i<4; i++){ - if (perturb == 0){ - //The 3rd and 4th test are redundant in the case of the - //same target levels as source levels - if(i > 1){ break;} - //In this case the target levels are the same as the source levels - n_layers_tgt[i] = n_layers_src[i]; - - printf (" -> Testing %d source layers, %d target layers\n", - n_layers_src[i],n_layers_tgt[i]); - } - else{ - printf (" -> Testing %d source layers, %d target layers\n", - n_layers_src[i],n_layers_tgt[i]); - } - auto npacks_src = ekat::PackInfo

::num_packs(n_layers_src[i]); - auto npacks_tgt = ekat::PackInfo

::num_packs(n_layers_tgt[i]); - auto p_tgt = view_1d>("",npacks_tgt); - auto p_tgt_h = Kokkos::create_mirror_view(p_tgt); - auto p_tgt_h_s = ekat::scalarize(p_tgt_h); - auto tmp_src = view_Nd,2>("",2,npacks_src); - auto tmp_src_h = Kokkos::create_mirror_view(tmp_src); - auto tmp_src_h_s = ekat::scalarize(tmp_src_h); - auto p_src = view_Nd,2>("",2,npacks_src); - auto p_src_h = Kokkos::create_mirror_view(p_src); - auto p_src_h_s = ekat::scalarize(p_src_h); - auto out = view_Nd,2>("",2,npacks_tgt); - auto out_h = Kokkos::create_mirror_view(out); - auto out_h_s = ekat::scalarize(out_h); - auto mask = view_Nd,2>("",2,npacks_tgt); - auto mask_h = Kokkos::create_mirror_view(mask); - - //Set target levels - for (int lev=0; lev<(n_layers_tgt[i]); lev++){ - p_tgt_h_s(lev) = 2*lev+perturb; - } - //Set source levels and source input (tmp_src) - for (int col=0; col<2; col++){ - for (int lev=0; lev(p_src, - p_tgt, - tmp_src, - out, - mask, - n_layers_src[i], - n_layers_tgt[i]); - - Kokkos::deep_copy(out_h,out); - Kokkos::deep_copy(mask_h,mask); - - //Check that output of interpolation is as expected - for(int col=0; col<2; col++){ - for(int lev=0; lev<(n_layers_tgt[i]-1); lev++){ - const int ipack = lev / P; - const int jpack = lev % P; - REQUIRE(mask_h(col,ipack)[jpack] == false); - - if (perturb == 0){ - REQUIRE(out_h_s(col,lev) == tmp_src_h_s(col,lev)); - } - else{ - auto avg_val = (tmp_src_h_s(col,lev+1)+tmp_src_h_s(col,lev))/2.; - REQUIRE(out_h_s(col,lev) == avg_val); - } - }//end of looping over levels - }//end of looping over columns - - auto out_1d_test = view_Nd,2>("",2,npacks_tgt); - auto out_1d_test_h = Kokkos::create_mirror_view(out_1d_test); - auto out_1d_test_h_s = ekat::scalarize(out_1d_test_h); - auto mask_1d_test = view_Nd,2>("",2,npacks_tgt); - auto mask_1d_test_h = Kokkos::create_mirror_view(mask_1d_test); - - //Take subview and run through the 1d interpolator function - //and make sure get same thing back - ekat::LinInterp vert_interp(2,n_layers_src[i],n_layers_tgt[i]); - const int num_vert_packs = p_tgt.extent(0); - const auto policy = ESU::get_default_team_policy(2, num_vert_packs); - auto loc_layers_src = n_layers_src[i]; - auto loc_layers_tgt = n_layers_tgt[i]; - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - const auto x1=ekat::subview(p_src, icol); - const auto in=ekat::subview(tmp_src, icol); - const auto out_1d=ekat::subview(out_1d_test, icol); - const auto msk=ekat::subview(mask_1d_test, icol); - apply_interpolation_impl_1d(x1, - p_tgt, - in, - out_1d, - msk, - loc_layers_src, - loc_layers_tgt, - icol, - masked_val, - team, - vert_interp); - }); - Kokkos::fence(); - - Kokkos::deep_copy(out_1d_test_h,out_1d_test); - Kokkos::deep_copy(mask_1d_test_h,mask_1d_test); - - //Check that 1d interpolator output is consistent with what is expected - for(int col=0; col<2; col++){ - for(int lev=0; lev<(n_layers_tgt[i]-1); lev++){ - const int ipack = lev / P; - const int jpack = lev % P; - REQUIRE(mask_1d_test_h(col,ipack)[jpack] == false); - - if (perturb == 0){ - REQUIRE(out_1d_test_h_s(col,lev) == tmp_src_h_s(col,lev)); - } - else{ - auto avg_val = (tmp_src_h_s(col,lev+1)+tmp_src_h_s(col,lev))/2.; - REQUIRE(out_1d_test_h_s(col,lev) == avg_val); - } - }//end of looping over levels - }//end of looping over columns - }//end of looping over pack size tests - }//end of looping over perturbed or not -} - -TEST_CASE("main_vertical_interpolation_test"){ - //This test first checks that you get the same answer if source and target levels - //are the same - //It then tests that you get the expected answer for nlevs % packsize==0 and nlevs%packsize!=0 - //and for different pack sizes. - run<1> (); - if (SCREAM_PACK_SIZE>1) { - run (); - } -} - -template -void check_mask(const view_Nd_host,2>& mask, int col, int lev){ - const int ipack = lev / P; - const int jpack = lev % P; - if ((col == 0 && lev == 16) || (col == 1 && lev == 0)){ - REQUIRE(mask(col,ipack)[jpack] == true); - } - else{ - REQUIRE(mask(col,ipack)[jpack] == false); - } -} - -TEST_CASE("testing_masking"){ - printf (" -- Testing Masking --\n"); - //This test performs 3 tests: - //1) That the interpolation is working properly using 2d views, - // including the masking of out-of-bounds values - //2) It checks the interpolation is working properly with - // a user defined masking value - //3) It checks that the interpolation is working properly when - // using the 1d interpolation function - const int n_layers_src = 9; - const int n_layers_tgt = 17; - const int P = SCREAM_PACK_SIZE; - - auto npacks_src = ekat::PackInfo

::num_packs(n_layers_src); - auto npacks_tgt = ekat::PackInfo

::num_packs(n_layers_tgt); - auto p_tgt = view_1d>("",npacks_tgt); - auto p_tgt_h = Kokkos::create_mirror_view(p_tgt); - auto p_tgt_h_s = ekat::scalarize(p_tgt_h); - auto tmp_src = view_Nd,2>("",2,npacks_src); - auto tmp_src_h = Kokkos::create_mirror_view(tmp_src); - auto tmp_src_h_s = ekat::scalarize(tmp_src_h); - auto p_src = view_Nd,2>("",2,npacks_src); - auto p_src_h = Kokkos::create_mirror_view(p_src); - auto p_src_h_s = ekat::scalarize(p_src_h); - auto out = view_Nd,2>("",2,npacks_tgt); - auto out_h = Kokkos::create_mirror_view(out); - auto mask = view_Nd,2>("",2,npacks_tgt); - auto mask_h = Kokkos::create_mirror_view(mask); - - //Fist test to see if interpolate properly using 2d views - //Also test that when out-of-bounds returns masked values - - //Set target levels from 25-105 - for (int i=0; i<17; i++){ - p_tgt_h_s(i) = 25.0 + i*5.0; - } - - //Set source levels: - //For column 1 from 20-100 - //For column 2 from 30-110 - //Set input variable, in this case - //set tmp_src from 200-280 by 10, with final level at 240 so - //can test interpolation with increasing/decreasing values - for (int i=0; i<9; i++){ - p_src_h_s(0,i) = 20.0 + i*10.0; - p_src_h_s(1,i) = 30.0 + i*10.0; - if (i<8){ - tmp_src_h_s(0,i) = 200.+i*10.; - tmp_src_h_s(1,i) = 200.+i*10.; - } - else{ - tmp_src_h_s(0,i) = 240.; - tmp_src_h_s(1,i) = 240.; - } - } - - Kokkos::deep_copy(p_src,p_src_h); - Kokkos::deep_copy(p_tgt,p_tgt_h); - Kokkos::deep_copy(tmp_src,tmp_src_h); - - perform_vertical_interpolation(p_src, - p_tgt, - tmp_src, - out, - mask, - n_layers_src, - n_layers_tgt); - - Kokkos::deep_copy(out_h,out); - Kokkos::deep_copy(mask_h,mask); - - Real correct_val[2][17]; - for (int i=0; i<15; i++){ - correct_val[0][i] = 205+i*5.; - correct_val[1][i] = 195+i*5.; - } - - correct_val[0][14] = 255.; - correct_val[0][15] = 240.; - correct_val[0][16] = masked_val; - correct_val[1][0] = masked_val; - correct_val[1][15] = 270.; - correct_val[1][16] = 255.; - - auto out_h_s = ekat::scalarize(out_h); - for(int col=0; col<2; col++){ - for(int lev=0; lev<17; lev++){ - REQUIRE(out_h_s(col,lev) == correct_val[col][lev]); - check_mask

(mask_h,col,lev); - } - } - - //Test to see if get same answer when call 1D interpolation function - //instead of 2D interpolation function - auto out_1d_test = view_Nd,2>("",2,npacks_tgt); - auto out_1d_test_h = Kokkos::create_mirror_view(out_1d_test); - auto out_1d_test_h_s = ekat::scalarize(out_1d_test_h); - auto mask_1d_test = view_Nd,2>("",2,npacks_tgt); - auto mask_1d_test_h = Kokkos::create_mirror_view(mask_1d_test); - - ekat::LinInterp vert_interp(2,n_layers_src,n_layers_tgt); - const int num_vert_packs = p_tgt.extent(0); - const auto policy = ESU::get_default_team_policy(2, num_vert_packs); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - const auto x1=ekat::subview(p_src, icol); - const auto in=ekat::subview(tmp_src, icol); - const auto out_1d=ekat::subview(out_1d_test, icol); - const auto msk=ekat::subview(mask_1d_test, icol); - apply_interpolation_impl_1d(x1, - p_tgt, - in, - out_1d, - msk, - n_layers_src, - n_layers_tgt, - icol, - masked_val, - team, - vert_interp); - }); - Kokkos::fence(); - - Kokkos::deep_copy(out_1d_test_h,out_1d_test); - Kokkos::deep_copy(mask_1d_test_h,mask_1d_test); - - for(int col=0; col<2; col++){ - for(int lev=0; lev<17; lev++){ - REQUIRE(out_1d_test_h_s(col,lev) == correct_val[col][lev]); - check_mask

(mask_h,col,lev); - } - } - - //Check to see if choose different masked value than default that it returns as expected - auto out_usr_msk = view_Nd,2>("",2,npacks_tgt); - auto out_usr_msk_h = Kokkos::create_mirror_view(out_usr_msk); - auto out_usr_msk_h_s = ekat::scalarize(out_usr_msk_h); - auto mask_usr_msk = view_Nd,2>("",2,npacks_tgt); - auto mask_usr_msk_h = Kokkos::create_mirror_view(mask_usr_msk); - Real mod_mask_val = -999.; - perform_vertical_interpolation(p_src, - p_tgt, - tmp_src, - out_usr_msk, - mask_usr_msk, - n_layers_src, - n_layers_tgt, - mod_mask_val); - correct_val[0][16] = -999.; - correct_val[1][0] = -999.; - - Kokkos::deep_copy(out_usr_msk_h,out_usr_msk); - Kokkos::deep_copy(mask_usr_msk_h,mask_usr_msk); - - for(int col=0; col<2; col++){ - for(int lev=0; lev<17; lev++){ - REQUIRE(out_usr_msk_h_s(col,lev) == correct_val[col][lev]); - check_mask

(mask_h,col,lev); - } - } - -} - diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index f7bd62b79989..34351c65db8f 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -15,32 +15,6 @@ cmvc (const ViewT& v) { return vh; } -class VerticalRemapperTester : public VerticalRemapper { -public: - VerticalRemapperTester (const grid_ptr_type& src_grid, - const std::string& map_file, - const Field& lev_prof, - const Field& ilev_prof, - const Real mask_val) - : VerticalRemapper(src_grid, map_file, lev_prof, ilev_prof, mask_val) - { - // Nothing to do - } -}; - -template -bool contains (const ViewT& v, const typename ViewT::traits::value_type& entry) { - const auto vh = cmvc (v); - const auto beg = vh.data(); - const auto end = vh.data() + vh.size(); - for (auto it=beg; it!=end; ++it) { - if (*it == entry) { - return true; - } - } - return false; -} - void print (const std::string& msg, const ekat::Comm& comm) { if (comm.am_i_root()) { printf("%s",msg.c_str()); @@ -180,7 +154,7 @@ TEST_CASE ("vertical_remap") { } pmid_src.sync_to_dev(); pint_src.sync_to_dev(); - auto remap = std::make_shared(src_grid,filename,pmid_src,pint_src,mask_val); + auto remap = std::make_shared(src_grid,filename,pmid_src,pint_src,mask_val); print (" -> creating grid and remapper ... done!\n",comm); // -------------------------------------- // diff --git a/components/eamxx/src/share/util/scream_vertical_interpolation.hpp b/components/eamxx/src/share/util/scream_vertical_interpolation.hpp deleted file mode 100644 index 68b9997c8595..000000000000 --- a/components/eamxx/src/share/util/scream_vertical_interpolation.hpp +++ /dev/null @@ -1,321 +0,0 @@ -#ifndef SCREAM_VERTICAL_INTERPOLATION_HPP -#define SCREAM_VERTICAL_INTERPOLATION_HPP - -#include "share/scream_types.hpp" - -#include "ekat/util/ekat_lin_interp.hpp" -#include "ekat/ekat_pack_utils.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream { -namespace vinterp { - -/* This utility can perform vertical interpolation for a particular variable - * (for instance a field) from src levels to target levels - * The relevant function is perform_vertical_interpolation(). - * A user can decide to use the function without a masked value - * in which case the default masked value (masked_val) will be used. - * Also a user can decide to provide a view_Nd mask which will be filled - * based on whether each point is masked or not. - * Masking occurs when a value is out-of-bounds (i.e. requires an - * extrapolation). - * The main function assumes that the user is providing as input: - * a 2D view for the source vertical levels to interpolate from - * a 1D view for the target vertical levels to interpolate onto - * a set of Nd views for the input data, output data and an optional mask - * where Nd is a N-dimensional view of dimension 2 or 3. - * There is a function, perform_vertical_interpolation_impl_1d which - * is provided for 1d views (or lambdas). However, in this - * case the user must provide the team and ekat::LinInterp as input as well. - */ - -// ------- Types -------- - -template -using Pack = ekat::Pack; - -template -using Mask = ekat::Mask

; - -using KT = KokkosTypes; -using ExeSpace = typename KT::ExeSpace; -using ESU = ekat::ExeSpaceUtils; - -using MemberType = typename KT::MemberType; - -template -using LIV = ekat::LinInterp; - -template -using view_1d = typename KT::template view_1d; - -template -using view_2d = typename KT::template view_2d; - -template -using view_3d = typename KT::template view_3d; - -template -using view_Nd = typename KT::template view_ND; -template -using view_Nd_host = typename KT::template view_ND::HostMirror; - -constexpr Real masked_val = -std::numeric_limits::max(); - -template -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -template -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val = masked_val); - -/* ---------------------------------------------------------------------- - * Main interpolation routine that applies vertical interpolation to a - * single vertical slice of data. - * ---------------------------------------------------------------------- */ -template -KOKKOS_FUNCTION -void apply_interpolation_impl_1d( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_1d>& input, - const view_1d< Pack>& output, - const view_1d< Mask

>& mask, - const int nlevs_src, - const int nlevs_tgt, - const int icol, - const T msk_val, - const MemberType& team, - const LIV& vert_interp); - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 2-D view and x_tgt is a 2-D view - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask); - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 2-D view and x_tgt is a single 1-D vertical profile - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask); - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 1-D view and x_tgt is a 2-D view - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask); - -/* ---------------------------------------------------------------------- - * Versions where x_src is a single 1-D view and x_tgt is a single 1-D vertical profile - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask); - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask); - -// Helper function to allocate memory for an Nd mask on the fly. -template -view_Nd,N> allocate_mask(const std::vector& extents); - -} // namespace vinterp -} // namespace scream - -#include "scream_vertical_interpolation_impl.hpp" - -#endif // SCREAM_VERTICAL_INTERPOLATION_HPP diff --git a/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp b/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp deleted file mode 100644 index 7b7007482b48..000000000000 --- a/components/eamxx/src/share/util/scream_vertical_interpolation_impl.hpp +++ /dev/null @@ -1,627 +0,0 @@ -#include "share/util/scream_vertical_interpolation.hpp" - -namespace scream { -namespace vinterp { - -template -view_Nd,N> allocate_mask(const std::vector& extents) -{ - switch(extents.size()) { - case 1: - return view_Nd,N>("",extents[0]); - case 2: - return view_Nd,N>("",extents[0],extents[1]); - case 3: - return view_Nd,N>("",extents[0],extents[1],extents[2]); - default: - EKAT_ERROR_MSG("vertical_remap::allocate_mask only supports a rank of 2 or 3, received rank = " + std::to_string(N)); - } -} - -template -KOKKOS_FUNCTION -void apply_interpolation_impl_1d( - const view_1d>& x_src_col, - const view_1d>& x_tgt_col, - const view_1d>& input_col, - const view_1d< Pack>& output_col, - const view_1d< Mask

>& mask_col, - const int nlevs_src, - const int nlevs_tgt, - const int icol, - const T msk_val, - const MemberType& team, - const LIV& vert_interp) -{ - // Recast source views to support different packsizes - using PackInfo = ekat::PackInfo

; - const int num_src_packs = PackInfo::num_packs(nlevs_src); - const int num_tgt_packs = PackInfo::num_packs(nlevs_tgt); - - auto x_src = Kokkos::subview(x_src_col,Kokkos::pair(0,num_src_packs)); - auto input = Kokkos::subview(input_col,Kokkos::pair(0,num_src_packs)); - - auto x_tgt = Kokkos::subview(x_tgt_col, Kokkos::pair(0,num_tgt_packs)); - auto output = Kokkos::subview(output_col,Kokkos::pair(0,num_tgt_packs)); - auto mask = Kokkos::subview(mask_col, Kokkos::pair(0,num_tgt_packs)); - - // The input/output data and x_src/x_tgt data should match in the appropriate size, respectively. - EKAT_KERNEL_REQUIRE_MSG(x_tgt.size() == output.size(), "Error! vertical_interpolation::apply_interpolation_imple_1d - target pressure level size does not match the size of the target data output."); - EKAT_KERNEL_REQUIRE_MSG(x_src.size() == input.size() , "Error! vertical_interpolation::apply_interpolation_imple_1d - source pressure level size does not match the size of the source data input."); - - //Setup linear interpolation - vert_interp.setup(team, x_src, x_tgt); - //Run linear interpolation - vert_interp.lin_interp(team, x_src, x_tgt, input, output, icol); - const auto x_src_s = ekat::scalarize(x_src); - const auto x_tgt_s = ekat::scalarize(x_tgt); - const auto range = Kokkos::TeamVectorRange(team, x_tgt.extent(0)); - //Mask out values above (below) maximum (minimum) source grid - Kokkos::parallel_for(range, [&] (const Int & k) { - const auto above_max = x_tgt[k] > x_src_s[nlevs_src-1]; - const auto below_min = x_tgt[k] < x_src_s[0]; - const auto combined_m = above_max || below_min; - mask(k) = combined_m; - output(k).set(combined_m,msk_val); - }); - team.team_barrier(); -} - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 2-D view and x_tgt is a 2-D view - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt) -{ - auto rank = N; - EKAT_REQUIRE_MSG (rank>1 &&rank<=3,"Error::scream_vertical_interpolation, passed view of rank (" + std::to_string(rank) +"), only support ranks 2 or 3\n"); - - // The input data and x_src data should match in the appropriate size - EKAT_REQUIRE(x_src.extent_int(0) == input.extent_int(0)); - EKAT_REQUIRE(x_src.extent_int(1) == input.extent_int(input.rank-1)); - // The output data and x_tgt data should match in the appropriate size - EKAT_REQUIRE(x_tgt.extent_int(0) == output.extent_int(0)); - EKAT_REQUIRE(x_tgt.extent_int(1) == output.extent_int(input.rank-1)); - - // The output data and the input data should match in all sizes except the last one - for (int ii=0;ii -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = x_src.extent(0); - for (int ii=1; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = x_src.extent(0); - for (int ii=1; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - - std::vector extents; - for (int ii=0;ii(extents); - - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank(); - const auto x1 = ekat::subview(x_src, icol); - const auto xt = ekat::subview(x_tgt, icol); - const auto in = ekat::subview(input, icol); - const auto out = ekat::subview(output, icol); - const auto mask = ekat::subview(mask_out, icol); - - apply_interpolation_impl_1d(x1,xt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_2d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int num_vars = input.extent_int(1); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0*num_vars, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank() / num_vars; - const int ivar = team.league_rank() % num_vars; - const auto x1 = ekat::subview(x_src, icol); - const auto xt = ekat::subview(x_tgt, icol); - const auto in = ekat::subview(input, icol, ivar); - const auto out = ekat::subview(output, icol, ivar); - const auto mask = ekat::subview(mask_out, icol, ivar); - - apply_interpolation_impl_1d(x1,xt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 2-D view and x_tgt is a single 1-D vertical profile - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt) -{ - const int rank = input.rank; - EKAT_REQUIRE_MSG (rank>1 &&rank<=3,"Error::scream_vertical_interpolation, passed view of rank (" + std::to_string(rank) +"), only support ranks 2 or 3\n"); - - // The input data and x_src data should match in the appropriate size - EKAT_REQUIRE(x_src.extent_int(0) == input.extent_int(0)); - - - // The output and input data should match in rank - EKAT_REQUIRE(static_cast(output.rank)==rank); - // The output data and the input data should match in all sizes except the last one - for (int ii=0;ii -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = x_src.extent(0); - for (int ii=1; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void perform_vertical_interpolation( - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = x_src.extent(0); - for (int ii=1; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - - std::vector extents; - for (int ii=0;ii(extents); - - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank(); - const auto x1 = ekat::subview(x_src, icol); - const auto in = ekat::subview(input, icol); - const auto out = ekat::subview(output, icol); - const auto mask = ekat::subview(mask_out, icol); - - apply_interpolation_impl_1d(x1,x_tgt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_2d>& x_src, - const view_1d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int num_vars = input.extent_int(1); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0*num_vars, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank() / num_vars; - const int ivar = team.league_rank() % num_vars; - const auto x1 = ekat::subview(x_src, icol); - const auto in = ekat::subview(input, icol, ivar); - const auto out = ekat::subview(output, icol, ivar); - const auto mask = ekat::subview(mask_out, icol, ivar); - - apply_interpolation_impl_1d(x1,x_tgt,in,out,mask,num_levs_src,num_levs_tgt,team.league_rank(),mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 1-D view and x_tgt is a 2-D view - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt) -{ - auto rank = N; - EKAT_REQUIRE_MSG (rank>1 &&rank<=3,"Error::scream_vertical_interpolation, passed view of rank (" + std::to_string(rank) +"), only support ranks 2 or 3\n"); - - // The input data and x_src data should match in the appropriate size - EKAT_REQUIRE(x_src.extent_int(0) == input.extent_int(input.rank-1)); - // The output data and x_tgt data should match in the appropriate size - EKAT_REQUIRE(x_tgt.extent_int(0) == output.extent_int(0)); - EKAT_REQUIRE(x_tgt.extent_int(1) == output.extent_int(input.rank-1)); - - // The output data and the input data should match in all sizes except the last one - for (int ii=0;ii -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = 1; - for (int ii=0; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = 1; - for (int ii=0; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - - std::vector extents; - for (int ii=0;ii(extents); - - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank(); - const auto x1 = x_src; - const auto xt = ekat::subview(x_tgt, icol); - const auto in = ekat::subview(input, icol); - const auto out = ekat::subview(output, icol); - const auto mask = ekat::subview(mask_out, icol); - - apply_interpolation_impl_1d(x1,xt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_2d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int num_vars = input.extent_int(1); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0*num_vars, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank() / num_vars; - const int ivar = team.league_rank() % num_vars; - const auto x1 = x_src; - const auto xt = ekat::subview(x_tgt, icol); - const auto in = ekat::subview(input, icol, ivar); - const auto out = ekat::subview(output, icol, ivar); - const auto mask = ekat::subview(mask_out, icol, ivar); - - apply_interpolation_impl_1d(x1,xt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -/* ---------------------------------------------------------------------- - * Versions where x_src is a 1-D view and x_tgt is a single 1-D vertical profile - * ---------------------------------------------------------------------- */ -template -void perform_checks( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt) -{ - const int rank = input.rank; - EKAT_REQUIRE_MSG (rank>1 &&rank<=3,"Error::scream_vertical_interpolation, passed view of rank (" + std::to_string(rank) +"), only support ranks 2 or 3\n"); - - // The output and input data should match in rank - EKAT_REQUIRE(static_cast(output.rank)==rank); - // The output data and the input data should match in all sizes except the last one - for (int ii=0;ii -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const view_Nd< Mask

,N>& mask, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = 1; - for (int ii=0; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void perform_vertical_interpolation( - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_Nd,N>& input, - const view_Nd< Pack,N>& output, - const int nlevs_src, - const int nlevs_tgt, - const Real msk_val) -{ - int ndofs = 1; - for (int ii=1; ii(x_src, x_tgt, input, output, nlevs_src, nlevs_tgt); - - std::vector extents; - for (int ii=0;ii(extents); - - LIV vert_interp(ndofs,nlevs_src,nlevs_tgt); - apply_interpolation(nlevs_src, nlevs_tgt, msk_val, vert_interp, x_src, x_tgt, input, output, mask); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_2d>& input, - const view_2d< Pack>& output, - const view_2d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank(); - const auto in = ekat::subview(input, icol); - const auto out = ekat::subview(output, icol); - const auto mask = ekat::subview(mask_out, icol); - - apply_interpolation_impl_1d(x_src,x_tgt,in,out,mask,num_levs_src,num_levs_tgt,icol,mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -template -void apply_interpolation( - const int num_levs_src, - const int num_levs_tgt, - const T mask_val, - const LIV& vert_interp, - const view_1d>& x_src, - const view_1d>& x_tgt, - const view_3d>& input, - const view_3d< Pack>& output, - const view_3d< Mask

>& mask_out) -{ - const int d_0 = input.extent_int(0); - const int num_vars = input.extent_int(1); - const int npacks = output.extent_int(output.rank-1); - const auto policy = ESU::get_default_team_policy(d_0*num_vars, npacks); - Kokkos::parallel_for("scream_vert_interp_setup_loop", policy, - KOKKOS_LAMBDA(MemberType const& team) { - - const int icol = team.league_rank() / num_vars; - const int ivar = team.league_rank() % num_vars; - const auto in = ekat::subview(input, icol, ivar); - const auto out = ekat::subview(output, icol, ivar); - const auto mask = ekat::subview(mask_out, icol, ivar); - - apply_interpolation_impl_1d(x_src,x_tgt,in,out,mask,num_levs_src,num_levs_tgt,team.league_rank(),mask_val,team,vert_interp); - }); - Kokkos::fence(); -} - -} // namespace vinterp -} // namespace scream - From 5d0cee7d869c68a8a36784c79300d6af679f86c9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 13 May 2024 16:31:23 -0600 Subject: [PATCH 449/476] EAMxx: fixes to coarsening/vertical remappers, related to masks --- .../share/grid/remap/coarsening_remapper.cpp | 4 +- .../share/grid/remap/vertical_remapper.cpp | 47 ++++++++++++------- .../src/share/io/tests/io_remap_test.cpp | 6 +-- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index 2ae6f3110e7c..f915cb14d292 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -177,7 +177,7 @@ void CoarseningRemapper::do_remap_fwd () const auto& mask = m_src_fields[mask_idx]; // If possible, dispatch kernel with SCREAM_PACK_SIZE - if (can_pack_field(f_src) and can_pack_field(f_ov)) { + if (can_pack_field(f_src) and can_pack_field(f_ov) and can_pack_field(mask)) { local_mat_vec(f_src,f_ov,mask); } else { local_mat_vec<1>(f_src,f_ov,mask); @@ -214,7 +214,7 @@ void CoarseningRemapper::do_remap_fwd () if (mask_idx>0) { // Then this field did use a mask const auto& mask = m_tgt_fields[mask_idx]; - if (can_pack_field(f_tgt)) { + if (can_pack_field(f_tgt) and can_pack_field(mask)) { rescale_masked_fields(f_tgt,mask); } else { rescale_masked_fields<1>(f_tgt,mask); diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index a4c474685733..5e714c48aa0d 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -235,8 +235,9 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) // at a specific COL,LEV src_layout.strip_dims({CMP}); - // Check if this mask has already been created + // I this mask has already been created, retrieve it, otherwise create it const auto mask_name = m_tgt_grid->name() + "_" + ekat::join(src_layout.names(),"_") + "_mask"; + Field tgt_mask; if (m_field2type.count(mask_name)==0) { auto nondim = ekat::units::Units::nondimensional(); // Create this src/tgt mask fields, and assign them to these src/tgt fields extra data @@ -244,28 +245,39 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) FieldIdentifier src_mask_fid (mask_name, src_layout, nondim, m_src_grid->name() ); FieldIdentifier tgt_mask_fid = create_tgt_fid(src_mask_fid); - Field src_mask (src_mask_fid); - Field tgt_mask (tgt_mask_fid); - + Field src_mask (src_mask_fid); src_mask.allocate_view(); + + tgt_mask = Field (tgt_mask_fid); tgt_mask.allocate_view(); - EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_data"), - "[VerticalRemapper::do_bind_field] Error! Target field already has mask data assigned.\n" - " - tgt field name: " + tgt.name() + "\n"); - EKAT_REQUIRE_MSG(not tgt.get_header().has_extra_data("mask_value"), - "[VerticalRemapper::do_bind_field] Error! Target field already has mask value assigned.\n" - " - tgt field name: " + tgt.name() + "\n"); + // Initialize the src mask values to 1.0 + src_mask.deep_copy(1.0); - f_tgt.get_header().set_extra_data("mask_data",tgt_mask); - f_tgt.get_header().set_extra_data("mask_value",m_mask_val); m_src_masks.push_back(src_mask); m_tgt_masks.push_back(tgt_mask); auto& mt = m_field2type[src_mask_fid.name()]; mt.packed = false; mt.midpoints = src_layout.has_tag(LEV); + } else { + for (size_t i=0; i("mask_value"); - f_tgt.get_header().set_extra_data("mask_data",src_mask_val); + f_tgt.get_header().set_extra_data("mask_value",src_mask_val); } } @@ -406,9 +418,6 @@ void VerticalRemapper::do_remap_fwd () auto& f_tgt = m_tgt_masks[i]; const auto& type = m_field2type.at(f_src.name()); - // Initialize the mask source values to 1.0 - f_src.deep_copy(1.0); - // Dispatch interpolation to the proper lin interp object if (type.midpoints) { if (type.packed) { @@ -466,9 +475,10 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, auto p_src_v = p_src.get_view(); auto x_tgt = m_tgt_pressure.get_view(); + const auto& f_src_l = f_src.get_header().get_identifier().get_layout(); const int ncols = m_src_grid->get_num_local_dofs(); const int nlevs_tgt = m_tgt_grid->get_num_vertical_levels(); - const int nlevs_src = m_src_grid->get_num_vertical_levels(); + const int nlevs_src = f_src_l.dims().back(); const int npacks_tgt = ekat::PackInfo::num_packs(nlevs_tgt); const int last_src_pack_idx = ekat::PackInfo::last_pack_idx(nlevs_src); @@ -494,7 +504,8 @@ apply_vertical_interpolation(const ekat::LinInterp& lin_interp, auto x_min = x_src[0][0]; auto x_max = x_src[last_src_pack_idx][last_src_pack_end-1]; auto set_mask = [&](const int ipack) { - auto oob = x_tgt[ipack]x_max; + auto in_range = ekat::range(ipack*Packsize) < nlevs_tgt; + auto oob = (x_tgt[ipack]x_max) and in_range; if (oob.any()) { y_tgt[ipack].set(oob,mask_val); } diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index 47c157e1820e..96b2f3e28059 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -272,10 +272,10 @@ TEST_CASE("io_remap_test","io_remap_test") // Confirm that remapped fields are correct. print (" -> Test Remapped Output ... \n",io_comm); - // ------------------------------------------------------------------------------------------------------ - // --- Vertical Remapping --- std::vector fnames = {"Y_flat","Y_mid","Y_int","V_mid","V_int"}; + // ------------------------------------------------------------------------------------------------------ + // --- Vertical Remapping --- { // Note, the vertical remapper defaults to a mask value of std numeric limits scaled by 0.1; const float mask_val = vert_remap_control.isParameter("Fill Value") @@ -461,7 +461,7 @@ TEST_CASE("io_remap_test","io_remap_test") // Test vertically + horizontally remapped output. // This test is a combination of the vertical test and horizontal test above. // There should be maksing in the vertical in all locations where the target pressure - // is lower higher than the surface pressure, just like in the vertical test. This should + // is lower/higher than the min/max of the surface pressure, just like in the vertical test. This should // also translate to more masking in the horizontal reamapping. So we must check for potential // masking for all variables rather than just the Y_int_at_XPa variable for the horizontal interpolation. // From ea3a693352fb2afb4042bfa053701c0c29c1a4d0 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Wed, 12 Jun 2024 11:41:54 -0600 Subject: [PATCH 450/476] Fix uninitialized variable and a few Cuda compiler warnings. --- .../mam/eamxx_mam_aci_process_interface.cpp | 1 + ...mxx_mam_microphysics_process_interface.cpp | 34 +++++++++++-------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 252745679029..8f9552537abb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -301,6 +301,7 @@ void MAMAci::initialize_impl(const RunType run_type) { wet_atm_.ni = get_field_in("ni").get_view(); // store rest fo the atm fields in dry_atm_in + dry_atm_.z_surf = 0; dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_int = get_field_in("p_int").get_view(); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 6ae368353025..7ea2adfddb63 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -306,6 +306,12 @@ void MAMMicrophysics::run_impl(const double dt) { // allocation perspective auto o3_col_dens = buffer_.scratch[8]; + const_view_1d &col_latitudes = col_latitudes_; + mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + mam_coupling::AerosolState &dry_aero = dry_aero_; + mam4::mo_photo::PhotoTableData &photo_table = photo_table_; + const int nlev = nlev_; + const Config &config = config_; // FIXME: read relevant linoz climatology data from file(s) based on time // FIXME: read relevant chlorine loading data from file based on time @@ -314,21 +320,21 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index - Real col_lat = col_latitudes_(icol); // column latitude (degrees?) + Real col_lat = col_latitudes(icol); // column latitude (degrees?) // fetch column-specific atmosphere state data - auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); - auto z_iface = ekat::subview(dry_atm_.z_iface, icol); - Real phis = dry_atm_.phis(icol); + auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + auto z_iface = ekat::subview(dry_atm.z_iface, icol); + Real phis = dry_atm.phis(icol); // set surface state data haero::Surface sfc{}; // fetch column-specific subviews into aerosol prognostics - mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero, icol); // set up diagnostics - mam4::Diagnostics diags(nlev_); + mam4::Diagnostics diags(nlev); // calculate o3 column densities (first component of col_dens in Fortran code) auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); @@ -347,7 +353,7 @@ void MAMMicrophysics::run_impl(const double dt) { mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, - atm.cloud_fraction, esfact, photo_table_, photo_work_arrays); + atm.cloud_fraction, esfact, photo_table, photo_work_arrays); // compute external forcings at time t(n+1) [molecules/cm^3/s] constexpr int extcnt = mam4::gas_chemistry::extcnt; @@ -356,7 +362,7 @@ void MAMMicrophysics::run_impl(const double dt) { mam4::mo_setext::extfrc_set(forcings, extfrc); // compute aerosol microphysics on each vertical level within this column - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev), [&](const int k) { constexpr int num_modes = mam4::AeroConfig::num_modes(); constexpr int gas_pcnst = mam_coupling::gas_pcnst(); @@ -429,7 +435,7 @@ void MAMMicrophysics::run_impl(const double dt) { constexpr int indexm = 0; // FIXME: index of xhnm in invariants array (??) Real cldnum = 0.0; // FIXME: droplet number concentration: where do we get this? setsox_single_level(loffset, dt, pmid, pdel, temp, mbar, lwc(k), - cldfrac, cldnum, invariants[indexm], config_.setsox, vmrcw, vmr); + cldfrac, cldnum, invariants[indexm], config.setsox, vmrcw, vmr); // calculate aerosol water content using water uptake treatment // * dry and wet diameters [m] @@ -442,7 +448,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config_.amicphys, step_, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, @@ -467,15 +473,15 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_o3_clim(icol, k), linoz_t_clim(icol, k), linoz_o3col_clim(icol, k), linoz_PmL_clim(icol, k), linoz_dPmL_dO3(icol, k), linoz_dPmL_dT(icol, k), linoz_dPmL_dO3col(icol, k), linoz_cariolle_psc(icol, k), - chlorine_loading, config_.linoz.psc_T, vmr[o3_ndx], + chlorine_loading, config.linoz.psc_T, vmr[o3_ndx], do3_linoz, do3_linoz_psc, ss_o3, o3col_du_diag, o3clim_linoz_diag, zenith_angle_degrees); // update source terms above the ozone decay threshold - if (k > nlev_ - config_.linoz.o3_lbl - 1) { + if (k > nlev - config.linoz.o3_lbl - 1) { Real do3_mass; // diagnostic, not needed - mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config_.linoz.o3_sfc, - config_.linoz.o3_tau, do3_mass); + mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config.linoz.o3_sfc, + config.linoz.o3_tau, do3_mass); } // ... check for negative values and reset to zero diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index ed156a9de893..3a0011468826 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -726,7 +726,7 @@ void compute_wet_mixing_ratios(const Team& team, // Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses // "packs". Following function copies a 2d view till model levels -KOKKOS_INLINE_FUNCTION +inline void copy_view_lev_slice(haero::ThreadTeamPolicy team_policy, //inputs const_view_2d &inp_view, //input view to copy const int dim, //dimension till view should be copied From b6dc164c3274092c57d8eaa6b9841ec2d12d3c24 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Wed, 12 Jun 2024 12:22:28 -0700 Subject: [PATCH 451/476] minor fixes to some syntax in ml interface that wouldn't compile --- .../eamxx_ml_correction_process_interface.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 91f23a884fa7..88bf109b4d8f 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -173,11 +173,12 @@ void MLCorrection::run_impl(const double dt) { const auto num_levs = m_num_levs; const auto policy = ESU::get_default_team_policy(m_num_cols, m_num_levs); + const auto &qv_told = qv_in.get_view(); const auto &qv_tnew = get_field_in("qv").get_view(); Kokkos::parallel_for("Compute WVP diff", policy, KOKKOS_LAMBDA(const MT& team) { const int icol = team.league_rank(); - auto qold_icol = ekat::subview(qv_in,icol); + auto qold_icol = ekat::subview(qv_told,icol); auto qnew_icol = ekat::subview(qv_tnew,icol); auto rho_icol = ekat::subview(pseudo_density,icol); Real net_column_moistening = 0; @@ -195,7 +196,7 @@ void MLCorrection::run_impl(const double dt) { Kokkos::parallel_reduce(Kokkos::TeamVectorRange(team, num_levs), [&] (const int& ilev, Real& lsum) { lsum += (qnew_icol(ilev)-qold_icol(ilev)) * rho_icol(ilev) / g; - },,Kokkos::Sum(net_column_moistening)); + },Kokkos::Sum(net_column_moistening)); team.team_barrier(); // Adjust Precipitation // - Note, we subtract the water vapor path because positive precip represents @@ -220,16 +221,9 @@ void MLCorrection::run_impl(const double dt) { } }); } - if (precip_liq_surf_mass(icol)<0) { - Kokkos::single(Kokkos::PerTeam(team), [&] { - precip_liq_surf_mass(icol) = 0.0; - }); - } - if (precip_ice_surf_mass(icol)<0) { - Kokkos::single(Kokkos::PerTeam(team), [&] { - precip_ice_surf_mass(icol) = 0.0; - }); - } + // Note, with the above formulation it is possible for the precipitation to go negative. + // We rely on the field property checker defined in the intialization function to repair + // any instances of negative precipitation. }); } } From c65c84e3fc3e2d16428ee931cf6e6f8293fd6c35 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 13 Jun 2024 10:53:12 -0700 Subject: [PATCH 452/476] Fix gptl include and remove Kokkos_ROOT settings --- cime_config/machines/config_machines.xml | 2 -- components/eamxx/src/share/CMakeLists.txt | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 00aaf3f7cc56..5c6caed992c2 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -277,7 +277,6 @@ Generic $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc-cmake-fix; else echo "$Albany_ROOT"; fi} $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} - $SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Kokkos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -2660,7 +2659,6 @@ $SHELL{if [ -z "$Albany_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/AlbanyBuilds/build-gcc-sfad12-e3sm/install; else echo "$Albany_ROOT"; fi} $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Trilinos_ROOT"; fi} - $SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Kokkos_ROOT"; fi} diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index e83f9d98ab9f..105b39a98086 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -61,7 +61,9 @@ target_include_directories(scream_share PUBLIC if (GPTL_PATH) target_include_directories(scream_share PUBLIC ${GPTL_PATH}) -endif () +else() + target_include_directories(scream_share PUBLIC ${INSTALL_SHAREDPATH}/include) +endif() target_link_libraries(scream_share PUBLIC ekat pioc) target_compile_options(scream_share PUBLIC From 7cd5e11750625c170c0a888a351bf350e46e8965 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 10 Jun 2024 14:40:15 -0600 Subject: [PATCH 453/476] Update haero submodule Allows building shared libs --- externals/haero | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/haero b/externals/haero index def15327bedb..8e4c96ded375 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit def15327bedb978ee76889f34b326e6e60591760 +Subproject commit 8e4c96ded375bf03bb5e83b0779e45a3aec9659e From 76cbb78b1a9256d732906f227d2ff34bc69160ba Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 11 Jun 2024 17:32:56 -0600 Subject: [PATCH 454/476] EAMxx: fix usage of code deprecated in Kokkos 4 --- .../eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp | 4 ++-- .../eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp | 4 ++-- components/eamxx/src/share/field/field_impl.hpp | 2 +- components/eamxx/src/share/tests/column_ops.cpp | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp index 531a513fc708..65158fdc3903 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp @@ -413,7 +413,7 @@ do_remap_fwd() using TeamPolicy = typename KT::TeamTagPolicy; - const auto concurrency = KT::ExeSpace::concurrency(); + const auto concurrency = KT::ExeSpace().concurrency(); #ifdef KOKKOS_ENABLE_CUDA #ifdef KOKKOS_ENABLE_DEBUG const int team_size = std::min(256, std::min(128*m_num_phys_cols,32*(concurrency/this->m_num_fields+31)/32)); @@ -450,7 +450,7 @@ do_remap_bwd() using TeamPolicy = typename KT::TeamTagPolicy; - const auto concurrency = KT::ExeSpace::concurrency(); + const auto concurrency = KT::ExeSpace().concurrency(); #ifdef KOKKOS_ENABLE_CUDA const int num_levs = m_phys_grid->get_num_vertical_levels(); const int team_size = std::min(128,32*(int)ceil(((Real)num_levs)/32)); diff --git a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp index 0a346ad2c1b8..ff61b26e3273 100644 --- a/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp +++ b/components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp @@ -135,8 +135,8 @@ class PhysicsDynamicsRemapper : public AbstractRemapper // so we'll just force to call this as pack_view(v). template KOKKOS_INLINE_FUNCTION - view_Nd pack_view (const OldViewT& v) const { - constexpr int N = OldViewT::Rank; + view_Nd pack_view (const OldViewT& v) const { + constexpr int N = OldViewT::rank; Kokkos::LayoutRight kl; for (int i=0; i Date: Fri, 14 Jun 2024 10:53:38 -0600 Subject: [PATCH 455/476] EAMxx: fix relative humidity unit test --- .../eamxx/src/diagnostics/tests/relative_humidity_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp b/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp index b36c6d1a3f4c..bb11fa794900 100644 --- a/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/relative_humidity_tests.cpp @@ -170,7 +170,7 @@ void run(std::mt19937_64& engine) Kokkos::parallel_for(Kokkos::TeamVectorRange(team,num_mid_packs), [&] (const Int& jpack) { dpdry_sub(jpack) = dpwet_sub(jpack) - dpwet_sub(jpack)*qv_sub(jpack); - auto qv_sat_l = physics::qv_sat_dry(T_mid_v(icol,jpack), p_dry_mid_v(icol,jpack), false, range_mask); + auto qv_sat_l = physics::qv_sat_dry(T_mid_v(icol,jpack), p_dry_mid_v(icol,jpack), true, range_mask); qv_sat_l *= dpdry_v(icol,jpack) ; qv_sat_l /= dpwet_v(icol,jpack) ; rh_v(icol,jpack) = qv_v(icol,jpack)/qv_sat_l; From 83d015a53456e57ffe6307799ad9a1f9cc4b4c79 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 13 Jun 2024 16:56:24 -0600 Subject: [PATCH 456/476] Update EKAT submodule --- externals/ekat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/ekat b/externals/ekat index 1c944624fe48..b39fd9fbd814 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 1c944624fe4888bf4cd7ddf14bec1e4e7e596504 +Subproject commit b39fd9fbd81439470e146d7915ed9a22c688ac6a From a59d502b0e4f9357224df7aa1239dc8fc90b9340 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 17 Jun 2024 13:19:12 -0600 Subject: [PATCH 457/476] EAMxx: add buildnml-time check for IC file and cmake option compatibility --- .../eamxx/cime_config/eamxx_buildnml.py | 41 +++++++++++++++++- .../eamxx/cime_config/eamxx_buildnml_impl.py | 2 + .../tests/eamxx_buildnml_unittest.nc | Bin 0 -> 26368 bytes components/eamxx/scripts/cime-nml-tests | 1 + 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/cime_config/tests/eamxx_buildnml_unittest.nc diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 0c7ed8e52aac..93ae2a9476c9 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -11,12 +11,14 @@ import xml.dom.minidom as md # Add path to scream libs -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts")) +EAMXXROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(os.path.join(EAMXXROOT, "scripts")) # SCREAM imports from eamxx_buildnml_impl import get_valid_selectors, get_child, refine_type, \ resolve_all_inheritances, gen_atm_proc_group, check_all_values, find_node from atm_manip import apply_atm_procs_list_changes_from_buffer, apply_non_atm_procs_list_changes_from_buffer +from utils import run_cmd_no_fail from utils import ensure_yaml # pylint: disable=no-name-in-module ensure_yaml() @@ -163,6 +165,22 @@ def perform_consistency_checks(case, xml): CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. Please, ensure restart happens on a step when rad is ON For daily (or less frequent) restart, rad_frequency must divide ATM_NCPL + >>> case = MockCase({'SCREAM_CMAKE_OPTIONS':'SCREAM_NUM_VERTICAL_LEV 15'}) + >>> xml_str = ''' + ... + ... + ... {}/eamxx_buildnml_unittest.nc + ... + ... + ... '''.format(os.path.join(EAMXXROOT,'cime_config/tests')) + >>> xml = ET.fromstring(xml_str) + >>> perform_consistency_checks(case,xml) + >>> case = MockCase({'SCREAM_CMAKE_OPTIONS':'SCREAM_NUM_VERTICAL_LEV 20'}) + >>> perform_consistency_checks(case,xml) + Traceback (most recent call last): + CIME.utils.CIMEError: ERROR: Error! IC file contains a number of levels different from the cmake option SCREAM_NUM_VERTICAL_LEV + ic file, lev: 15 + SCREAM_NUM_VERTICAL_LEV: 20 """ # RRTMGP can be supercycled. Restarts cannot fall in the middle @@ -205,6 +223,27 @@ def perform_consistency_checks(case, xml): " Please, ensure restart happens on a step when rad is ON\n" " For daily (or less frequent) restart, rad_frequency must divide ATM_NCPL") + # Check that SCREAM_NUM_VERTICAL_LEV matches nlev in the IC file + scream_opts = case.get_value("SCREAM_CMAKE_OPTIONS") + tokens = scream_opts.split() + expect (len(tokens) % 2 == 0, "Error! SCREAM_CMAKE_OPTIONS should contain a string of the form 'option1 value1 option2 value2 ...'\n") + it = iter(tokens) + cmake_args_dict = {} + for item in it: + cmake_args_dict[item] = next(it) + + nlevs_cmake = int(cmake_args_dict['SCREAM_NUM_VERTICAL_LEV']) + ic_xml = find_node(xml,"initial_conditions") + if ic_xml is not None: + ic_fname = get_child(ic_xml,"Filename") + cmd = f"ncdump -h {ic_fname.text}" + " | sed '/variables:/q' | awk '/\lQM^sJUY@%LeA5pgXs;~2H?k(GAj=KA9U)ICUe3P@> zUA6tbahCsB4)?R;>E2wTiIY3W-Q6HQ^P;$JhvqX5WdZ~U5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs w0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5csda4_V3)*#H0l literal 0 HcmV?d00001 diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index b128121cf95f..843781b0e231 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -429,6 +429,7 @@ def scripts_tests(machine=None): if machine: expect(is_machine_supported(machine), "Machine {} is not supported".format(machine)) CONFIG["machine"] = machine + setup_mach_env(machine) unittest.main(verbosity=2) From c7a803b007d8a425bd00d83c27eb64a077d8bdaa Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 17 Jun 2024 14:26:28 -0600 Subject: [PATCH 458/476] test-all-scream: Simplify the baseline model By default, test-all-scream will not run baseline tests. If you set -b AUTO, baseline tests will be done with the pre-existing public baselines in the location specified by the machine spec. You can regenerate baselines any time by using the -g flag, but be aware this will impact everyone if you regenerate the public baselines. You can change the target baseline area using -b $path. If -g is provided, no tests will be run; -g means generate only. The general workflow for baseline-changing PRs is: 1) Issue PR 2) AT will fail with differences in the baseline tests 3) Review and merge. 4) Ask JimF or LucaB for a bless to baselines on mappy and weaver. If you are developing a branch and baselines tests are failing unexpectedly, it is likely that your branch has fallen out of date. You should upstream merge or rebase your branch. To sum up, the baseline handling for test-all-scream should basically match what we do for create_test tests, the only difference is that test-all-scream does baseline comparison tests by default. --- .../scripts/jenkins/jenkins_common_impl.sh | 21 +- components/eamxx/scripts/scripts-tests | 49 +---- components/eamxx/scripts/test-all-scream | 55 +++-- components/eamxx/scripts/test_all_scream.py | 195 +++--------------- 4 files changed, 78 insertions(+), 242 deletions(-) diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index 664c911585e2..6e55482b10ce 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -72,7 +72,7 @@ if [ $skip_testing -eq 0 ]; then # IF such dir is not found, then the default (ctest-build/baselines) is used BASELINES_DIR=AUTO - TAS_ARGS="--baseline-dir $BASELINES_DIR \$compiler -p -c EKAT_DISABLE_TPL_WARNINGS=ON -i -m \$machine" + TAS_ARGS="--baseline-dir $BASELINES_DIR \$compiler -p -c EKAT_DISABLE_TPL_WARNINGS=ON -m \$machine" # pm-gpu needs to do work in scratch area in order not to fill home quota if [[ "$SCREAM_MACHINE" == "pm-gpu" ]]; then TAS_ARGS="${TAS_ARGS} -w /pscratch/sd/e/e3smtest/e3sm_scratch/pm-gpu/ctest-build" @@ -93,6 +93,16 @@ if [ $skip_testing -eq 0 ]; then fi fi + # AT runs may need an upstream merge in order to ensure that any DIFFs + # are caused by this PR and not simply because the PR is too far behind master. + if [ -z "$SCREAM_FAKE_ONLY" && $is_at_run == 1 ]; then + ./scripts/git-merge-ref origin/master + if [[ $? != 0 ]]; then + echo "MERGE FAILED! Please resolve conflicts" + exit 1 + fi + fi + SA_FAILURES_DETAILS="" # Run scream stand-alone tests (SA) if [ $test_SA -eq 1 ]; then @@ -167,15 +177,6 @@ if [ $skip_testing -eq 0 ]; then if [[ -z "$SCREAM_FAKE_ONLY" && $is_at_run == 1 ]]; then if [[ $test_v0 == 1 || $test_v1 == 1 ]]; then - # AT CIME runs may need an upstream merge in order to ensure that any DIFFs - # are caused by this PR and not simply because the PR is too far behind master. - if [ -n "$PULLREQUESTNUM" ]; then - ./scripts/git-merge-ref origin/master - if [[ $? != 0 ]]; then - echo "MERGE FAILED! Please resolve conflicts" - exit 1 - fi - fi fi if [[ $test_v0 == 1 ]]; then diff --git a/components/eamxx/scripts/scripts-tests b/components/eamxx/scripts/scripts-tests index b5af87320243..1fd2e48c1b40 100755 --- a/components/eamxx/scripts/scripts-tests +++ b/components/eamxx/scripts/scripts-tests @@ -214,7 +214,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): ############################################################################### CMDS_TO_TEST = [ - "./test-all-scream -m $machine -p -i -c EKAT_DISABLE_TPL_WARNINGS=ON", + "./test-all-scream -m $machine -p -c EKAT_DISABLE_TPL_WARNINGS=ON", "./test-all-scream -m $machine -t dbg", ] @@ -390,7 +390,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): # Start a couple new tests, baselines will be generated in ctest-build/baselines env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE1" - opts = " -g -t dbg -t sp --no-tests" + opts = "-b LOCAL -g -t dbg -t sp" cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) @@ -400,7 +400,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): # Re-run reusing baselines from above env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir LOCAL -t dbg -t sp --no-tests" + opts = "-b LOCAL -t dbg -t sp" cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) @@ -408,49 +408,6 @@ class TestTestAllScream(TestBaseOuter.TestBase): test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") - # Re-run dbg reusing baselines from above with a fake commit that's not ahead - # The flag -u implies -g, but nothing should happen, since SCREAM_FAKE_AHEAD=0 - env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=0 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir LOCAL -t dbg -u --no-tests" - cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", - self._machine) - run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - - test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") - test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") - - # Re-run dbg reusing baselines from above but expire them - # The flag -u implies -g, and since SCREAM_FAKE_AHEAD=1, baseline should be regenerated - env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=1 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir LOCAL -t dbg -u --no-tests" - cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", - self._machine) - run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - - test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE2") - test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") - - # Re-run reusing some baselines and expiring others - # The dbg baselines were generated in the prev step, so this should only gen the sp baselines - env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=1 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir LOCAL -t dbg -t sp -u --no-tests" - cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", - self._machine) - run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - - test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE2") - test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE2") - - # Re-run without reusing baselines, should force regeneration - env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE3" - opts = "-g -t dbg -t sp --no-tests" - cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", - self._machine) - run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - - test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE3") - test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE3") - else: self.skipTest("Skipping full run") diff --git a/components/eamxx/scripts/test-all-scream b/components/eamxx/scripts/test-all-scream index bb55ad23c216..3dc820581f71 100755 --- a/components/eamxx/scripts/test-all-scream +++ b/components/eamxx/scripts/test-all-scream @@ -9,6 +9,27 @@ do batch submissions. IMPORTANT: the default behavior of this script *changes your environment*, by loading machine-specific modules and setting machine-specific env vars. To prevent this behavior, use --preserve-env flag. + +Baselines: By default, test-all-scream will not run baseline tests. If you set +-b AUTO, baseline tests will be done with the pre-existing public baselines in +the location specified by the machine spec. You can regenerate +baselines any time by using the -g flag, but be aware this will impact everyone +if you regenerate the public baselines. You can change the target baseline area +using -b $path. If -g is provided, no tests will be run; -g means generate only. + +The general workflow for baseline-changing PRs is: +1) Issue PR +2) AT will fail with differences in the baseline tests +3) Review and merge. +4) Ask JimF or LucaB for a bless to baselines on mappy and weaver. + +If you are developing a branch and baselines tests are failing unexpectedly, it is +likely that your branch has fallen out of date. You should upstream merge or rebase +your branch. + +To sum up, the baseline handling for test-all-scream should basically match what we +do for create_test tests, the only difference is that test-all-scream does baseline +comparison tests by default. """ from utils import check_minimum_python_version @@ -27,26 +48,26 @@ def parse_command_line(args, description): OR {0} --help -\033[1mEXAMPLES (assumes user is on machine melvin):\033[0m - \033[1;32m# Run all tests on current machine using the SCREAM-approved env for this machine (this is the default env behavior) and your origin/master common ancestor for baseline generation (the default baseline behavior) \033[0m +\033[1mEXAMPLES (assumes user is on machine mappy):\033[0m + \033[1;32m# Run all tests on current machine using the SCREAM-approved env for this machine \033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin + > ./scripts/{0} -m mappy \033[1;32m# Run all tests on current machine with defaut behavior except using your current shell env \033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} --preserve-env -m melvin + > ./scripts/{0} --preserve-env -m mappy - \033[1;32m# Run all tests on current machine with default behavior except using pre-existing baselines (skips baseline generation) \033[0m + \033[1;32m# Run all tests on current machine with default behavior except using non-default baselines \033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin --baseline-dir=PATH_TO_BASELINES + > ./scripts/{0} -m mappy --baseline-dir=PATH_TO_BASELINES - \033[1;32m# Run only the dbg test on current machine with default behavior otherwise\033[0m + \033[1;32m# Run all tests on current machine with default behavior except using local baselines \033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin -t dbg + > ./scripts/{0} -m mappy --baseline-dir=LOCAL - \033[1;32m# Run all tests on current machine with default behavior on a repo with uncommitted changes\033[0m + \033[1;32m# Run only the dbg test on current machine with default behavior otherwise\033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin -k -b AUTO + > ./scripts/{0} -m mappy -t dbg """.format(pathlib.Path(args[0]).name), description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -63,11 +84,8 @@ OR parser.add_argument("-g", "--generate", action="store_true", help="Instruct test-all-scream to generate baselines from current commit. Skips tests") - parser.add_argument("-b", "--baseline-dir", default=None, - help="Directory where baselines should be read from (or written to, if -g/-i is used)") - - parser.add_argument("-u", "--update-expired-baselines", action="store_true", - help="Update baselines that appear to be expired (only used with -g)") + parser.add_argument("-b", "--baseline-dir", + help="Directory where baselines should be read from (or written to, if -g is used). Default is None which skips all baseline tests. AUTO means use public baselines. You can also use LOCAL to manage baselines in your local work dir. Lastly, you can provide a path here as well.") parser.add_argument("-m", "--machine", help="Provide machine name. This is *always* required. It can, but does not" @@ -76,7 +94,6 @@ OR "used as the CTEST_SITE for cdash if the tests are submitted. It is" "expected that a scream machine file exists for this value.") - parser.add_argument("--no-tests", action="store_true", help="Only build baselines, skip testing phase") parser.add_argument("--config-only", action="store_true", help="In the testing phase, only run config step, skip build and tests") @@ -95,9 +112,6 @@ OR parser.add_argument("-t", "--test", dest="tests", action="append", default=[], help=f"Only run specific test configurations, choices={choices_doc}") - parser.add_argument("-i", "--integration-test", action="store_true", - help="Merge origin/master into this branch before testing (implies -u).") - parser.add_argument("-l", "--local", action="store_true", help="Allow to not specify a machine name, and have test-all-scream to look" "for '~/.cime/scream_mach_specs.py' for machine specifications.") @@ -133,9 +147,6 @@ OR help="Set the testing level. Defaults to medium unless the test is cov or mem_check" "(short is default in those cases).") - parser.add_argument("--force-baseline-regen", action="store_true", - help="Existing baseline will always be considered expired even if they do not appear to be.") - return parser.parse_args(args[1:]) ############################################################################### diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index ffbe70e94f68..22c09642b02c 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -1,8 +1,6 @@ from utils import run_cmd, run_cmd_no_fail, expect, check_minimum_python_version, ensure_psutil, \ SharedArea, safe_copy -from git_utils import get_current_head, get_current_commit, get_current_branch, is_repo_clean, \ - cleanup_repo, merge_git_ref, git_refs_difference, print_last_commit, \ - create_backup_commit, checkout_git_ref +from git_utils import get_current_head, get_current_commit from test_factory import create_tests, COV @@ -30,14 +28,13 @@ class TestAllScream(object): ########################################################################### def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, - submit=False, parallel=False, generate=False, no_tests=False, + submit=False, parallel=False, generate=False, baseline_dir=None, machine=None, config_only=False, custom_cmake_opts=(), custom_env_vars=(), preserve_env=False, tests=(), - integration_test=False, local=False, root_dir=None, work_dir=None, + local=False, root_dir=None, work_dir=None, quick_rerun=False,quick_rerun_failed=False, - make_parallel_level=0, ctest_parallel_level=0, update_expired_baselines=False, - extra_verbose=False, limit_test_regex=None, test_level="at", test_size=None, - force_baseline_regen=False): + make_parallel_level=0, ctest_parallel_level=0, + extra_verbose=False, limit_test_regex=None, test_level="at", test_size=None): ########################################################################### # When using scripts-tests, we can't pass "-l" to test-all-scream, @@ -54,7 +51,6 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._parallel = parallel self._machine = machine self._local = local - self._run_tests = not no_tests self._config_only = config_only self._baseline_dir = baseline_dir self._custom_cmake_opts = custom_cmake_opts @@ -62,19 +58,13 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._preserve_env = preserve_env self._root_dir = root_dir self._work_dir = None if work_dir is None else Path(work_dir) - self._integration_test = integration_test self._quick_rerun = quick_rerun self._quick_rerun_failed = quick_rerun_failed self._extra_verbose = extra_verbose self._limit_test_regex = limit_test_regex self._test_level = test_level self._test_size = test_size - self._force_baseline_regen = force_baseline_regen - # Integration test always updates expired baselines - self._update_expired_baselines= update_expired_baselines or self._integration_test or self._force_baseline_regen - # If we are to update expired baselines, then we must run the generate phase - # NOTE: the gen phase will do nothing if baselines are present and not expired - self._generate = generate or self._update_expired_baselines + self._generate = generate if self._quick_rerun_failed: self._quick_rerun = True @@ -194,26 +184,6 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, expect(get_current_commit(), f"Root dir: {self._root_dir}, does not appear to be a git repo") - # Get git status info. Besides printing this info, we will need it to restore the repo initial - # configuration if we are running an integration test (where baselines need to be created - # from the origin/master commit) - self._original_branch = get_current_branch() - self._original_commit = get_current_commit() - - print_last_commit(git_ref=self._original_branch) - - # If we have an integration test, we need to merge master. Hence, do two things: - # 1) create bkp commit for all uncommitted/unstaged changes - # 2) save commit, so we can undo the merge after testing - self._has_backup_commit = False - if self._integration_test: - if not is_repo_clean(): - # Back up work in a temporary commit - create_backup_commit() - self._has_backup_commit = True - - self._original_commit = get_current_commit() - ################################### # Compute baseline info # ################################### @@ -232,40 +202,22 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._baseline_dir = local_baseline_dir elif self._baseline_dir == "AUTO": self._baseline_dir = auto_dir - elif self._baseline_dir is None: - if self._generate and not self._integration_test: - print ("No '--baseline-dir XYZ' provided. Baselines will be generated in {local_baseline_dir}.") - print ("NOTE: test-all-scream will proceed as if --force-baseline-regen was passed") - self._baseline_dir = local_baseline_dir - self._force_baseline_regen = True - self._update_expired_baselines = True - else: - print ("No '--baseline-dir XYZ' provided. Testing against default baselines dir for this machine.") - self._baseline_dir = auto_dir - - self._baseline_dir = Path(self._baseline_dir).absolute() - - # Only integration tests can overwrite the mach-specific baselines - if self._baseline_dir==auto_dir: - expect (not self._generate or self._integration_test or self._force_baseline_regen, - "You are not allowed to overwrite baselines in AUTO dir folder. Only -i and --force-baseline-regen can do that\n" - f" AUTO dir: {auto_dir}") + elif self._baseline_dir is not None: + self._baseline_dir = Path(self._baseline_dir).absolute() # Make the baseline dir, if not already existing. if self._generate: + expect(self._baseline_dir is not None, "Cannot generate without --baseline-dir") self.create_tests_dirs(self._baseline_dir, clean=False) - # For now, assume baselines are generated from HEAD. If -i was used, we'll change this - self._baseline_ref = "origin/master" if self._integration_test else self._original_commit - # Check baselines status - print (f"Checking baselines directory: {self._baseline_dir}") - missing_baselines = self.check_baselines_are_present() - expect (len(missing_baselines)==0 or self._generate, - f"Missing baselines for builds {missing_baselines}. Re-run with -g to generate them") - - if self._update_expired_baselines: - self.check_baselines_are_expired() + if self._baseline_dir is not None: + print (f"Checking baselines directory: {self._baseline_dir}") + missing_baselines = self.check_baselines_are_present() + expect (len(missing_baselines)==0 or self._generate, + f"Missing baselines for builds {missing_baselines}. Re-run with -g to generate them") + else: + print("baseline_dir is None! Skipping all baseline tests") ############################################ # Deduce compilers if needed/possible # @@ -301,16 +253,6 @@ def create_tests_dirs(self, root, clean): # Create the 'data' subdir (if not already existing) (test_dir / "data").mkdir(parents=False,exist_ok=True) - ############################################################################### - def get_baseline_file_sha(self, test): - ############################################################################### - baseline_file = (self.get_preexisting_baseline(test).parent)/"baseline_git_sha" - if baseline_file.exists(): - with baseline_file.open("r", encoding="utf-8") as fd: - return fd.read().strip() - - return None - ############################################################################### def set_baseline_file_sha(self, test): ############################################################################### @@ -362,7 +304,7 @@ def check_baselines_are_present(self): data_dir = self.get_preexisting_baseline(test) if not data_dir.is_dir(): test.baselines_missing = True - missing += [test.longname] + missing.append(test.longname) print(f" -> Test {test} is missing baselines") else: print(f" -> Test {test} appears to have baselines") @@ -371,61 +313,6 @@ def check_baselines_are_present(self): return missing - ############################################################################### - def check_baselines_are_expired(self): - ############################################################################### - """ - Baselines are expired if either: - 1) there is no file in baseline_dir containing the sha of the baselines - 2) the baselines sha does not match baseline_ref - """ - baseline_ref_sha = get_current_commit(commit=self._baseline_ref) - - for test in self._tests: - if not test.uses_baselines or test.baselines_missing: - continue - - if self._force_baseline_regen: - test.baselines_expired = True - print(f" -> Test {test} baselines are expired because self._force_baseline_regen=True") - continue - - # this test is not missing a baseline, but it may be expired. - baseline_file_sha = self.get_baseline_file_sha(test) - if baseline_file_sha is None: - test.baselines_missing = True - print(f" -> Test {test} has no stored sha so must be considered expired") - continue - - # There is a sha file, so check how it compares with self._baseline_ref - try: - num_ref_is_behind_file, num_ref_is_ahead_file = git_refs_difference(baseline_file_sha, baseline_ref_sha) - except SystemExit as e: - test.baselines_expired = True - reason = f"Failed to get refs difference between {baseline_file_sha} and {baseline_ref_sha} because: {e}" - print(f" -> Test {test} baselines are expired because {reason}") - continue - - # If the copy in our repo is behind, then we need to update the repo - expect (num_ref_is_behind_file==0 or not self._integration_test, -f"""Error! Your repo seems stale, since the baseline sha in your repo is behind -the one last used to generated them. We do *not* allow an integration -test to replace baselines with older ones, for security reasons. -If this is a legitimate case where baselines need to be 'rewound', -e.g. b/c of a (hopefully VERY RARE) force push to master, then -remove existing baselines first. Otherwise, please run 'git fetch $remote'. -- baseline_ref: {self._baseline_ref} -- repo baseline sha: {baseline_ref_sha} -- last used baseline sha: {baseline_file_sha}""") - - # If the copy in our repo is ahead, then baselines are expired - if num_ref_is_ahead_file > 0: - test.baselines_expired = True - reason = f"{self._baseline_ref} is ahead of the existing baseline commit {baseline_file_sha} by {num_ref_is_ahead_file}" - print(f" -> Test {test} baselines are expired because {reason}") - else: - print(f" -> Test {test} baselines are valid and do not need to be regenerated") - ############################################################################### def get_machine_file(self): ############################################################################### @@ -578,6 +465,8 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): if self._baseline_dir is not None and test.uses_baselines: cmake_config += f" -DSCREAM_BASELINES_DIR={self.get_preexisting_baseline(test).parent}" + else: + cmake_config += " -DSCREAM_ENABLE_BASELINE_TESTS=Off" if not self._submit: result += "-DNO_SUBMIT=True " @@ -671,7 +560,8 @@ def generate_baselines(self, test): dst = baseline_dir / "data" / src.name safe_copy(src, dst) - # Store the sha used for baselines generation + # Store the sha used for baselines generation. This is only for record + # keeping. self.set_baseline_file_sha(test) test.baselines_missing = False @@ -683,16 +573,14 @@ def generate_baselines(self, test): ############################################################################### def generate_all_baselines(self): ############################################################################### + git_head = get_current_head() tests_needing_baselines = self.baselines_to_be_generated() if len(tests_needing_baselines)==0: return True - # Switch to baseline ref - checkout_git_ref (self._baseline_ref) - print("###############################################################################") - print(f"Generating baselines from git ref {self._baseline_ref}") + print(f"Generating baselines using '{git_head}'") print("###############################################################################") tas_baseline_bld = self._work_dir / "tas_baseline_build" @@ -712,9 +600,6 @@ def generate_all_baselines(self): test = future_to_test[future] success &= future.result() - # Restore original commit - checkout_git_ref (self._original_commit) - return success ############################################################################### @@ -831,14 +716,11 @@ def baselines_to_be_generated(self): """ Return list of baselines to generate. Baselines need to be generated if - they are missing - - they are expired and we asked to update expired baselines """ ret = [] for test in self._tests: if test.baselines_missing: ret.append(test) - elif self._update_expired_baselines and test.baselines_expired: - ret.append(test) return ret @@ -852,31 +734,16 @@ def test_all_scream(self): os.environ.update( { key : val } ) success = True - try: - - if self._integration_test: - # Merge origin/master - merge_git_ref(git_ref=self._baseline_ref, verbose=True) - if self._generate: - success = self.generate_all_baselines() - if not success: - print ("Error(s) occurred during baselines generation phase") - - # Do not continue testing, as you may be testing against old/invalid baselines - return False - - if self._run_tests: - # First, create build directories (one per test). If existing, nuke the content - self.create_tests_dirs(self._work_dir, not self._quick_rerun) + if self._generate: + success = self.generate_all_baselines() - success &= self.run_all_tests() - if not success: - print ("Error(s) occurred during test phase") + else: + # First, create build directories (one per test). If existing, nuke the content + self.create_tests_dirs(self._work_dir, not self._quick_rerun) - finally: - # Cleanup the repo if needed - if self._original_commit!=get_current_commit(): - cleanup_repo(self._original_branch, self._original_commit, self._has_backup_commit) + success &= self.run_all_tests() + if not success: + print ("Error(s) occurred during test phase") return success From b99a4650981e13be2764c5fa9fa85b441b0e2e9e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 17 Jun 2024 14:59:47 -0600 Subject: [PATCH 459/476] Remove unnecessary upstream master merge We are already handling this in the git plugin --- .../eamxx/scripts/jenkins/jenkins_common_impl.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index 6e55482b10ce..69fe6ce0440f 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -93,16 +93,6 @@ if [ $skip_testing -eq 0 ]; then fi fi - # AT runs may need an upstream merge in order to ensure that any DIFFs - # are caused by this PR and not simply because the PR is too far behind master. - if [ -z "$SCREAM_FAKE_ONLY" && $is_at_run == 1 ]; then - ./scripts/git-merge-ref origin/master - if [[ $? != 0 ]]; then - echo "MERGE FAILED! Please resolve conflicts" - exit 1 - fi - fi - SA_FAILURES_DETAILS="" # Run scream stand-alone tests (SA) if [ $test_SA -eq 1 ]; then From 50acdfe3c64070bb88a95eaf0ab688cc492dbcf8 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 17 Jun 2024 15:37:29 -0600 Subject: [PATCH 460/476] Fix baselines_to_be_generated --- components/eamxx/scripts/test_all_scream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index 22c09642b02c..2d1ee9378ff8 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -714,12 +714,12 @@ def get_last_ctest_file(self,test,phase): def baselines_to_be_generated(self): ############################################################################### """ - Return list of baselines to generate. Baselines need to be generated if - - they are missing + Return list of baselines to generate. Baselines will always be generated + for tests that use baselines. """ ret = [] for test in self._tests: - if test.baselines_missing: + if test.uses_baselines: ret.append(test) return ret From 770c7d249803ebe1ab6413b021526c34d4406b4e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 17 Jun 2024 16:20:27 -0600 Subject: [PATCH 461/476] EAMxx: fix invocation of cime-nml-tests in jenkins script --- components/eamxx/scripts/cime-nml-tests | 2 +- components/eamxx/scripts/jenkins/jenkins_common_impl.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 843781b0e231..00dfc110d13c 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -10,7 +10,7 @@ from utils import check_minimum_python_version, expect, ensure_pylint, get_times check_minimum_python_version(3, 6) -from machines_specs import is_machine_supported +from machines_specs import is_machine_supported, setup_mach_env import unittest, argparse, sys, os, shutil from pathlib import Path diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index 664c911585e2..26f7604bb052 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -151,7 +151,7 @@ if [ $skip_testing -eq 0 ]; then scripts_fail=1 fi - ./scripts/cime-nml-tests + ./scripts/cime-nml-tests -m mappy if [[ $? != 0 ]]; then fails=$fails+1; scripts_fail=1 From fa7032348e91d6f12e41fe8c6bc40f8a9d021c06 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 17 Jun 2024 17:29:01 -0600 Subject: [PATCH 462/476] EAMxx: use special selector for cime scripts test --- .../eamxx/cime_config/namelist_defaults_scream.xml | 10 ++++++---- components/eamxx/scripts/cime-nml-tests | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50019833d8ae..ca34de90bf99 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -37,6 +37,8 @@ be lost if SCREAM_HACK_XML is not enabled. + + @@ -301,10 +303,10 @@ be lost if SCREAM_HACK_XML is not enabled. 1,2 - 3,4 - 5,6 - 3,4 - 5,6 + 3,4 + 5,6 + 3,4 + 5,6 diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 00dfc110d13c..ad107dae0b91 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -202,12 +202,12 @@ class TestBuildnml(unittest.TestCase): self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) # Test case 1: append to base, then to last. Should give all 3 entries stacked - run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_VERTICAL_LEV 1'", from_dir=case) + run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_TRACERS 1'", from_dir=case) run_cmd_no_fail("./case.setup", from_dir=case) self._get_values(case,"my_param","1,2,3,4,5,6") # Test case 2: append to last, then to base. Should give 1st and 3rd entry - run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_VERTICAL_LEV 2'", from_dir=case) + run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_TRACERS 2'", from_dir=case) run_cmd_no_fail("./case.setup", from_dir=case) self._get_values(case,"my_param","1,2,5,6") From e4dab90d3ef600a0bd6124013e56edf3b6c66bad Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 18 Jun 2024 09:26:25 -0600 Subject: [PATCH 463/476] EAMxx: fix sneaky error when parsing string in buildnml --- components/eamxx/cime_config/eamxx_buildnml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 93ae2a9476c9..bda3028be02c 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -236,7 +236,7 @@ def perform_consistency_checks(case, xml): ic_xml = find_node(xml,"initial_conditions") if ic_xml is not None: ic_fname = get_child(ic_xml,"Filename") - cmd = f"ncdump -h {ic_fname.text}" + " | sed '/variables:/q' | awk '/\ Date: Tue, 18 Jun 2024 10:17:28 -0600 Subject: [PATCH 464/476] EAMxx: fix typo in dummy XML selector for scripts unit tests --- components/eamxx/cime_config/namelist_defaults_scream.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index ca34de90bf99..c655cacf3277 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -38,7 +38,7 @@ be lost if SCREAM_HACK_XML is not enabled. - + From a67c1ee513683e3a5f6166178ff71606a4d53dd2 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Tue, 18 Jun 2024 11:06:52 -0600 Subject: [PATCH 465/476] Addresses some suggestions in issue #2858. There were two issues to be addressed: Use lambdas instead of functions in some Kokkos::parallel_for loops over team members. Fuse some Kokkos::parallel_for loops to improve efficiency. --- .../eamxx/src/diagnostics/aerocom_cld.hpp | 4 +- components/eamxx/src/diagnostics/aodvis.hpp | 2 +- .../physics/mam/eamxx_mam_aci_functions.hpp | 248 +++++++----------- .../mam/eamxx_mam_aci_process_interface.cpp | 21 +- .../mam/eamxx_mam_aci_process_interface.hpp | 8 +- .../atm_process/atmosphere_process_hash.cpp | 8 +- .../share/grid/remap/identity_remapper.hpp | 4 +- 7 files changed, 113 insertions(+), 182 deletions(-) diff --git a/components/eamxx/src/diagnostics/aerocom_cld.hpp b/components/eamxx/src/diagnostics/aerocom_cld.hpp index d4ead7848284..694eed76f097 100644 --- a/components/eamxx/src/diagnostics/aerocom_cld.hpp +++ b/components/eamxx/src/diagnostics/aerocom_cld.hpp @@ -15,7 +15,7 @@ class AeroComCld : public AtmosphereDiagnostic { AeroComCld(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // The name of the diagnostic - std::string name() const; + std::string name() const override; // Set the grid void set_grids( @@ -25,7 +25,7 @@ class AeroComCld : public AtmosphereDiagnostic { #ifdef KOKKOS_ENABLE_CUDA public: #endif - void compute_diagnostic_impl(); + void compute_diagnostic_impl() override; // Grid info int m_ncols; diff --git a/components/eamxx/src/diagnostics/aodvis.hpp b/components/eamxx/src/diagnostics/aodvis.hpp index 4d8289131d0e..20dc58ae8f25 100644 --- a/components/eamxx/src/diagnostics/aodvis.hpp +++ b/components/eamxx/src/diagnostics/aodvis.hpp @@ -26,7 +26,7 @@ class AODVis : public AtmosphereDiagnostic { #ifdef KOKKOS_ENABLE_CUDA public: #endif - void compute_diagnostic_impl(); + void compute_diagnostic_impl() override; int m_ncols; int m_nlevs; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 9fedb64555ef..03b04ac739a0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -9,30 +9,6 @@ namespace scream { namespace { -KOKKOS_INLINE_FUNCTION -void compute_w0_and_rho(const haero::ThreadTeam &team, - const MAMAci::const_view_2d omega, - const MAMAci::const_view_2d T_mid, - const MAMAci::const_view_2d p_mid, const int icol, - const int top_lev, const int nlev, - // output - MAMAci::view_2d w0, MAMAci::view_2d rho) { - // Get physical constants - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - // Gas constant for dry air [J/(kg*K) or J/Kg/K] - static constexpr auto rair = C::Rair; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - w0(icol, kk) = 0; - rho(icol, kk) = -999.0; - }); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); - w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); - }); -} void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, const mam_coupling::DryAtmosphere &dry_atmosphere, const int top_lev, const int nlev, @@ -44,17 +20,30 @@ void compute_w0_and_rho(haero::ThreadTeamPolicy team_policy, Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_w0_and_rho(team, omega, T_mid, p_mid, icol, top_lev, nlev, - // output - w0, rho); + // Get physical constants + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + // Gas constant for dry air [J/(kg*K) or J/Kg/K] + static constexpr auto rair = C::Rair; + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0u, top_lev), [&](int kk) { + w0(icol, kk) = 0; + rho(icol, kk) = -999.0; + }); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, top_lev, nlev), [&](int kk) { + rho(icol, kk) = p_mid(icol, kk) / (rair * T_mid(icol, kk)); + w0(icol, kk) = -1.0 * omega(icol, kk) / (rho(icol, kk) * gravit); + }); }); } -void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, +void compute_tke_at_interfaces(haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d var_mid, const MAMAci::view_2d dz, const int nlev_, - // output - MAMAci::view_2d var_int) { + MAMAci::view_2d w_sec_int, + // output + MAMAci::view_2d tke) { using CO = scream::ColumnOps; Kokkos::parallel_for( @@ -62,64 +51,23 @@ void compute_values_at_interfaces(haero::ThreadTeamPolicy team_policy, const int icol = team.league_rank(); const auto var_mid_col = ekat::subview(var_mid, icol); - const auto var_int_col = ekat::subview(var_int, icol); + const auto w_sec_int_col = ekat::subview(w_sec_int, icol); const auto dz_col = ekat::subview(dz, icol); const Real bc_top = var_mid_col(0); const Real bc_bot = var_mid_col(nlev_ - 1); CO::compute_interface_values_linear(team, nlev_, var_mid_col, dz_col, - bc_top, bc_bot, var_int_col); + bc_top, bc_bot, w_sec_int_col); + team.team_barrier(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nlev_ + 1), + [&](int kk) { + tke(icol, kk) = (3.0 / 2.0) * w_sec_int(icol, kk); + }); }); } -KOKKOS_INLINE_FUNCTION -void compute_tke_using_w_sec(const haero::ThreadTeam &team, - const MAMAci::const_view_2d w_sec, const int icol, - const int nlev, - // output - MAMAci::view_2d tke) { - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, nlev + 1), - KOKKOS_LAMBDA(int kk) { tke(icol, kk) = (3.0 / 2.0) * w_sec(icol, kk); }); -} -void compute_tke_using_w_sec(haero::ThreadTeamPolicy team_policy, - const MAMAci::const_view_2d w_sec, const int nlev, - // output - MAMAci::view_2d tke) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - compute_tke_using_w_sec(team, w_sec, icol, nlev, - // output - tke); - }); -} -KOKKOS_INLINE_FUNCTION -void compute_subgrid_scale_velocities( - const haero::ThreadTeam &team, const MAMAci::const_view_2d tke, - const Real wsubmin, const int icol, const int top_lev, const int nlev, - // output - MAMAci::view_2d wsub, MAMAci::view_2d wsubice, MAMAci::view_2d wsig) { - // More refined computation of sub-grid vertical velocity - // Set to be zero at the surface by initialization. - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, top_lev), KOKKOS_LAMBDA(int kk) { - wsub(icol, kk) = wsubmin; - wsubice(icol, kk) = 0.001; - wsig(icol, kk) = 0.001; - }); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * - (2.0 / 3.0)); - wsig(icol, kk) = - mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); - wsubice(icol, kk) = - mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); - wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); - }); -} void compute_subgrid_scale_velocities( haero::ThreadTeamPolicy team_policy, const MAMAci::const_view_2d tke, const Real wsubmin, const int top_lev, const int nlev, @@ -128,11 +76,26 @@ void compute_subgrid_scale_velocities( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_subgrid_scale_velocities(team, tke, wsubmin, icol, top_lev, - nlev, - // output - wsub, wsubice, wsig); - }); + // More refined computation of sub-grid vertical velocity + // Set to be zero at the surface by initialization. + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, top_lev), [&](int kk) { + wsub(icol, kk) = wsubmin; + wsubice(icol, kk) = 0.001; + wsig(icol, kk) = 0.001; + }); + // parallel_for ranges do not overlap, no need for barrier. + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, top_lev, nlev), [&](int kk) { + wsub(icol, kk) = haero::sqrt(0.5 * (tke(icol, kk) + tke(icol, kk + 1)) * + (2.0 / 3.0)); + wsig(icol, kk) = + mam4::utils::min_max_bound(0.001, 10.0, wsub(icol, kk)); + wsubice(icol, kk) = + mam4::utils::min_max_bound(0.2, 10.0, wsub(icol, kk)); + wsub(icol, kk) = haero::max(wsubmin, wsub(icol, kk)); + }); + }); } void compute_nucleate_ice_tendencies( @@ -204,31 +167,6 @@ void compute_nucleate_ice_tendencies( surf, progs, diags, tends); }); } -KOKKOS_INLINE_FUNCTION -void store_liquid_cloud_fraction( - const haero::ThreadTeam &team, const MAMAci::const_view_2d qc, - const MAMAci::const_view_2d qi, const MAMAci::const_view_2d liqcldf, - const MAMAci::const_view_2d liqcldf_prev, const int icol, const int top_lev, - const int nlev, - // output - MAMAci::view_2d cloud_frac, MAMAci::view_2d cloud_frac_prev) { - //------------------------------------------------------------- - // Get old and new liquid cloud fractions when amount of cloud - // is above qsmall threshold value - //------------------------------------------------------------- - // cut-off for cloud amount (ice or liquid) - static constexpr auto qsmall = 1e-18; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, top_lev, nlev), KOKKOS_LAMBDA(int kk) { - if((qc(icol, kk) + qi(icol, kk)) > qsmall) { - cloud_frac(icol, kk) = liqcldf(icol, kk); - cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); - } else { - cloud_frac(icol, kk) = 0; - cloud_frac_prev(icol, kk) = 0; - } - }); -} void store_liquid_cloud_fraction( haero::ThreadTeamPolicy team_policy, const mam_coupling::DryAtmosphere &dry_atmosphere, @@ -241,25 +179,25 @@ void store_liquid_cloud_fraction( Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - store_liquid_cloud_fraction(team, qc, qi, liqcldf, liqcldf_prev, icol, - top_lev, nlev, - // output - cloud_frac, cloud_frac_prev); - }); -} -KOKKOS_INLINE_FUNCTION -void compute_recipical_pseudo_density(const haero::ThreadTeam &team, - const MAMAci::const_view_2d pdel, - const int icol, const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, nlev), KOKKOS_LAMBDA(int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); + //------------------------------------------------------------- + // Get old and new liquid cloud fractions when amount of cloud + // is above qsmall threshold value + //------------------------------------------------------------- + // cut-off for cloud amount (ice or liquid) + static constexpr auto qsmall = 1e-18; // BAD_CONSTANT + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, top_lev, nlev), [&](int kk) { + if((qc(icol, kk) + qi(icol, kk)) > qsmall) { + cloud_frac(icol, kk) = liqcldf(icol, kk); + cloud_frac_prev(icol, kk) = liqcldf_prev(icol, kk); + } else { + cloud_frac(icol, kk) = 0; + cloud_frac_prev(icol, kk) = 0; + } + }); }); } + void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, MAMAci::const_view_2d pdel, const int nlev, @@ -268,16 +206,20 @@ void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - compute_recipical_pseudo_density(team, pdel, icol, nlev, - // output - rpdel); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); }); } void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, - const MAMAci::const_view_2d kvh, const MAMAci::view_2d wsub, + const MAMAci::const_view_2d kvh_mid, const MAMAci::view_2d kvh_int, + const MAMAci::view_2d wsub, const MAMAci::view_2d cloud_frac, const MAMAci::view_2d cloud_frac_prev, const mam_coupling::AerosolState &dry_aero, const int nlev, @@ -305,6 +247,8 @@ void call_function_dropmixnuc( MAMAci::view_2d raercol[mam4::ndrop::pver][2], MAMAci::view_3d state_q_work, MAMAci::view_3d nact, MAMAci::view_3d mact, MAMAci::view_2d dropmixnuc_scratch_mem[MAMAci::dropmix_scratch_]) { + + using CO = scream::ColumnOps; // Extract atmosphere variables MAMAci::const_view_2d T_mid = dry_atmosphere.T_mid; MAMAci::const_view_2d p_mid = dry_atmosphere.p_mid; @@ -312,6 +256,7 @@ void call_function_dropmixnuc( MAMAci::const_view_2d pdel = dry_atmosphere.p_del; MAMAci::const_view_2d p_int = dry_atmosphere.p_int; MAMAci::const_view_2d nc = dry_atmosphere.nc; + MAMAci::const_view_2d atm_dz= dry_atmosphere.dz; //---------------------------------------------------------------------- // ## Declare local variables for class variables @@ -428,10 +373,21 @@ void call_function_dropmixnuc( haero::Atmosphere haero_atm = atmosphere_for_column(dry_atmosphere, icol); + // Compute kvh at interfaces + const auto kvh_mid_col = ekat::subview(kvh_mid, icol); + const auto kvh_int_col = ekat::subview(kvh_int, icol); + const auto dz_col = ekat::subview(atm_dz, icol); + + const Real bc_top = kvh_mid_col(0); + const Real bc_bot = kvh_mid_col(nlev - 1); + + CO::compute_interface_values_linear(team, nlev, kvh_mid_col, dz_col, + bc_top, bc_bot, kvh_int_col); + // Construct state_q (interstitial) and qqcw (cloud borne) arrays constexpr auto pver = mam4::ndrop::pver; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0u, pver), [&](int klev) { + Kokkos::TeamVectorRange(team, pver), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -466,14 +422,14 @@ void call_function_dropmixnuc( } } }); - + team.team_barrier(); mam4::ndrop::dropmixnuc( team, dt, ekat::subview(T_mid, icol), ekat::subview(p_mid, icol), ekat::subview(p_int, icol), ekat::subview(pdel, icol), ekat::subview(rpdel, icol), // in zm[kk] - zm[kk+1], for pver zm[kk-1] - zm[kk] ekat::subview(zm, icol), ekat::subview(state_q_work_loc, icol), - ekat::subview(nc, icol), ekat::subview(kvh, icol), // kvh[kk+1] + ekat::subview(nc, icol), ekat::subview(kvh_int, icol), // kvh[kk+1] ekat::subview(cloud_frac, icol), lspectype_amode, specdens_amode, spechygro, lmassptr_amode, num2vol_ratio_min_nmodes, num2vol_ratio_max_nmodes, numptr_amode, nspec_amode, exp45logsig, @@ -518,19 +474,6 @@ void update_cloud_borne_aerosols( } } -// Update interstitial aerosols using tendencies - levels -KOKKOS_INLINE_FUNCTION -void update_interstitial_aerosols_levs(const haero::ThreadTeam &team, - const int nlev, const int icol, - const Real dt, - const MAMAci::view_2d ptend_view, - // output - MAMAci::view_2d aero_mr) { - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { - aero_mr(icol, kk) += ptend_view(icol, kk) * dt; - }); -} - // Update interstitial aerosols using tendencies- cols and levs void update_interstitial_aerosols( haero::ThreadTeamPolicy team_policy, @@ -553,10 +496,9 @@ void update_interstitial_aerosols( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, dt, - ptend_view, - // output - aero_mmr); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + aero_mmr(icol, kk) += ptend_view(icol, kk) * dt; + }); }); // update index for the next species (only if aero_mmr.data() is True) ++s_idx; @@ -569,9 +511,9 @@ void update_interstitial_aerosols( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); // update values for all levs at this column - update_interstitial_aerosols_levs(team, nlev, icol, dt, ptend_view, - // output - aero_nmr); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + aero_nmr(icol, kk) += ptend_view(icol, kk) * dt; + }); }); ++s_idx; // update index for the next species } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 8f9552537abb..e919aa066b47 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -564,14 +564,9 @@ void MAMAci::run_impl(const double dt) { // output w0_, rho_); - // Get w_sec_int_ from w_sec_mid_ - compute_values_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, - // output - w_sec_int_); - Kokkos::fence(); // wait for w_sec_int_ to be computed. - compute_tke_using_w_sec(team_policy, w_sec_int_, nlev_, - // output - tke_); + compute_tke_at_interfaces(team_policy, w_sec_mid_, dry_atm_.dz, nlev_, w_sec_int_, + // output + tke_); Kokkos::fence(); // wait for tke_ to be computed. compute_subgrid_scale_velocities(team_policy, tke_, wsubmin_, top_lev_, nlev_, @@ -583,7 +578,7 @@ void MAMAci::run_impl(const double dt) { aitken_dry_dia_, ekat::subview_1(dgnum_, static_cast(mam4::ModeIndex::Aitken))); - Kokkos::fence(); // wait for aitken_dry_dia_ to be computed. + Kokkos::fence(); // wait for aitken_dry_dia_ to be copied. // Compute Ice nucleation // NOTE: The Fortran version uses "ast" for cloud fraction which is @@ -608,16 +603,10 @@ void MAMAci::run_impl(const double dt) { Kokkos::fence(); // wait for rpdel_ to be computed. - // Get kvh_int_ from kvh_mid_ - compute_values_at_interfaces(team_policy, kvh_mid_, dry_atm_.dz, nlev_, - // output - kvh_int_); - - Kokkos::fence(); // Compute activated CCN number tendency (tendnd_) and updated // cloud borne aerosols (stored in a work array) and interstitial // aerosols tendencies - call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_int_, wsub_, + call_function_dropmixnuc(team_policy, dt, dry_atm_, rpdel_, kvh_mid_, kvh_int_, wsub_, cloud_frac_, cloud_frac_prev_, dry_aero_, nlev_, // output coltend_, coltend_cw_, qcld_, ndropcol_, ndropmix_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index 97bc90d3f982..fc930ea11d7b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -159,17 +159,17 @@ class MAMAci final : public scream::AtmosphereProcess { MAMAci(const ekat::Comm &comm, const ekat::ParameterList ¶ms); // Process metadata: Return type of the process - AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } // Return name of the process - std::string name() const { return "mam4_aci"; } + std::string name() const override { return "mam4_aci"; } // grid void set_grids( const std::shared_ptr grids_manager) override; // management of common atm process memory - size_t requested_buffer_size_in_bytes() const { + size_t requested_buffer_size_in_bytes() const override { return mam_coupling::buffer_size(ncol_, nlev_); } @@ -178,7 +178,7 @@ class MAMAci final : public scream::AtmosphereProcess { // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; - void finalize_impl(){/*DO NOTHING*/}; + void finalize_impl() override {/*DO NOTHING*/}; // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d7796..6b2f2fd080bb 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,9 +129,9 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lld (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), - i, gaccum[i], label.c_str()); + i, (long long int)gaccum[i], label.c_str()); } void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) const { @@ -140,8 +140,8 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", - timestamp().get_num_steps(), gaccum, label.c_str()); + fprintf(stderr, "bfbhash> %14d %16lld (%s)\n", + timestamp().get_num_steps(), (long long int) gaccum, label.c_str()); } } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 2106e1b93696..768c4c6cbc13 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -63,7 +63,7 @@ class IdentityRemapper : public AbstractRemapper return src_layout; } - void register_field_from_src (const field_type& src) { + void register_field_from_src (const field_type& src) override { EKAT_REQUIRE_MSG (m_aliasing!=SrcAliasTgt, "Error! Makes no sense to register from src and ask that src alias tgt.\n"); if (m_aliasing==TgtAliasSrc) { @@ -72,7 +72,7 @@ class IdentityRemapper : public AbstractRemapper AbstractRemapper::register_field_from_src(src); } } - void register_field_from_tgt (const field_type& tgt) { + void register_field_from_tgt (const field_type& tgt) override { EKAT_REQUIRE_MSG (m_aliasing!=TgtAliasSrc, "Error! Makes no sense to register from tgt and ask that tgt alias src.\n"); if (m_aliasing==SrcAliasTgt) { From e31ec132c723530a78ee3409e642758e1ba2cd17 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 18 Jun 2024 19:56:53 -0600 Subject: [PATCH 466/476] EAMxx: remove empty if statement in jenkins bash script --- components/eamxx/scripts/jenkins/jenkins_common_impl.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index 69fe6ce0440f..75330d612762 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -166,9 +166,6 @@ if [ $skip_testing -eq 0 ]; then # Also, for the nightlies, we use a separate job to run CIME tests if [[ -z "$SCREAM_FAKE_ONLY" && $is_at_run == 1 ]]; then - if [[ $test_v0 == 1 || $test_v1 == 1 ]]; then - fi - if [[ $test_v0 == 1 ]]; then ../../cime/scripts/create_test e3sm_scream_v0 -c -b master --wait if [[ $? != 0 ]]; then From 5e4a1aede705af6c138085254ba51412ed841578 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 20 Jun 2024 11:52:31 -0500 Subject: [PATCH 467/476] Updating to SCORPIO v1.6.4 Updating SCORPIO from v1.6.3 to v1.6.4. SCORPIO v1.6.4 release includes the following enhancements, * Ability to write output using ADIOS without the I/O decomposition information (disabled by default, enable via configure opts) --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index cdd541e0cd70..34422f362e9e 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit cdd541e0cd708bece13b1f3ee42f66dfd6440aa7 +Subproject commit 34422f362e9e83b5bc5e013b5992fc2555d02ece From 4e2aaaa4e87d27b06b6dc3d987e00a6ade6a0654 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 20 Jun 2024 14:16:52 -0700 Subject: [PATCH 468/476] Reduces buffer size by 1 for shoc small kernels --- .../eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index d4403c5bce14..6dfadb1eee7b 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -395,7 +395,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess static constexpr int num_2d_vector_mid = 18; static constexpr int num_2d_vector_int = 12; #else - static constexpr int num_2d_vector_mid = 23; + static constexpr int num_2d_vector_mid = 22; static constexpr int num_2d_vector_int = 13; #endif static constexpr int num_2d_vector_tr = 1; From 4917fc8cfd678580e5c41f04ec850d7c87298b6c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 20 Jun 2024 17:16:30 -0600 Subject: [PATCH 469/476] Revert "EAMxx: add buildnml-time check for IC file and cmake option compatibility" --- .../eamxx/cime_config/eamxx_buildnml.py | 41 +----------------- .../eamxx/cime_config/eamxx_buildnml_impl.py | 2 - .../cime_config/namelist_defaults_scream.xml | 10 ++--- .../tests/eamxx_buildnml_unittest.nc | Bin 26368 -> 0 bytes components/eamxx/scripts/cime-nml-tests | 7 ++- .../scripts/jenkins/jenkins_common_impl.sh | 2 +- 6 files changed, 9 insertions(+), 53 deletions(-) delete mode 100644 components/eamxx/cime_config/tests/eamxx_buildnml_unittest.nc diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index bda3028be02c..0c7ed8e52aac 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -11,14 +11,12 @@ import xml.dom.minidom as md # Add path to scream libs -EAMXXROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(os.path.join(EAMXXROOT, "scripts")) +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts")) # SCREAM imports from eamxx_buildnml_impl import get_valid_selectors, get_child, refine_type, \ resolve_all_inheritances, gen_atm_proc_group, check_all_values, find_node from atm_manip import apply_atm_procs_list_changes_from_buffer, apply_non_atm_procs_list_changes_from_buffer -from utils import run_cmd_no_fail from utils import ensure_yaml # pylint: disable=no-name-in-module ensure_yaml() @@ -165,22 +163,6 @@ def perform_consistency_checks(case, xml): CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. Please, ensure restart happens on a step when rad is ON For daily (or less frequent) restart, rad_frequency must divide ATM_NCPL - >>> case = MockCase({'SCREAM_CMAKE_OPTIONS':'SCREAM_NUM_VERTICAL_LEV 15'}) - >>> xml_str = ''' - ... - ... - ... {}/eamxx_buildnml_unittest.nc - ... - ... - ... '''.format(os.path.join(EAMXXROOT,'cime_config/tests')) - >>> xml = ET.fromstring(xml_str) - >>> perform_consistency_checks(case,xml) - >>> case = MockCase({'SCREAM_CMAKE_OPTIONS':'SCREAM_NUM_VERTICAL_LEV 20'}) - >>> perform_consistency_checks(case,xml) - Traceback (most recent call last): - CIME.utils.CIMEError: ERROR: Error! IC file contains a number of levels different from the cmake option SCREAM_NUM_VERTICAL_LEV - ic file, lev: 15 - SCREAM_NUM_VERTICAL_LEV: 20 """ # RRTMGP can be supercycled. Restarts cannot fall in the middle @@ -223,27 +205,6 @@ def perform_consistency_checks(case, xml): " Please, ensure restart happens on a step when rad is ON\n" " For daily (or less frequent) restart, rad_frequency must divide ATM_NCPL") - # Check that SCREAM_NUM_VERTICAL_LEV matches nlev in the IC file - scream_opts = case.get_value("SCREAM_CMAKE_OPTIONS") - tokens = scream_opts.split() - expect (len(tokens) % 2 == 0, "Error! SCREAM_CMAKE_OPTIONS should contain a string of the form 'option1 value1 option2 value2 ...'\n") - it = iter(tokens) - cmake_args_dict = {} - for item in it: - cmake_args_dict[item] = next(it) - - nlevs_cmake = int(cmake_args_dict['SCREAM_NUM_VERTICAL_LEV']) - ic_xml = find_node(xml,"initial_conditions") - if ic_xml is not None: - ic_fname = get_child(ic_xml,"Filename") - cmd = f"ncdump -h {ic_fname.text}" + r" | sed '/variables:/q' | awk '/\ - - @@ -303,10 +301,10 @@ be lost if SCREAM_HACK_XML is not enabled. 1,2 - 3,4 - 5,6 - 3,4 - 5,6 + 3,4 + 5,6 + 3,4 + 5,6 diff --git a/components/eamxx/cime_config/tests/eamxx_buildnml_unittest.nc b/components/eamxx/cime_config/tests/eamxx_buildnml_unittest.nc deleted file mode 100644 index fc3f7d26148f23bdaa61e62a14f961b8a8e05e47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26368 zcmeIuO%B0O5CGt+vOvO7+<`cOy#lQM^sJUY@%LeA5pgXs;~2H?k(GAj=KA9U)ICUe3P@> zUA6tbahCsB4)?R;>E2wTiIY3W-Q6HQ^P;$JhvqX5WdZ~U5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs w0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5csda4_V3)*#H0l diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index ad107dae0b91..b128121cf95f 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -10,7 +10,7 @@ from utils import check_minimum_python_version, expect, ensure_pylint, get_times check_minimum_python_version(3, 6) -from machines_specs import is_machine_supported, setup_mach_env +from machines_specs import is_machine_supported import unittest, argparse, sys, os, shutil from pathlib import Path @@ -202,12 +202,12 @@ class TestBuildnml(unittest.TestCase): self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) # Test case 1: append to base, then to last. Should give all 3 entries stacked - run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_TRACERS 1'", from_dir=case) + run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_VERTICAL_LEV 1'", from_dir=case) run_cmd_no_fail("./case.setup", from_dir=case) self._get_values(case,"my_param","1,2,3,4,5,6") # Test case 2: append to last, then to base. Should give 1st and 3rd entry - run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_TRACERS 2'", from_dir=case) + run_cmd_no_fail(f"./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_NUM_VERTICAL_LEV 2'", from_dir=case) run_cmd_no_fail("./case.setup", from_dir=case) self._get_values(case,"my_param","1,2,5,6") @@ -429,7 +429,6 @@ def scripts_tests(machine=None): if machine: expect(is_machine_supported(machine), "Machine {} is not supported".format(machine)) CONFIG["machine"] = machine - setup_mach_env(machine) unittest.main(verbosity=2) diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index f5b748b88ae5..75330d612762 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -151,7 +151,7 @@ if [ $skip_testing -eq 0 ]; then scripts_fail=1 fi - ./scripts/cime-nml-tests -m mappy + ./scripts/cime-nml-tests if [[ $? != 0 ]]; then fails=$fails+1; scripts_fail=1 From 87c3d05eceda2546275a12e49597d289cb52d59e Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 24 Jun 2024 11:23:09 -0600 Subject: [PATCH 470/476] EAMxx: fix compiler warning --- components/eamxx/src/diagnostics/field_at_pressure_level.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp index ed94e45bc9f2..21c1ac78dd90 100644 --- a/components/eamxx/src/diagnostics/field_at_pressure_level.cpp +++ b/components/eamxx/src/diagnostics/field_at_pressure_level.cpp @@ -116,8 +116,6 @@ void FieldAtPressureLevel::compute_diagnostic_impl() { using KT = KokkosTypes; using MemberType = typename KT::MemberType; - using view_1d = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; //This is 2D source pressure const Field& p_src = get_field_in(m_pressure_name); From d93f14d273eaecaf6b4bf0db8d2123cef218028b Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 24 Jun 2024 11:23:35 -0600 Subject: [PATCH 471/476] EAMxx: add timers to output manager on a per-yaml-file basis --- components/eamxx/src/control/atmosphere_driver.cpp | 1 + components/eamxx/src/share/io/scream_output_manager.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 60722664630d..b44b3e776389 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -724,6 +724,7 @@ void AtmosphereDriver::initialize_output_managers () { for (const auto& fname : output_yaml_files) { ekat::ParameterList params; ekat::parse_yaml_file(fname,params); + params.rename(ekat::split(fname,"/").back()); auto& checkpoint_pl = params.sublist("Checkpoint Control"); checkpoint_pl.set("frequency_units",checkpoint_params.get("frequency_units")); checkpoint_pl.set("Frequency",checkpoint_params.get("Frequency")); diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 9e25c0aa7483..e6bbe0221818 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -334,6 +334,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) std::string timer_root = m_is_model_restart_output ? "EAMxx::IO::restart" : "EAMxx::IO::standard"; start_timer(timer_root); + start_timer("EAMxx::IO::" + m_params.name()); // Check if this is a write step (and what kind) // Note: a full checkpoint not only writes globals in the restart file, but also all the history variables. @@ -545,6 +546,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) } } + stop_timer("EAMxx::IO::" + m_params.name()); stop_timer(timer_root); } /*===============================================================================================*/ From 1bd69ad552534b6ab071f7da24e681643ff5bc2c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 24 Jun 2024 16:04:57 -0600 Subject: [PATCH 472/476] EAMxx: bump micro version of pyeamxx --- components/eamxx/src/python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index 4d1861a242e3..fa5d6fc44817 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyeamxx" -version = "0.0.1" +version = "0.0.2" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] From c65acf174c3398282fbdbc505a9f4a0e1fcf8237 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 24 Jun 2024 17:04:19 -0700 Subject: [PATCH 473/476] remove redundant operation, fix the python library path in ruby cmake --- components/eamxx/cmake/machine-files/ruby-intel.cmake | 2 -- .../ml_correction/eamxx_ml_correction_process_interface.cpp | 1 - 2 files changed, 3 deletions(-) diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake index 70d8750df7ed..4b548e1767ec 100644 --- a/components/eamxx/cmake/machine-files/ruby-intel.cmake +++ b/components/eamxx/cmake/machine-files/ruby-intel.cmake @@ -1,7 +1,5 @@ include(${CMAKE_CURRENT_LIST_DIR}/ruby.cmake) set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) -set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE) -set(Python_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE) set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) diff --git a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp index 88bf109b4d8f..c423cb25cee9 100644 --- a/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp +++ b/components/eamxx/src/physics/ml_correction/eamxx_ml_correction_process_interface.cpp @@ -118,7 +118,6 @@ void MLCorrection::run_impl(const double dt) { // to use with precip adjustment. auto qv_src = get_field_in("qv"); auto qv_in = qv_src.clone(); - qv_in.deep_copy(qv_src); auto h_lat = m_lat.get_view(); auto h_lon = m_lon.get_view(); From 7142dffccf6f6de9b9c90e1ee0c78b0d0a87507b Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 26 Jun 2024 15:03:02 -0400 Subject: [PATCH 474/476] let's not lint files in components/eamxx for now --- .github/workflows/e3sm-gh-md-linter.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 6484335213dd..8ee8b7a7f13c 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -7,6 +7,8 @@ on: branches: ["master"] paths: - '**/*.md' + # for now let's not lint files in eamxx + - '!components/eamxx/**/*.md' jobs: linter: From 62b9f871368687da81abea8bb82734f3863160e1 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 26 Jun 2024 15:50:12 -0600 Subject: [PATCH 475/476] Fixes for rrtmgp updates --- components/cmake/build_model.cmake | 3 +++ .../physics/rrtmgp/cpp/rrtmgp_interface.cpp | 18 +++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 535081663a68..f0e6298d6850 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -137,6 +137,9 @@ macro(build_model COMP_CLASS COMP_NAME) # Add rrtmgp++ source code if asked for if (USE_RRTMGPXX) message(STATUS "Building RRTMGPXX") + # For now, hardcode YAKL mode in rrtmgp + set(RRTMGP_ENABLE_YAKL On) + add_definitions("-DRRTMGP_ENABLE_YAKL") # Build the static rrtmgpxx library set(RRTMGPXX_BIN ${CMAKE_CURRENT_BINARY_DIR}/rrtmgp) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/external/cpp ${RRTMGPXX_BIN}) diff --git a/components/eam/src/physics/rrtmgp/cpp/rrtmgp_interface.cpp b/components/eam/src/physics/rrtmgp/cpp/rrtmgp_interface.cpp index 697f35e86adc..404db9f73c3c 100644 --- a/components/eam/src/physics/rrtmgp/cpp/rrtmgp_interface.cpp +++ b/components/eam/src/physics/rrtmgp/cpp/rrtmgp_interface.cpp @@ -49,7 +49,7 @@ GasOpticsRRTMGP k_dist_sw; GasOpticsRRTMGP k_dist_lw; // Vector of strings to hold active gas names. -string1d active_gases; +string1dv active_gases; extern "C" void rrtmgp_initialize_cxx(int ngas, char *gas_names[], char const *coefficients_file_sw, char const *coefficients_file_lw) { // First, make sure yakl has been initialized @@ -66,9 +66,9 @@ extern "C" void rrtmgp_initialize_cxx(int ngas, char *gas_names[], char const *c // impossible from this initialization routine because I do not think the // rad_cnst objects are setup yet. // the other tasks! - active_gases = string1d("active_gases", ngas); + active_gases = string1dv(ngas); for (int igas=0; igas(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { - tmp2d(icol,ilay) = gas_vmr(igas,icol,ilay); + tmp2d(icol,ilay) = gas_vmr(igas+1,icol,ilay); }); - gas_concs.set_vmr(active_gases(igas), tmp2d); + gas_concs.set_vmr(active_gases[igas], tmp2d); } // Do gas optics @@ -460,11 +460,11 @@ extern "C" void rrtmgp_run_lw ( gas_concs.init(active_gases, ncol, nlay); real2d tmp2d; tmp2d = real2d("tmp", ncol, nlay); - for (int igas = 1; igas <= ngas; igas++) { + for (int igas = 0; igas < ngas; igas++) { parallel_for(SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) { - tmp2d(icol,ilay) = gas_vmr(igas,icol,ilay); + tmp2d(icol,ilay) = gas_vmr(igas+1,icol,ilay); }); - gas_concs.set_vmr(active_gases(igas), tmp2d); + gas_concs.set_vmr(active_gases[igas], tmp2d); } // Boundary conditions From 8002c3c8ceb107e9aab3cf950afebf7085e7b52a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 28 Jun 2024 18:04:54 -0500 Subject: [PATCH 476/476] Update rrtmgp sub to remove netcdf include --- components/eam/src/physics/rrtmgp/external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 59e6f72cc922..4641dc245ba0 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 59e6f72cc92268c69d1bd6ede570a959c1ebc28a +Subproject commit 4641dc245ba00cb230c740f4ab3fd0d5a7543a6b