E3SM-Project · jgfouca · Jul 1, 2024 · Mar 15, 2024 · Mar 17, 2024 · Mar 18, 2024
diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml
@@ -53,14 +53,14 @@ jobs:
         run: |
           echo "= The job was automatically triggered by a ${{github.event_name}} event."
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5.1.0
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install Python deps
         run: |
-          pip install mkdocs pymdown-extensions mkdocs-material mdutils
+          pip install mkdocs pymdown-extensions mkdocs-material mdutils mkdocs-bibtex
 
       - name: Generate EAMxx params docs
         working-directory: components/eamxx/scripts

diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,10 @@
 *.la
 *.a
 
+# Shared libraries
+*.so
+*.so.*
+
 # Executables
 *.exe
 
@@ -41,3 +45,9 @@ site
 components/eamxx/site/*
 # Ignore auto-generated eamxx_params.md file
 components/eamxx/docs/common/eamxx_params.md
+
+# Python packaging
+*.egg-info
+components/eamxx/src/python/build
+components/eamxx/src/python/build_src
+components/eamxx/src/python/dist
diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
@@ -277,7 +277,6 @@
       <env name="BLA_VENDOR">Generic</env>
       <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc-cmake-fix; else echo "$Albany_ROOT"; fi}</env>
       <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi}</env>
-      <env name="Kokkos_ROOT">$SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Kokkos_ROOT"; fi}</env>
     </environment_variables>
     <environment_variables compiler="nvidia" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi}</env>
@@ -1444,7 +1443,7 @@
         <command name="reset"></command>
         <command name="load">PrgEnv-cray</command>
         <command name="load">craype-accel-amd-gfx90a</command>
-        <command name="load">rocm/5.1.0</command>
+        <command name="load">rocm/5.4.0</command>
         <command name="load">libunwind/1.6.2</command>
       </modules>
       <modules>
@@ -1483,6 +1482,10 @@
       <env name="OMP_PROC_BIND">spread</env>
       <env name="OMP_PLACES">threads</env>
     </environment_variables>
+
+    <environment_variables compiler="crayclang-scream" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
   </machine>
 
   <!-- Skylake nodes of Stampede2 at TACC -->
@@ -2656,7 +2659,6 @@
     <environment_variables compiler="gnu">
       <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/AlbanyBuilds/build-gcc-sfad12-e3sm/install; else echo "$Albany_ROOT"; fi}</env>
       <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Trilinos_ROOT"; fi}</env>
-      <env name="Kokkos_ROOT">$SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Kokkos_ROOT"; fi}</env>
     </environment_variables>
   </machine>
 

diff --git a/cime_config/machines/config_workflow.xml b/cime_config/machines/config_workflow.xml
@@ -37,7 +37,7 @@
       <template>template.case.test</template>
       <prereq>$BUILD_COMPLETE and $TEST</prereq>
     </job>
-    <job name="case.post_run_io">
+    <!--job name="case.post_run_io">
       <template>template.post_run_io</template>
       <dependency>case.run</dependency>
       <prereq>case.get_value("PIO_TYPENAME_ATM") == 'adios' or \
@@ -53,11 +53,12 @@
       <runtime_parameters>
         <walltime>0:30:00</walltime>
       </runtime_parameters>
-    </job>
+    </job-->
     <job name="case.st_archive">
       <template>template.st_archive</template>
       <!-- If DOUT_S is true and case.run (or case.test) exits successfully then run st_archive-->
-      <dependency>(case.run and case.post_run_io) or case.test</dependency>
+      <!--dependency>(case.run and case.post_run_io) or case.test</dependency-->
+      <dependency>case.run or case.test</dependency>
       <prereq>$DOUT_S</prereq>
       <runtime_parameters>
         <task_count>1</task_count>

diff --git a/cime_config/tests.py b/cime_config/tests.py
@@ -723,6 +723,7 @@
         "time"  : "01:00:00",
         "tests" : (
             "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics",
+            "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci",
         )
     },
 

diff --git a/components/eam/src/dynamics/se/se_iop_intr_mod.F90 b/components/eam/src/dynamics/se/se_iop_intr_mod.F90
@@ -422,10 +422,6 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete
       call outfld('QDIFF',qdiff_dyn,plon,begchunk)
     endif
 
-    if (iop_coriolis) then
-      call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt)
-    endif
-
     call outfld('TOBS',tobs,plon,begchunk)
     call outfld('QOBS',qobs,plon,begchunk)
     call outfld('DIVQ',divq,plon,begchunk)
@@ -441,6 +437,10 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete
 
   enddo
 
+  if (iop_coriolis) then
+    call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt)
+  endif
+
   if ((iop_nudge_tq .or. iop_nudge_uv) .and. dp_crm) then
     ! If running in a doubly periodic CRM mode, then nudge the domain
     !   based on the domain mean and observed quantities of T, Q, u, and v

diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external
diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt
@@ -203,13 +203,24 @@ if (NOT SCREAM_SMALL_KERNELS)
   set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "")
 endif()
 
-# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS
+# Add RRTMGP settings. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS
 # to turn on the RRTMGP internal checks here as well, via
 #     option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG})
 # and then adding to scream_config.h:
 #     #cmakedefine RRTMGP_EXPENSIVE_CHECKS
 option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG})
 
+option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE)
+option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE)
+if (SCREAM_RRTMGP_ENABLE_YAKL)
+  add_definitions("-DRRTMGP_ENABLE_YAKL")
+endif()
+
+if (SCREAM_RRTMGP_ENABLE_KOKKOS)
+  add_definitions("-DRRTMGP_ENABLE_KOKKOS")
+endif()
+
+
 set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)")
 
 # For now, only used in share/grid/remap/refining_remapper_rma.*pp
@@ -261,6 +272,12 @@ set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(SCREAM_SRC_DIR  ${CMAKE_CURRENT_SOURCE_DIR}/src)
 set(SCREAM_BIN_DIR  ${CMAKE_CURRENT_BINARY_DIR})
 
+option (EAMXX_ENABLE_PYBIND "Whether to enable python interface to eamxx, via pybind11" OFF)
+if (EAMXX_ENABLE_PYBIND)
+  # Pybind11 requires shared libraries
+  set (BUILD_SHARED_LIBS ON)
+endif()
+
 ####################################################################
 #                    Packs-related settings                        #
 ####################################################################
@@ -618,4 +635,3 @@ print_var(SCREAM_TEST_THREAD_INC)
 print_var(SCREAM_TEST_MAX_RANKS)
 
 message ("**************************************************")
-
diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml
@@ -234,6 +234,12 @@ be lost if SCREAM_HACK_XML is not enabled.
       <Ckm type="real" doc="Eddy diffusivity coefficient for momentum">0.1</Ckm>
     </shoc>
 
+    <!-- MAM4xx-ACI -->
+    <mam4_aci inherit="atm_proc_base">
+      <wsubmin type="real" doc="Minimum diagnostic sub-grid vertical velocity">0.001</wsubmin>
+      <top_level_mam4xx type="integer" doc="Top level for MAM4xx">6</top_level_mam4xx>
+    </mam4_aci>
+
     <!-- CLD fraction -->
     <cldFraction inherit="atm_proc_base"/>
 
@@ -481,6 +487,33 @@ be lost if SCREAM_HACK_XML is not enabled.
     <surf_sens_flux    >0.0</surf_sens_flux>
     <surf_lw_flux_up   >0.0</surf_lw_flux_up>
     <surf_mom_flux     type="array(real)">0.0,0.0</surf_mom_flux>
+    <!-- Note: MAM4xx-ACI needs dry aerosol diameter (dgnum), currently set to aitken model nominal diameter, remove it once all of MAM4xx is integrated -->
+    <dgnum type="real" doc="Dry aerosol particles diameter in meters">2.6e-08</dgnum>
+        <bc_c1 type="real">0.0</bc_c1>
+    <bc_c3 type="real">0.0</bc_c3>
+    <bc_c4 type="real">0.0</bc_c4>
+    <dst_c1 type="real">0.0</dst_c1>
+    <dst_c3 type="real">0.0</dst_c3>
+    <so4_c1 type="real">0.0</so4_c1>
+    <so4_c2 type="real">0.0</so4_c2>
+    <so4_c3 type="real">0.0</so4_c3>
+    <pom_c1 type="real">0.0</pom_c1>
+    <pom_c3 type="real">0.0</pom_c3>
+    <pom_c4 type="real">0.0</pom_c4>
+    <soa_c1 type="real">0.0</soa_c1>
+    <soa_c2 type="real">0.0</soa_c2>
+    <soa_c3 type="real">0.0</soa_c3>
+    <nacl_c1 type="real">0.0</nacl_c1>
+    <nacl_c2 type="real">0.0</nacl_c2>
+    <nacl_c3 type="real">0.0</nacl_c3>
+    <mom_c1 type="real">0.0</mom_c1>
+    <mom_c2 type="real">0.0</mom_c2>
+    <mom_c3 type="real">0.0</mom_c3>
+    <mom_c4 type="real">0.0</mom_c4>
+    <num_c1 type="real">0.0</num_c1>
+    <num_c2 type="real">0.0</num_c2>
+    <num_c3 type="real">0.0</num_c3>
+    <num_c4 type="real">0.0</num_c4>
     <!-- default ne1024 initial condition files do not have these, so init to zero here -->
     <!-- TODO: delete this once we can tell the AD that some fields can be inited by procs -->
     <qc hgrid="ne256np4|ne1024np4">0.0</qc>
@@ -503,6 +536,7 @@ be lost if SCREAM_HACK_XML is not enabled.
     <output_yaml_files type="array(string)"/>
     <model_restart>
       <filename_prefix>./${CASE}.scream</filename_prefix>
+      <iotype>default</iotype>
       <output_control locked="true">
         <Frequency>${REST_N}</Frequency>
         <frequency_units>${REST_OPTION}</frequency_units>
@@ -551,6 +585,12 @@ be lost if SCREAM_HACK_XML is not enabled.
     <iop_dosubsidence COMPSET=".*DYCOMSrf01">true</iop_dosubsidence>
     <iop_srf_prop type="logical" doc="Use IOP file specified surface values in surface coupling importer.">false</iop_srf_prop>
     <iop_srf_prop COMPSET=".*DYCOMSrf01">true</iop_srf_prop>
+    <iop_nudge_uv type="logical" doc="Nudge winds to those in the IOP file.">false</iop_nudge_uv>
+    <iop_nudge_tq type="logical" doc="Nudge thermodynamics to those in the IOP file.">false</iop_nudge_tq>
+    <iop_nudge_tq_low type="real" doc="Lowest layer to apply nudging for t and q (pressure in hPa).">1100</iop_nudge_tq_low>
+    <iop_nudge_tq_high type="real" doc="Highest layer to apply nudging for t and q (pressure in hPa).">0</iop_nudge_tq_high>
+    <iop_nudge_tscale type="real" doc="Time scale to nudge thermodynamics or winds to.">10800</iop_nudge_tscale>
+    <iop_coriolis type="logical" doc="Apply coriolis forcing to winds based on large scale winds in IOP file.">false</iop_coriolis>
   </iop_options>
 
   <!-- Homme control namelist -->

diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands
@@ -0,0 +1,23 @@
+
+#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer
+#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options
+./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41"
+
+#modify initial condition file to get aerosol species ICs
+$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b
+
+# Add spa as RRTMG needs spa
+$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b
+
+# Replace spa with mam4_aci to invoke mam4 aci scheme
+$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b
+
+#Set precribed ccn to false so that P3 uses input from ACI
+$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_prescribed_ccn=false -b
+
+#Set predicted ccn to true so that P3 uses input from ACI
+$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_predict_nc=true -b
+
+
+
+
diff --git a/components/eamxx/cmake/CompareNCFiles.cmake b/components/eamxx/cmake/CompareNCFiles.cmake
@@ -13,19 +13,19 @@ function(CompareNCFiles)
   set (argsMv LABELS FIXTURES_REQUIRED)
 
   cmake_parse_arguments(PARSE "${options}" "${args1v}" "${argsMv}" ${ARGN})
-  CheckMacroArgs(CompareNCFilesFamily PARSE "${options}" "${args1v}" "${argsMv}")
+  CheckMacroArgs(CompareNCFiles PARSE "${options}" "${args1v}" "${argsMv}")
 
   # Sanity checks
   if (NOT PARSE_TEST_NAME)
-    message ("Error! CompareNCFilesPair requires the keyword argument TEST_NAME")
+    message ("Error! CompareNCFiles requires the keyword argument TEST_NAME")
     message (FATAL_ERROR "Aborting...")
   endif()
   if (NOT PARSE_SRC_FILE)
-    message ("Error! CompareNCFilesPair requires the keyword argument SRC_FILE")
+    message ("Error! CompareNCFiles requires the keyword argument SRC_FILE")
     message (FATAL_ERROR "Aborting...")
   endif()
   if (NOT PARSE_TGT_FILE)
-    message ("Error! CompareNCFilesPair requires the keyword argument TGT_FILE")
+    message ("Error! CompareNCFiles requires the keyword argument TGT_FILE")
     message (FATAL_ERROR "Aborting...")
   endif()
 

diff --git a/components/eamxx/cmake/Findmpi4py.cmake b/components/eamxx/cmake/Findmpi4py.cmake
@@ -0,0 +1,36 @@
+# - FindMPI4PY
+# Find mpi4py includes
+# This module defines:
+# MPI4PY_INCLUDE_DIR, where to find mpi4py.h, etc.
+# MPI4PY_FOUND
+
+function (SetMpi4pyIncludeDir)
+endfunction()
+
+if (NOT TARGET mpi4py)
+  # If user provided an include dir, we will use that, otherwise we'll ask python to find it
+  if (NOT MPI4PY_INCLUDE_DIR)
+    execute_process(COMMAND
+      "${PYTHON_EXECUTABLE}" "-c" "import mpi4py; print (mpi4py.get_include())"
+      OUTPUT_VARIABLE OUTPUT
+      RESULT_VARIABLE RESULT
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if (RESULT)
+      set(MPI4PY_FOUND FALSE)
+    else ()
+      set (MPI4PY_INCLUDE_DIR ${OUTPUT} CACHE PATH "Path to mpi4py include directory" FORCE)
+    endif()
+  endif()
+
+  # If we still don't have an include dir, it means we have no mpi4py installed
+  if (NOT MPI4PY_INCLUDE_DIR)
+    set(MPI4PY_FOUND FALSE)
+  else ()
+    add_library(mpi4py INTERFACE)
+    target_include_directories(mpi4py INTERFACE SYSTEM ${MPI4PY_INCLUDE_DIR})
+    set(MPI4PY_FOUND TRUE)
+  endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(mpi4py DEFAULT_MSG MPI4PY_INCLUDE_DIR)
diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake
@@ -0,0 +1,12 @@
+include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
+common_setup()
+
+include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake)
+include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)
+
+# Remove this if you are using a resource manager (slurm etc)
+set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "")
+
+# EKAT MPI settings
+set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec")
+set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n")
diff --git a/components/eamxx/cmake/machine-files/compy.cmake b/components/eamxx/cmake/machine-files/compy.cmake
@@ -7,4 +7,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake)
 include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake)
 
 #Compy SLURM specific settings
-set(EKAT_MPI_NP_FLAG "-p short -n" CACHE STRING "" FORCE)
+set(EKAT_MPI_NP_FLAG "-p short --mpi=pmi2 -n" CACHE STRING "" FORCE)
diff --git a/components/eamxx/cmake/machine-files/pm-gpu.cmake b/components/eamxx/cmake/machine-files/pm-gpu.cmake
@@ -18,6 +18,8 @@ endif()
 
 include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake)
 
+set(EKAT_MPI_EXTRA_ARGS "${EKAT_MPI_EXTRA_ARGS} --gpus-per-task=1" CACHE STRING "" FORCE)
+
 #option(Kokkos_ARCH_AMPERE80 "" ON)
 set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE)
 

diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake
@@ -1,7 +1,6 @@
 include(${CMAKE_CURRENT_LIST_DIR}/ruby.cmake)
 set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE)
-set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE STRING "" FORCE)
 set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE)
 option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON)
 set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE)
-execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate)
+execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate)
diff --git a/components/eamxx/cmake/tpls/Scorpio.cmake b/components/eamxx/cmake/tpls/Scorpio.cmake
@@ -42,6 +42,17 @@ macro (CreateScorpioTargets)
     option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON)
 
     add_subdirectory (${E3SM_EXTERNALS_DIR}/scorpio ${CMAKE_BINARY_DIR}/externals/scorpio)
+
+    set (SCORPIO_Fortran_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/externals/scorpio/src/flib CACHE INTERNAL "SCORPIO Fortran include dirs")
+    set (C_INCLUDE_DIRS "${E3SM_EXTERNALS_DIR}/scorpio/src/clib")
+    list(APPEND C_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/externals/scorpio/src/clib")
+    set (SCORPIO_C_INCLUDE_DIRS "${C_INCLUDE_DIRS}" CACHE INTERNAL "SCORPIO C include dirs")
+
+    # Add GPTL from SCORPIO
+    if (NOT GPTL_PATH)
+      set (GPTL_PATH ${E3SM_EXTERNALS_DIR}/scorpio/src/gptl CACHE INTERNAL "Path to GPTL library")
+    endif ()
+
     EkatDisableAllWarning(pioc)
     EkatDisableAllWarning(piof)
     EkatDisableAllWarning(gptl)