UFS-dev PR#75

.gitmodules

@@ -1,7 +1,9 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NCAR/fv3atm
-  branch = main
+  #url = https://github.com/NCAR/fv3atm
+  #branch = main
+  url = https://github.com/grantfirl/fv3atm
+  branch = ufs-dev-PR75
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3

.readthedocs.yaml

@@ -4,6 +4,10 @@
 
 # Required
 version: 2
+build: 
+  os: ubuntu-20.04
+  tools: 
+    python: "3.9"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -18,6 +22,5 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
   install:
     - requirements: doc/UsersGuide/requirements.txt

CICE-interface/CMakeLists.txt

@@ -2,6 +2,9 @@
 if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
   set(CMAKE_Fortran_FLAGS "-g -fbacktrace")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -ffree-line-length-none ")
+  if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz")
+  endif()
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
   set(CMAKE_Fortran_LINK_FLAGS "" )

CMEPS-interface/CMakeLists.txt

@@ -29,7 +29,7 @@ endif()
 ### Source files
 ###############################################################################
 
-list(APPEND _nems_util_files
+list(APPEND _ufs_util_files
   CMEPS/ufs/flux_atmocn_mod.F90
   CMEPS/ufs/glc_elevclass_mod.F90
   CMEPS/ufs/ufs_kind_mod.F90
@@ -60,7 +60,7 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_merge_mod.F90
   CMEPS/mediator/med_constants_mod.F90
   CMEPS/mediator/med_kind_mod.F90
-  CMEPS/mediator/esmFldsExchange_nems_mod.F90
+  CMEPS/mediator/esmFldsExchange_ufs_mod.F90
   CMEPS/mediator/med_phases_prep_lnd_mod.F90
   CMEPS/mediator/med_phases_prep_atm_mod.F90
   CMEPS/mediator/med_phases_prep_ocn_mod.F90
@@ -134,7 +134,7 @@ if(CMEPS_AOFLUX)
     endif(return_code EQUAL 0)
 
     ### Add extra driver file to the source
-    list(APPEND _nems_util_files
+    list(APPEND _ufs_util_files
       CMEPS/ufs/ccpp/driver/med_ccpp_driver.F90
       CMEPS/ufs/flux_atmocn_ccpp_mod.F90
       CMEPS/ufs/ufs_io_mod.F90)
@@ -172,7 +172,7 @@ endif()
 ###############################################################################
 
 ### Create target library
-add_library(cmeps STATIC ${_nems_util_files} ${_mediator_files} ${SCHEMES} ${CAPS} ${API})
+add_library(cmeps STATIC ${_ufs_util_files} ${_mediator_files} ${SCHEMES} ${CAPS} ${API})
 
 ### Set PUBLIC interfaces on the library
 set_target_properties(cmeps PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)

doc/UsersGuide/source/AutomatedTesting.rst

@@ -31,14 +31,14 @@ The CI/CD workflow then executes the following steps:
       subcomponents are up to date with the top of the ``develop`` branch.
 
    #. If the check is successful, build jobs are started on GHA-provided virtual machines
-      by downloading the hpc-stack Docker container stored in Docker Hub.
+      by downloading the HPC-Stack Docker container stored in Docker Hub.
 
    #. Once all build jobs are successful, the created executable files are stored as
       artifacts in GHA.
 
    #. A number of AWS EC2 instances are started.
 
-   #. Test jobs are started on AWS after downloading the hpc-stack Docker container,
+   #. Test jobs are started on AWS after downloading the HPC-Stack Docker container,
       the executable file from the build job, and the input-data Docker container.
 
    #. When all tests are complete, EC2 instances are stopped. Test results are reported

doc/UsersGuide/source/BuildingAndRunning.rst

@@ -14,15 +14,15 @@ through NOAA and its affiliates. These systems are named (e.g., Hera, Orion, Che
 Level 3 & 4 systems include certain personal computers or non-NOAA-affiliated HPC systems. 
 The prerequisite software libraries for building the WM already exist in a centralized location on Level 1/preconfigured 
 systems, so users may skip directly to :ref:`getting the data <GetData>` and downloading the code. 
-On other systems, users will need to build the prerequisite libraries using :term:`HPC-Stack` or :term:`spack-stack`. 
+On other systems, users will need to build the prerequisite libraries using :term:`spack-stack` or :term:`HPC-Stack`. 
 
 =======================
 Prerequisite Libraries
 =======================
 
 The UFS WM requires a number of libraries.
 The WM uses two categories of libraries, which are available as a bundle via 
-:term:`HPC-Stack` or :term:`spack-stack`:
+:term:`spack-stack` or :term:`HPC-Stack`:
 
    #. :term:`NCEP` libraries (:term:`NCEPLIBS`): These are libraries developed for use with NOAA weather models.
       Most have an NCEPLIBS prefix in the repository (e.g., NCEPLIBS-bacio). Select tools from the UFS
@@ -34,11 +34,12 @@ The WM uses two categories of libraries, which are available as a bundle via
       instead. 
 
 .. note::
-   Currently, HPC-Stack is the software stack validated by the UFS WM for running :term:`regression tests <RT>`. 
-   However, UFS applications are shifting to spack-stack, which is a Spack-based 
-   method for installing UFS prerequisite software libraries. The spack-stack is currently 
-   used on NOAA Cloud platforms and in containers, while HPC-Stack is still used on NOAA
-   Research & Development HPC Systems (RDHPCS). 
+   Currently, spack-stack is the software stack validated by the UFS WM for running 
+   :term:`regression tests <RT>`. Spack-stack is a Spack-based method for installing UFS 
+   prerequisite software libraries. UFS applications and components are also shifting to 
+   spack-stack from HPC-Stack but are at various stages of this transition. 
+   Although users can still build and use HPC-Stack, the UFS WM no longer uses HPC-Stack 
+   for validation, and support for this option is being deprecated. 
 
 ----------------
 Common Modules
@@ -70,8 +71,8 @@ The most updated list of common modules can be viewed in ``ufs_common.lua``
 `here <https://github.com/ufs-community/ufs-weather-model/blob/develop/modulefiles/ufs_common.lua>`__.
 
 .. attention::
-   Documentation is available for installing `HPC-Stack <https://hpc-stack.readthedocs.io/en/latest/>`__ 
-   and `spack-stack <https://spack-stack.readthedocs.io/en/latest/>`__, respectively. 
+   Documentation is available for installing `spack-stack <https://spack-stack.readthedocs.io/en/latest/>`__
+   and `HPC-Stack <https://hpc-stack.readthedocs.io/en/latest/>`__, respectively. 
    One of these software stacks (or the libraries they contain) must be installed before running the UFS Weather Model. 
 
 .. _GetData:

doc/UsersGuide/source/CodeOverview.rst

@@ -19,16 +19,16 @@ Four levels of support have been defined for :term:`UFS` applications, and the U
 
 Level 1 Systems
 ==================
-Preconfigured (Level 1) systems for the UFS WM already have the required external libraries available in a central location via :term:`HPC-Stack` or :term:`spack-stack`. The WM is expected to build and run out-of-the-box on these systems, and users can download the WM code without first installing prerequisite software. Additionally, regression test data is already available on these systems. In general, users must have access to these Level 1 systems in order to use them.
+Preconfigured (Level 1) systems for the UFS WM already have the required external libraries available in a central location via :term:`spack-stack`. The WM is expected to build and run out-of-the-box on these systems, and users can download the WM code without first installing prerequisite software. Additionally, regression test data is already available on these systems. In general, users must have access to these Level 1 systems in order to use them.
 
 Currently, Level 1 (or Tier-1) platforms for regression testing are: 
 
    * WCOSS2 (Intel)
-   * Cheyenne (Intel/GNU compilers)
    * Gaea (Intel)
    * Hera (Intel/GNU compilers)
    * Jet (Intel)
    * Orion (Intel)
+   * Hercules (Intel/GNU compilers)
    * AWS Docker container (Intel)
 
 More information is available in the `UFS WM wiki <https://github.com/ufs-community/ufs-weather-model/wiki/Regression-Test-Policy-for-Weather-Model-Platforms-and-Compilers>`__. 

doc/UsersGuide/source/Glossary.rst

@@ -120,11 +120,11 @@ Glossary
 
    NCEPLIBS
       The software libraries created and maintained by :term:`NCEP` that are required for running 
-      :term:`chgres_cube`, the UFS Weather Model, and the :term:`UPP`. They are included in the `HPC-Stack <https://github.com/NOAA-EMC/hpc-stack>`__ and in `spack-stack <https://github.com/NOAA-EMC/spack-stack>`__. 
+      :term:`chgres_cube`, the UFS Weather Model, and the :term:`UPP`. They are included in `spack-stack <https://github.com/NOAA-EMC/spack-stack>`__ and `HPC-Stack <https://github.com/NOAA-EMC/hpc-stack>`__. 
 
    NCEPLIBS-external
       A collection of third-party libraries required to build :term:`NCEPLIBS`, :term:`chgres_cube`, 
-      the UFS Weather Model, and the :term:`UPP`. They are included in the :term:`HPC-Stack` and in :term:`spack-stack`.  
+      the UFS Weather Model, and the :term:`UPP`. They are included in :term:`spack-stack` and :term:`HPC-Stack`.  
 
    NEMS
       The NOAA Environmental Modeling System is a common modeling framework whose purpose is 

doc/UsersGuide/source/conf.py

@@ -41,6 +41,7 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinx_rtd_theme',
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',

driver/UFS.F90

@@ -91,7 +91,7 @@ PROGRAM UFS
 !***  Initialize the ESMF framework.
 !-----------------------------------------------------------------------
 !
-      CALL ESMF_Initialize(configFileName="nems.configure"              & !<-- top level configuration
+      CALL ESMF_Initialize(configFileName="ufs.configure"               & !<-- top level configuration
                           ,defaultCalKind =ESMF_CALKIND_GREGORIAN       & !<-- Set up the default calendar.
                           ,VM             =VM                           & !<-- The ESMF Virtual Machine
                           ,rc             =RC)

driver/UFSDriver.F90

@@ -199,13 +199,13 @@ SUBROUTINE UFSDriver_SS(driver,RC)
       ! create, open, and set the config
       config = ESMF_ConfigCreate(rc=RC)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return
-      call ESMF_ConfigLoadFile(config, "nems.configure", rc=RC)
+      call ESMF_ConfigLoadFile(config, "ufs.configure", rc=RC)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return
       call ESMF_GridCompSet(driver, config=config, rc=RC)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
-      ! Load the required entries from the fd_nems.yaml file
-      call NUOPC_FieldDictionarySetup("fd_nems.yaml", rc=rc)
+      ! Load the required entries from the fd_ufs.yaml file
+      call NUOPC_FieldDictionarySetup("fd_ufs.yaml", rc=rc)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
 !-----------------------------------------------------------------------

modulefiles/ufs_acorn.intel.lua

@@ -2,11 +2,10 @@ help([[
 Load environment to build UFS on Acorn with Intel compiler
 ]])
 
-prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 load("stack-intel")
 load("stack-cray-mpich")
-load("stack-python")
 
 --Avoid production installations; use spack-stack only:
 remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/compiler/intel/19.1.3.304")

modulefiles/ufs_common.lua

@@ -16,12 +16,12 @@ local ufs_modules = {
   {["crtm"]            = "2.4.0"},
   {["g2"]              = "3.4.5"},
   {["g2tmpl"]          = "1.10.2"},
-  {["ip"]              = "3.3.3"},
+  {["ip"]              = "4.3.0"},
   {["sp"]              = "2.3.3"},
-  {["w3emc"]           = "2.9.2"},
+  {["w3emc"]           = "2.10.0"},
   {["gftl-shared"]     = "1.5.0"},
   {["mapl"]            = "2.35.2-esmf-8.4.2"},
-  {["scotch"]          = "7.0.3"},
+  {["scotch"]          = "7.0.4"},
 }
 
 for i = 1, #ufs_modules do

modulefiles/ufs_gaea.intel.lua

@@ -1,5 +1,5 @@
 help([[
-  This module loads libraries required for building and running UFS Weather Model 
+  This module loads libraries required for building and running UFS Weather Model
   on the NOAA RDHPC machine Gaea using Intel-2022.0.2
 ]])
 
@@ -10,18 +10,14 @@ unload("cray-mpich")
 unload("cray-python")
 unload("darshan")
 
-prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c4/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20"
 load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
@@ -30,6 +26,9 @@ load("alps")
 
 load("ufs_common")
 
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
 setenv("CC","cc")
 setenv("CXX","CC")
 setenv("FC","ftn")

modulefiles/ufs_hera.gnu.lua

@@ -2,7 +2,8 @@ help([[
 loads UFS Model prerequisites for Hera/GNU
 ]])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
+-- for openmpi, need
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
 stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0"
@@ -11,14 +12,14 @@ load(pathJoin("stack-gcc", stack_gnu_ver))
 stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
 load(pathJoin("stack-openmpi", stack_openmpi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
 load("ufs_common")
 
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
 setenv("CC", "mpicc")
 setenv("CXX", "mpic++")
 setenv("FC", "mpif90")

modulefiles/ufs_hera.intel.lua

@@ -2,23 +2,22 @@ help([[
 loads UFS Model prerequisites for Hera/Intel
 ]])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
 load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
 load("ufs_common")
 
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
 setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")

modulefiles/ufs_hercules.gnu.lua

@@ -2,49 +2,23 @@ help([[
 loads UFS Model prerequisites for Hercules/GNU
 ]])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230825/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env-mvap2/install/modulefiles/Core")
+-- for mvapich2, need:
 prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles")
 
 stack_gnu_ver=os.getenv("stack_gnu_ver") or "11.3.1"
 load(pathJoin("stack-gcc", stack_gnu_ver))
 
-stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
-load(pathJoin("stack-openmpi", stack_openmpi_ver))
-
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.14"
-load(pathJoin("stack-python", stack_python_ver))
+stack_mvapich2_ver=os.getenv("stack_mvapich2_ver") or "2.3.7"
+load(pathJoin("stack-mvapich2", stack_mvapich2_ver))
 
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
-local ufs_modules = {
-  {["jasper"]          = "2.0.32"},
-  {["zlib"]            = "1.2.13"},
-  {["libpng"]          = "1.6.37"},
-  {["hdf5"]            = "1.14.0"},
-  {["netcdf-c"]        = "4.9.2"},
-  {["netcdf-fortran"]  = "4.6.0"},
-  {["parallelio"]      = "2.5.10"},
-  {["esmf"]            = "8.4.2"},
-  {["fms"]             = "2023.01"},
-  {["bacio"]           = "2.4.1"},
-  {["crtm"]            = "2.4.0"},
-  {["g2"]              = "3.4.5"},
-  {["g2tmpl"]          = "1.10.2"},
-  {["ip"]              = "3.3.3"},
-  {["sp"]              = "2.3.3"},
-  {["w3emc"]           = "2.9.2"},
-  {["gftl-shared"]     = "1.5.0"},
-  {["mapl"]            = "2.35.2-esmf-8.4.2"},
-  {["scotch"]          = "7.0.4"},
-}
-
-for i = 1, #ufs_modules do
-  for name, default_version in pairs(ufs_modules[i]) do
-    local env_version_name = string.gsub(name, "-", "_") .. "_ver"
-    load(pathJoin(name, os.getenv(env_version_name) or default_version))
-  end
-end
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
 
 setenv("CC", "mpicc")
 setenv("CXX", "mpic++")