From 1603cf0f865abc71ed58451e31f9d93f845336da Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 13 Jul 2023 12:05:11 +0000 Subject: [PATCH 01/30] Testing environment for spack-stack on Hera #589 --- modulefiles/gsi_common.lua | 6 ++--- modulefiles/gsi_hera.intel.lua | 41 +++++++++++++++++----------------- src/gsi/CMakeLists.txt | 2 +- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index c54f6ddb92..8ab7985d4b 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -2,9 +2,9 @@ help([[ Load common modules to build GSI on all machines ]]) -local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" +local netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" -local bufr_ver=os.getenv("bufr_ver") or "11.7.0" +local bufr_ver=os.getenv("bufr_ver") or "12.0.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" local sp_ver=os.getenv("sp_ver") or "2.3.3" @@ -30,4 +30,4 @@ load(pathJoin("nemsio", nemsio_ver)) load(pathJoin("wrf_io", wrf_io_ver)) load(pathJoin("ncio", ncio_ver)) load(pathJoin("crtm", crtm_ver)) -load(pathJoin("ncdiag",ncdiag_ver)) +load(pathJoin("gsi-ncdiag",ncdiag_ver)) diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index 866af02d50..64d652be78 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -1,26 +1,27 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/miniconda3/modulefiles") -miniconda3_ver=os.getenv("miniconda3_ver") or "4.12.0" -load(pathJoin("miniconda3", miniconda3_ver)) - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") - -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -local cmake_ver=os.getenv("cmake_ver") or "3.20.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" - -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) -load(pathJoin("cmake", cmake_ver)) - -load("gsi_common") - -load(pathJoin("prod_util", prod_util_ver)) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/stmp4/Cameron.Book/dave_gsi/envs/gsi_test/install/modulefiles/Core") + +local spack_python_ver=os.getenv("spack_python_ver") or "3.9.12" +local spack_intel_ver=os.getenv("spack_intel_ver") or "2021.5.0" +local spack_impi_ver=os.getenv("spack_impi_ver") or "2021.5.1" +local spack_python_ver=os.getenv("spack_impi_ver") or "3.9.12" +local ecflow_ver=os.getenv("ecflow_ver") or "5.5.3" +local mysql_ver=os.getenv("mysql_ver") or "8.0.31" + +load(pathJoin("miniconda", spack_python_ver)) +load(pathJoin("ecflow", ecflow_ver)) +load(pathJoin("mysql", mysql_ver)) +load(pathJoin("stack-intel", spack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", spack_impi_ver)) +load(pathJoin("stack-python", spack_python_ver)) + +load("ufs-srw-app-env/unified-dev") +load("gsi-ncdiag/1.1.1") +load("ncio/1.1.2") +load("bufr/12.0.0") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") diff --git a/src/gsi/CMakeLists.txt b/src/gsi/CMakeLists.txt index af94224c05..603a0b5f4e 100644 --- a/src/gsi/CMakeLists.txt +++ b/src/gsi/CMakeLists.txt @@ -146,7 +146,7 @@ target_link_libraries(gsi_fortran_obj PUBLIC nemsio::nemsio) target_link_libraries(gsi_fortran_obj PUBLIC ncio::ncio) target_link_libraries(gsi_fortran_obj PUBLIC w3emc::w3emc_d) target_link_libraries(gsi_fortran_obj PUBLIC sp::sp_d) -target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_d) +target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_4) target_link_libraries(gsi_fortran_obj PUBLIC crtm::crtm) if(GSI_MODE MATCHES "Regional") target_link_libraries(gsi_fortran_obj PUBLIC wrf_io::wrf_io) From 27c099f12be5be7a5beb299a482f80e885a19260 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 31 Jul 2023 14:57:27 +0000 Subject: [PATCH 02/30] Updated modulefiles to point to spack-stack #474 #571 #589 --- modulefiles/gsi_cheyenne.gnu.lua | 39 +++++++++++++++++------------- modulefiles/gsi_cheyenne.intel.lua | 32 +++++++++++++----------- modulefiles/gsi_common.lua | 12 +++++---- modulefiles/gsi_gaea.lua | 28 ++++++++++++--------- modulefiles/gsi_hera.gnu.lua | 26 +++++++++++--------- modulefiles/gsi_hera.intel.lua | 27 ++++++++------------- modulefiles/gsi_jet.lua | 25 +++++++------------ modulefiles/gsi_orion.lua | 24 +++++++----------- modulefiles/gsi_wcoss2.lua | 26 +++++++++----------- 9 files changed, 118 insertions(+), 121 deletions(-) diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua index 43e6aaf02c..8f3292cfe3 100644 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ b/modulefiles/gsi_cheyenne.gnu.lua @@ -1,30 +1,35 @@ help([[ ]]) -load("cmake/3.22.0") -load("python/3.7.9") -load("ncarenv/1.3") -load("gnu/11.2.0") -load("mpt/2.25") -load("ncarcompilers/0.5.0") -unload("intel") -unload("netcdf") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack") +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_fix/fix") -load("hpc/1.2.0") -load("hpc-gnu/11.2.0") -load("hpc-mpt/2.25") +unload("ncarenv/1.3") +unload("intel/19.1.1") +unload("ncarcompilers/0.5.0") +unload("mpt/2.25") +unload("netcdf/4.8.1") -load("gsi_common") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") + +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_gnu_ver=os.getenv("stack_gnu_ver") or "10.1.0" +local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.1" +local cmake_ver=os.getenv("cmake_ver") or "3.22.0" +load(pathJoin("stack-gcc", stack_gnu_ver)) +load(pathJoin("stack-openmpi", stack_openmpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_fix/fix") +load("gsi_common") + +pushenv("CFLAGS", "-xHOST") +pushenv("FFLAGS", "-xHOST") -pushenv("CC", "mpicc") -pushenv("FC", "mpif90") -pushenv("CXX", "mpicxx") +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") whatis("Description: GSI environment on Cheyenne with GNU Compilers") diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua index 4a3525bca1..611aaea440 100644 --- a/modulefiles/gsi_cheyenne.intel.lua +++ b/modulefiles/gsi_cheyenne.intel.lua @@ -1,26 +1,30 @@ help([[ ]]) -load("cmake/3.22.0") -load("python/3.7.9") -load("ncarenv/1.3") -load("intel/2022.1") -load("mpt/2.25") -load("ncarcompilers/0.5.0") +unload("ncarenv/1.3") +unload("intel/19.1.1") +unload("ncarcompilers/0.5.0") +unload("mpt/2.25") +unload("netcdf/4.8.1") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") -load("hpc/1.2.0") -load("hpc-intel/2022.1") -load("hpc-mpt/2.25") -load("mkl/2022.1") +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "19.1.1.217" +local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2019.7.217" +local cmake_ver=os.getenv("cmake_ver") or "3.22.0" -load("gsi_common") +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-mpi", stack_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) -load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") +load("gsi_common") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") + whatis("Description: GSI environment on Cheyenne with Intel Compilers") diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index 8ab7985d4b..c4779fcf65 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -2,22 +2,24 @@ help([[ Load common modules to build GSI on all machines ]]) -local netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" +local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" -local bufr_ver=os.getenv("bufr_ver") or "12.0.0" +local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" local sp_ver=os.getenv("sp_ver") or "2.3.3" local ip_ver=os.getenv("ip_ver") or "3.3.3" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" -local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +local nemsio_ver=os.getenv("nemsio_ver") or "2.5.2" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" local crtm_ver=os.getenv("crtm_ver") or "2.4.0" local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" -load(pathJoin("netcdf", netcdf_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) load(pathJoin("bufr", bufr_ver)) load(pathJoin("bacio", bacio_ver)) @@ -27,7 +29,7 @@ load(pathJoin("ip", ip_ver)) load(pathJoin("sigio", sigio_ver)) load(pathJoin("sfcio", sfcio_ver)) load(pathJoin("nemsio", nemsio_ver)) -load(pathJoin("wrf_io", wrf_io_ver)) +load(pathJoin("wrf-io", wrf_io_ver)) load(pathJoin("ncio", ncio_ver)) load(pathJoin("crtm", crtm_ver)) load(pathJoin("gsi-ncdiag",ncdiag_ver)) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index f76c8f3ad9..a41940bfb7 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -1,33 +1,39 @@ help([[ ]]) -load("cmake/3.20.1") +unload("intel") +unload("cray-mpich") +unload("cray-python") +unload("darshan") -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles") -load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) -load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) -load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) -load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load("gsi_common") +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("prod_util", prod_util_ver)) +load("gsi_common") -- Needed at runtime: load("alps") local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +M pushenv("MKLROOT", MKLROOT) pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") setenv("CC","cc") -setenv("FC","ftn") setenv("CXX","CC") +setenv("FC","ftn") pushenv("CRAYPE_LINK_TYPE","dynamic") whatis("Description: GSI environment on Gaea with Intel Compilers") diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index c309e67fe0..a52f8dbe92 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -1,26 +1,28 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local gnu_ver=os.getenv("gnu_ver") or "9.2.0" -local hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2" -local hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2" -local cmake_ver=os.getenv("cmake_ver") or "3.20.1" + +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" +local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -local openblas_ver=os.getenv("openblas_ver") or "0.3.23" +local openblas_ver=os.getenv("openblas_ver") or "0.3.19" -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("gnu", gnu_ver)) -load(pathJoin("hpc-gnu", hpc_gnu_ver)) -load(pathJoin("hpc-mpich", hpc_mpich_ver)) +load(pathJoin("stack-gcc", stack_gnu_ver)) +load(pathJoin("stack-openmpi", stack_openmpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("prod-util", prod_util_ver)) load(pathJoin("openblas", openblas_ver)) +pushenv("CFLAGS", "-xHOST") +pushenv("FFLAGS", "-xHOST") pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601") diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index 64d652be78..eca55596d2 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -1,27 +1,20 @@ help([[ ]]) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/stmp4/Cameron.Book/dave_gsi/envs/gsi_test/install/modulefiles/Core") -local spack_python_ver=os.getenv("spack_python_ver") or "3.9.12" -local spack_intel_ver=os.getenv("spack_intel_ver") or "2021.5.0" -local spack_impi_ver=os.getenv("spack_impi_ver") or "2021.5.1" -local spack_python_ver=os.getenv("spack_impi_ver") or "3.9.12" -local ecflow_ver=os.getenv("ecflow_ver") or "5.5.3" -local mysql_ver=os.getenv("mysql_ver") or "8.0.31" +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("miniconda", spack_python_ver)) -load(pathJoin("ecflow", ecflow_ver)) -load(pathJoin("mysql", mysql_ver)) -load(pathJoin("stack-intel", spack_intel_ver)) -load(pathJoin("stack-intel-oneapi-mpi", spack_impi_ver)) -load(pathJoin("stack-python", spack_python_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) -load("ufs-srw-app-env/unified-dev") -load("gsi-ncdiag/1.1.1") -load("ncio/1.1.2") -load("bufr/12.0.0") +load("gsi_common") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index e2ea2ef1d0..8f7d4d97cb 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -1,31 +1,24 @@ help([[ ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles") -miniconda3_ver=os.getenv("miniconda3_ver") or "4.12.0" -load(pathJoin("miniconda3", miniconda3_ver)) +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -local cmake_ver=os.getenv("cmake_ver") or "3.20.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" - -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod_util", prod_util_ver)) - pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2") pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2") - pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230601") whatis("Description: GSI environment on Jet with Intel Compilers") diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index a7ea874fb2..987531b43c 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -1,27 +1,21 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") -miniconda3_ver=os.getenv("miniconda3_ver") or "4.12.0" -load(pathJoin("miniconda3", miniconda3_ver)) +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles") -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack") +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.7" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -local cmake_ver=os.getenv("cmake_ver") or "3.22.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" - -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod_util", prod_util_ver)) - pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index 1872f89d17..c57a7946c8 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -1,22 +1,20 @@ help([[ ]]) -local PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" -local intel_ver=os.getenv("intel_ver") or "19.1.3.304" -local craype_ver=os.getenv("craype_ver") or "2.7.8" -local cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" -local cmake_ver= os.getenv("cmake_ver") or "3.20.2" -local python_ver=os.getenv("python_ver") or "3.8.6" -local prod_util_ver=os.getenv("prod_util_ver") or "2.0.10" +prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) -load(pathJoin("intel", intel_ver)) -load(pathJoin("craype", craype_ver)) -load(pathJoin("cray-mpich", cray_mpich_ver)) -load(pathJoin("cmake", cmake_ver)) -load(pathJoin("python", python_ver)) +load("stack-intel") +load("stack-cray-mpich") +load("stack-python") -load(pathJoin("prod_util", prod_util_ver)) +--Avoid production installations; use spack-stack only: +remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/compiler/intel/19.1.3.304") +remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.4") +remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7") +remove_path("MODULEPATH", "/apps/prod/lmodules/intel/19.1.3.304") +remove_path("MODULEPATH", "/apps/prod/lmodules/INTEL_cray_mpich/19.1.3.304/cray-mpich/8.1.4") + +load("cmake") load("gsi_common") From b708ffc330a1ad131749aa4f9c16d6ff2e583d40 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 31 Jul 2023 15:56:25 +0000 Subject: [PATCH 03/30] Updated noaa-cloud spack-stack location #589. --- modulefiles/gsi_noaacloud.lua | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 modulefiles/gsi_noaacloud.lua diff --git a/modulefiles/gsi_noaacloud.lua b/modulefiles/gsi_noaacloud.lua new file mode 100644 index 0000000000..bf4af7d180 --- /dev/null +++ b/modulefiles/gsi_noaacloud.lua @@ -0,0 +1,20 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/contrib/EPIC/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/modulefiles/core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.3.0" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("cmake", cmake_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load("gsi_common") + +pushenv("GSI_BINARY_SOURCE_DIR", "/contrib/EPIC/GSI_data/fix/20230601") +whatis("Description: GSI environment on Cloud with Intel Compilers") From e01fc679513a114fc3bce3216cf5ef42885b5815 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 1 Aug 2023 04:06:18 -0600 Subject: [PATCH 04/30] Update Orion stack location for the new role-epic account (#15) * corrections to module-setup.sh * update to Orion software stack in a new role-epic location * updated regression/regression_param.sh srun command for Orion --------- Co-authored-by: Natalie Perlin Co-authored-by: Natalie Perlin From ade69628091540a8ca67f2919d68fcf8fda626d5 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 18 Aug 2023 14:55:05 +0000 Subject: [PATCH 05/30] Revert bufr_4 to bufr_d for bufr/11.7.0. --- src/gsi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gsi/CMakeLists.txt b/src/gsi/CMakeLists.txt index 603a0b5f4e..af94224c05 100644 --- a/src/gsi/CMakeLists.txt +++ b/src/gsi/CMakeLists.txt @@ -146,7 +146,7 @@ target_link_libraries(gsi_fortran_obj PUBLIC nemsio::nemsio) target_link_libraries(gsi_fortran_obj PUBLIC ncio::ncio) target_link_libraries(gsi_fortran_obj PUBLIC w3emc::w3emc_d) target_link_libraries(gsi_fortran_obj PUBLIC sp::sp_d) -target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_4) +target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_d) target_link_libraries(gsi_fortran_obj PUBLIC crtm::crtm) if(GSI_MODE MATCHES "Regional") target_link_libraries(gsi_fortran_obj PUBLIC wrf_io::wrf_io) From 76885f59939bca829ae16a375e7208a551f87c16 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 18 Aug 2023 14:55:51 +0000 Subject: [PATCH 06/30] Prevent undefined error for non-cloud platforms. --- ush/detect_machine.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 6f0673ce29..6874809099 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -37,6 +37,15 @@ case $(hostname -f) in discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 esac +case $(echo ${PW_CSP:-}) in + + aws) MACHINE_ID=aws ;; ### parallelworks aws + google) MACHINE_ID=gcp ;; ### parallelworks gcp + azure) MACHINE_ID=azure ;; ### parallelworks azure + +esac +[[ ${MACHINE_ID} =~ "aws" || ${MACHINE_ID} =~ "gcp" || ${MACHINE_ID} =~ "azure" ]] && MACHINE_ID=noaacloud + # Overwrite auto-detect with MACHINE if set MACHINE_ID=${MACHINE:-${MACHINE_ID}} From 0fd3489bcd809db3352900f2763804c6b60e2405 Mon Sep 17 00:00:00 2001 From: Russ-Treadon-NOAA Date: Mon, 21 Aug 2023 14:38:49 +0000 Subject: [PATCH 07/30] modifications for acorn (wcoss2 test system) build using intel19 spack-stack (#447) --- modulefiles/gsi_common_wcoss2.lua | 35 +++++++++++++++++++++++++++++++ modulefiles/gsi_wcoss2.lua | 27 +++++++++++------------- ush/sub_wcoss2 | 1 + 3 files changed, 48 insertions(+), 15 deletions(-) create mode 100644 modulefiles/gsi_common_wcoss2.lua diff --git a/modulefiles/gsi_common_wcoss2.lua b/modulefiles/gsi_common_wcoss2.lua new file mode 100644 index 0000000000..12d4efd5c6 --- /dev/null +++ b/modulefiles/gsi_common_wcoss2.lua @@ -0,0 +1,35 @@ +help([[ +Load common modules to build GSI on all machines +]]) + +local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" + +local bufr_ver=os.getenv("bufr_ver") or "11.7.0" +local bacio_ver=os.getenv("bacio_ver") or "2.4.1" +local w3emc_ver=os.getenv("w3emc_ver") or "2.9.1" +local sp_ver=os.getenv("sp_ver") or "2.3.3" +local ip_ver=os.getenv("ip_ver") or "3.3.3" +local sigio_ver=os.getenv("sigio_ver") or "2.3.2" +local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +local nemsio_ver=os.getenv("nemsio_ver") or "2.5.2" +local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" +local ncio_ver=os.getenv("ncio_ver") or "1.1.2" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" + +load("netcdf-c") +load("netcdf-fortran") + +load("bufr/11.7.0") +load("bacio") +load("w3emc") +load("sp") +load("ip/3.3.3") +load("sigio") +load("sfcio") +load("nemsio") +load("wrf-io") +load("ncio") +load("crtm") +load(pathJoin("gsi-ncdiag",ncdiag_ver)) + diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index c57a7946c8..d3d72a284f 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -1,23 +1,20 @@ help([[ ]]) -prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +load("PrgEnv-intel") +load("intel") +load("craype") +load("cray-mpich") +load("cmake") +prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/Mark.Potts/spack-stack/spack-stack-1.4.1/envs/unified-dev-19/install/modulefiles/Core") load("stack-intel") load("stack-cray-mpich") -load("stack-python") - ---Avoid production installations; use spack-stack only: -remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/compiler/intel/19.1.3.304") -remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.4") -remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7") -remove_path("MODULEPATH", "/apps/prod/lmodules/intel/19.1.3.304") -remove_path("MODULEPATH", "/apps/prod/lmodules/INTEL_cray_mpich/19.1.3.304/cray-mpich/8.1.4") - -load("cmake") - -load("gsi_common") +load("prod-util") +load("gsi_common_wcoss2") +setenv("CC", "cc") +setenv("CXX", "CC") +setenv("FC", "ftn") pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230601") - -whatis("Description: GSI environment on WCOSS2") +whatis("Description: GSI environment on Hera with Intel Compilers") diff --git a/ush/sub_wcoss2 b/ush/sub_wcoss2 index f2df099f23..f41fd305e9 100755 --- a/ush/sub_wcoss2 +++ b/ush/sub_wcoss2 @@ -115,6 +115,7 @@ echo "#PBS -j oe" >> $cfile echo "#PBS -A "$accnt >> $cfile echo "" >> $cfile +echo "set -x" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile echo "export ntasks=$(( $nodes * $procs ))" >> $cfile echo "export ppn=$procs" >> $cfile From aff4365270746ca5e4d9594da2aade4689e65617 Mon Sep 17 00:00:00 2001 From: Russ-Treadon-NOAA Date: Mon, 21 Aug 2023 18:02:23 +0000 Subject: [PATCH 08/30] add version numbers to module loads in gsi_common_wcoss2.lua (#447) --- modulefiles/gsi_common_wcoss2.lua | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/modulefiles/gsi_common_wcoss2.lua b/modulefiles/gsi_common_wcoss2.lua index 12d4efd5c6..1b2a88db92 100644 --- a/modulefiles/gsi_common_wcoss2.lua +++ b/modulefiles/gsi_common_wcoss2.lua @@ -2,11 +2,12 @@ help([[ Load common modules to build GSI on all machines ]]) -local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" +local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" -local w3emc_ver=os.getenv("w3emc_ver") or "2.9.1" +local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" local sp_ver=os.getenv("sp_ver") or "2.3.3" local ip_ver=os.getenv("ip_ver") or "3.3.3" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" @@ -17,19 +18,20 @@ local ncio_ver=os.getenv("ncio_ver") or "1.1.2" local crtm_ver=os.getenv("crtm_ver") or "2.4.0" local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" -load("netcdf-c") -load("netcdf-fortran") +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) -load("bufr/11.7.0") -load("bacio") -load("w3emc") -load("sp") -load("ip/3.3.3") -load("sigio") -load("sfcio") -load("nemsio") -load("wrf-io") -load("ncio") -load("crtm") + +load(pathJoin("bufr", bufr_ver)) +load(pathJoin("bacio", bacio_ver)) +load(pathJoin("w3emc", w3emc_ver)) +load(pathJoin("sp", sp_ver)) +load(pathJoin("ip", ip_ver)) +load(pathJoin("sigio", sigio_ver)) +load(pathJoin("sfcio", sfcio_ver)) +load(pathJoin("nemsio", nemsio_ver)) +load(pathJoin("wrf_io", wrf_io_ver)) +load(pathJoin("ncio", ncio_ver)) +load(pathJoin("crtm", crtm_ver)) load(pathJoin("gsi-ncdiag",ncdiag_ver)) From a9a910894e2a51a9a02dfd19b4c5462b4ca26345 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Mon, 11 Sep 2023 17:45:35 +0000 Subject: [PATCH 09/30] revert gsi_wcoss2.lua to develop (#447) --- modulefiles/gsi_wcoss2.lua | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index d3d72a284f..1872f89d17 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -1,20 +1,25 @@ help([[ ]]) -load("PrgEnv-intel") -load("intel") -load("craype") -load("cray-mpich") -load("cmake") +local PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" +local intel_ver=os.getenv("intel_ver") or "19.1.3.304" +local craype_ver=os.getenv("craype_ver") or "2.7.8" +local cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" +local cmake_ver= os.getenv("cmake_ver") or "3.20.2" +local python_ver=os.getenv("python_ver") or "3.8.6" +local prod_util_ver=os.getenv("prod_util_ver") or "2.0.10" -prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/Mark.Potts/spack-stack/spack-stack-1.4.1/envs/unified-dev-19/install/modulefiles/Core") -load("stack-intel") -load("stack-cray-mpich") -load("prod-util") -load("gsi_common_wcoss2") +load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) +load(pathJoin("intel", intel_ver)) +load(pathJoin("craype", craype_ver)) +load(pathJoin("cray-mpich", cray_mpich_ver)) +load(pathJoin("cmake", cmake_ver)) +load(pathJoin("python", python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) + +load("gsi_common") -setenv("CC", "cc") -setenv("CXX", "CC") -setenv("FC", "ftn") pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230601") -whatis("Description: GSI environment on Hera with Intel Compilers") + +whatis("Description: GSI environment on WCOSS2") From 76fed6aecc9463187f6ebc0695c34ba5eb35fc0a Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 15 Sep 2023 12:58:20 +0000 Subject: [PATCH 10/30] Removed unused gsi_common_wcoss2 modulefile. --- modulefiles/gsi_common_wcoss2.lua | 37 ------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 modulefiles/gsi_common_wcoss2.lua diff --git a/modulefiles/gsi_common_wcoss2.lua b/modulefiles/gsi_common_wcoss2.lua deleted file mode 100644 index 1b2a88db92..0000000000 --- a/modulefiles/gsi_common_wcoss2.lua +++ /dev/null @@ -1,37 +0,0 @@ -help([[ -Load common modules to build GSI on all machines -]]) - -local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" - -local bufr_ver=os.getenv("bufr_ver") or "11.7.0" -local bacio_ver=os.getenv("bacio_ver") or "2.4.1" -local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" -local sp_ver=os.getenv("sp_ver") or "2.3.3" -local ip_ver=os.getenv("ip_ver") or "3.3.3" -local sigio_ver=os.getenv("sigio_ver") or "2.3.2" -local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" -local nemsio_ver=os.getenv("nemsio_ver") or "2.5.2" -local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" -local ncio_ver=os.getenv("ncio_ver") or "1.1.2" -local crtm_ver=os.getenv("crtm_ver") or "2.4.0" -local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" - -load(pathJoin("netcdf-c", netcdf_c_ver)) -load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - - -load(pathJoin("bufr", bufr_ver)) -load(pathJoin("bacio", bacio_ver)) -load(pathJoin("w3emc", w3emc_ver)) -load(pathJoin("sp", sp_ver)) -load(pathJoin("ip", ip_ver)) -load(pathJoin("sigio", sigio_ver)) -load(pathJoin("sfcio", sfcio_ver)) -load(pathJoin("nemsio", nemsio_ver)) -load(pathJoin("wrf_io", wrf_io_ver)) -load(pathJoin("ncio", ncio_ver)) -load(pathJoin("crtm", crtm_ver)) -load(pathJoin("gsi-ncdiag",ncdiag_ver)) - From e56757c0f027f89370c91612e8d6c7e9566066c6 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 15 Sep 2023 13:06:05 +0000 Subject: [PATCH 11/30] Cleanup sub_wcoss2. --- ush/sub_wcoss2 | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/sub_wcoss2 b/ush/sub_wcoss2 index f41fd305e9..f2df099f23 100755 --- a/ush/sub_wcoss2 +++ b/ush/sub_wcoss2 @@ -115,7 +115,6 @@ echo "#PBS -j oe" >> $cfile echo "#PBS -A "$accnt >> $cfile echo "" >> $cfile -echo "set -x" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile echo "export ntasks=$(( $nodes * $procs ))" >> $cfile echo "export ppn=$procs" >> $cfile From 82bb3cd4fe68731a8f674301e83a25773c3aa261 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 18 Sep 2023 19:22:58 +0000 Subject: [PATCH 12/30] Cleaned up modulefiles. #589 --- modulefiles/gsi_cheyenne.gnu.lua | 3 --- modulefiles/gsi_gaea.lua | 6 ++++-- modulefiles/gsi_hera.gnu.lua | 3 --- modulefiles/gsi_hera.intel.lua | 2 ++ modulefiles/gsi_jet.lua | 2 ++ modulefiles/gsi_noaacloud.lua | 2 ++ modulefiles/gsi_orion.lua | 2 ++ 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua index 8f3292cfe3..1d903082a8 100644 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ b/modulefiles/gsi_cheyenne.gnu.lua @@ -2,8 +2,6 @@ help([[ ]]) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_fix/fix") - unload("ncarenv/1.3") unload("intel/19.1.1") unload("ncarcompilers/0.5.0") @@ -29,7 +27,6 @@ load("gsi_common") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") - pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") whatis("Description: GSI environment on Cheyenne with GNU Compilers") diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index 752d98eaa0..ef6b9ddba7 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -21,19 +21,21 @@ load(pathJoin("cmake", cmake_ver)) load("gsi_common") +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("prod-util", prod_util_ver)) + -- Needed at runtime: load("alps") local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) -M pushenv("MKLROOT", MKLROOT) pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230911") setenv("CC","cc") -setenv("CXX","CC") setenv("FC","ftn") +setenv("CXX","CC") pushenv("CRAYPE_LINK_TYPE","dynamic") whatis("Description: GSI environment on Gaea with Intel Compilers") diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index 0c41f9411d..7dcbc2e0c8 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -4,7 +4,6 @@ help([[ prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") - local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" @@ -21,8 +20,6 @@ load("gsi_common") load(pathJoin("prod-util", prod_util_ver)) load(pathJoin("openblas", openblas_ver)) -pushenv("CFLAGS", "-xHOST") -pushenv("FFLAGS", "-xHOST") pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230911") diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index b4593a2649..bdc82db365 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -8,6 +8,7 @@ local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) @@ -15,6 +16,7 @@ load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") +load(pathJoin("prod-util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index a614b46d03..967a03fd0d 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -8,6 +8,7 @@ local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) @@ -15,6 +16,7 @@ load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") +load(pathJoin("prod-util", prod_util_ver)) pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2") pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2") diff --git a/modulefiles/gsi_noaacloud.lua b/modulefiles/gsi_noaacloud.lua index bf4af7d180..7b64da11e2 100644 --- a/modulefiles/gsi_noaacloud.lua +++ b/modulefiles/gsi_noaacloud.lua @@ -8,6 +8,7 @@ local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.3.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local prod_util_ver=os.getenv("stack_python_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) @@ -15,6 +16,7 @@ load(pathJoin("cmake", cmake_ver)) load(pathJoin("stack-python", stack_python_ver)) load("gsi_common") +load(pathJoin("prod-util", prod_util_ver)) pushenv("GSI_BINARY_SOURCE_DIR", "/contrib/EPIC/GSI_data/fix/20230601") whatis("Description: GSI environment on Cloud with Intel Compilers") diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index 71a4e89b77..1fa2eb24f1 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -8,6 +8,7 @@ local stack_python_ver=os.getenv("stack_python_ver") or "3.9.7" local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) @@ -15,6 +16,7 @@ load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") +load(pathJoin("prod-util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") From 3e7ec558d9b752df1c93e64a0e50dbaa45c71fcb Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 19 Sep 2023 13:23:21 -0500 Subject: [PATCH 13/30] Move noaa-cloud to a different branch. (noaa-cloud) --- modulefiles/gsi_noaacloud.lua | 22 ---------------------- ush/detect_machine.sh | 9 --------- 2 files changed, 31 deletions(-) delete mode 100644 modulefiles/gsi_noaacloud.lua diff --git a/modulefiles/gsi_noaacloud.lua b/modulefiles/gsi_noaacloud.lua deleted file mode 100644 index 7b64da11e2..0000000000 --- a/modulefiles/gsi_noaacloud.lua +++ /dev/null @@ -1,22 +0,0 @@ -help([[ -]]) - -prepend_path("MODULEPATH", "/contrib/EPIC/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/contrib/spack-stack/modulefiles/core") - -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0" -local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.3.0" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -local prod_util_ver=os.getenv("stack_python_ver") or "1.2.2" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -load(pathJoin("cmake", cmake_ver)) -load(pathJoin("stack-python", stack_python_ver)) - -load("gsi_common") -load(pathJoin("prod-util", prod_util_ver)) - -pushenv("GSI_BINARY_SOURCE_DIR", "/contrib/EPIC/GSI_data/fix/20230601") -whatis("Description: GSI environment on Cloud with Intel Compilers") diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 6874809099..6f0673ce29 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -37,15 +37,6 @@ case $(hostname -f) in discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 esac -case $(echo ${PW_CSP:-}) in - - aws) MACHINE_ID=aws ;; ### parallelworks aws - google) MACHINE_ID=gcp ;; ### parallelworks gcp - azure) MACHINE_ID=azure ;; ### parallelworks azure - -esac -[[ ${MACHINE_ID} =~ "aws" || ${MACHINE_ID} =~ "gcp" || ${MACHINE_ID} =~ "azure" ]] && MACHINE_ID=noaacloud - # Overwrite auto-detect with MACHINE if set MACHINE_ID=${MACHINE:-${MACHINE_ID}} From 9966313c8e678ccc3a45d72a6ed03a06ab4777c9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 19 Sep 2023 18:26:38 +0000 Subject: [PATCH 14/30] Update modulefile for S4 to use spack-stack. --- modulefiles/gsi_s4.lua | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua index 03c21e708d..7480eb210f 100644 --- a/modulefiles/gsi_s4.lua +++ b/modulefiles/gsi_s4.lua @@ -1,24 +1,22 @@ help([[ ]]) -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1" -local miniconda_ver=os.getenv("miniconda_ver") or "3.8-s4" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" - -prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/modulefiles") -load("license_intel/S4") -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("miniconda", miniconda_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) load("gsi_common") - -load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("prod-util", prod_util_ver)) pushenv("CFLAGS", "-march=ivybridge") pushenv("FFLAGS", "-march=ivybridge") From b5e2f2b2638740ab1e23fcbc3f742485c0a475cc Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 21 Sep 2023 13:02:27 +0000 Subject: [PATCH 15/30] Update CI libraries for spack-stack #589 --- ci/spack.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index 0fc66547e5..deacdff0b5 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -6,8 +6,8 @@ spack: - intel - gcc@10:10 specs: - - netcdf-c@4.7.4 - - netcdf-fortran@4.5.3 + - netcdf-c@4.9.2 + - netcdf-fortran@4.6.0 - bufr@11.7.0 - bacio@2.4.1 - w3emc@2.9.2 From dcbb77f6f3c1297b21f3d32ca09224c357b54f88 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 17 Oct 2023 17:38:16 +0000 Subject: [PATCH 16/30] Free up space on github-cli spinup. --- .github/workflows/intel.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index d48c00a21e..b30d24c1f8 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -27,6 +27,15 @@ jobs: runs-on: ubuntu-latest steps: + # Free up disk space + - name: free-disk-spack + run: | + df -h + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + docker rmi $(docker image ls -aq) + # Checkout the GSI to get the ci/spack.yaml file - name: checkout-gsi if: steps.cache-env.outputs.cache-hit != 'true' From 382914b063ee4336a4e087724ddb6f4022f29a29 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 8 Nov 2023 07:11:44 -0600 Subject: [PATCH 17/30] Update Orion modulefiles to spack-stack/1.5.1. --- modulefiles/gsi_common.lua | 8 ++++---- modulefiles/gsi_orion.lua | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index c4779fcf65..d3365a98dc 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -7,16 +7,16 @@ local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" -local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" local sp_ver=os.getenv("sp_ver") or "2.3.3" -local ip_ver=os.getenv("ip_ver") or "3.3.3" +local ip_ver=os.getenv("ip_ver") or "4.3.0" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" -local nemsio_ver=os.getenv("nemsio_ver") or "2.5.2" +local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" local crtm_ver=os.getenv("crtm_ver") or "2.4.0" -local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" +local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2" load(pathJoin("netcdf-c", netcdf_c_ver)) load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index 1fa2eb24f1..80ec342c93 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -1,10 +1,9 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.7" +local stack_python_ver=os.getenv("python_ver") or "3.10.8" local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" @@ -12,11 +11,11 @@ local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod-util", prod_util_ver)) +load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") From 21a9a88835a4f9b78a7fc190e49a10923241bca9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 9 Nov 2023 13:41:47 -0600 Subject: [PATCH 18/30] Add initial Hercules support. #574 --- modulefiles/gsi_hercules.lua | 26 +++++ regression/regression_param.sh | 33 ++++++- regression/regression_var.sh | 19 +++- ush/build.sh | 2 +- ush/detect_machine.sh | 2 + ush/module-setup.sh | 7 ++ ush/sub_hercules | 170 +++++++++++++++++++++++++++++++++ 7 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 modulefiles/gsi_hercules.lua create mode 100755 ush/sub_hercules diff --git a/modulefiles/gsi_hercules.lua b/modulefiles/gsi_hercules.lua new file mode 100644 index 0000000000..bf29bc21db --- /dev/null +++ b/modulefiles/gsi_hercules.lua @@ -0,0 +1,26 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gsi_common") +load(pathJoin("prod_util", prod_util_ver)) +load("intel-oneapi-mkl/2022.2.1") + +pushenv("CFLAGS", "-xHOST") +pushenv("FFLAGS", "-xHOST") + +pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230911") + +whatis("Description: GSI environment on Hercules with Intel Compilers") diff --git a/regression/regression_param.sh b/regression/regression_param.sh index ea27521251..5d4cd5dfda 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -9,10 +9,15 @@ case $machine in memnode=96 numcore=40 ;; - Orion) + Orion) sub_cmd="sub_orion" memnode=192 numcore=40 + ;; + Hercules) + sub_cmd="sub_hercules" + memnode=512 + numcore=40 ;; Jet) sub_cmd="sub_jet" @@ -59,6 +64,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2" @@ -92,6 +100,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" @@ -137,6 +148,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -170,6 +184,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" @@ -200,6 +217,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" @@ -230,6 +250,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" @@ -290,6 +313,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" @@ -349,7 +375,10 @@ if [[ "$machine" = "Hera" ]]; then export APRUN="srun" elif [[ "$machine" = "Orion" ]]; then export OMP_STACKSIZE=2048M - export APRUN="srun -n \$ntasks" + export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" +elif [[ "$machine" = "Hercules" ]]; then + export OMP_STACKSIZE=2048M + export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" elif [[ "$machine" = "Jet" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 3176372a3b..234e698795 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -42,8 +42,10 @@ elif [[ -d /sw/gaea ]]; then # Gaea export machine="Gaea" elif [[ -d /data/prod ]]; then # S4 export machine="S4" -elif [[ -d /work ]]; then # Orion +elif [[ -d /work && $(hostname -d) =~ "Orion" ]]; then # Orion export machine="Orion" +elif [[ -d /work && $(hostname -d) =~ "Hercules" ]]; then # Hercules + export machine="Hercules" elif [[ -d /lfs/h2 ]]; then # wcoss2 export machine="wcoss2" fi @@ -98,18 +100,25 @@ case $machine in export check_resource="no" export accnt="${accnt:-GFS-DEV}" ;; - Orion) + Orion | Hercules) export local_or_default="${local_or_default:-/work/noaa/da/$LOGNAME}" if [ -d $local_or_default ]; then - export noscrub="$local_or_default/noscrub" + export noscrub="$local_or_default/noscrub" elif [ -d /work/noaa/global/$LOGNAME ]; then - export noscrub="/work/noaa/global/$LOGNAME/noscrub" + export noscrub="/work/noaa/global/$LOGNAME/noscrub" fi export queue="${queue:-batch}" + + if [[ "${machine}" == "Orion" ]]; then + export partition="${partition:-orion}" + else + export partition="${partition:-hercules}" + fi + export group="${group:-global}" if [[ "$cmaketest" = "false" ]]; then - export basedir="/work/noaa/da/$LOGNAME/gsi" + export basedir="/work/noaa/da/$LOGNAME/gsi" fi export ptmp="${ptmp:-/work/noaa/stmp/$LOGNAME/$ptmpName}" diff --git a/ush/build.sh b/ush/build.sh index 9a280c4e55..f0cab6ffe5 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -29,7 +29,7 @@ module list set -x # Set CONTROLPATH variable to user develop installation -CONTROLPATH="$DIR_ROOT/../develop/install/bin" +CONTROLPATH="$DIR_ROOT/install/bin" # Collect BUILD Options CMAKE_OPTS+=" -DCMAKE_BUILD_TYPE=$BUILD_TYPE" diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 6f0673ce29..ac6c7f58d1 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -25,6 +25,8 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index ab92477a56..192a21509c 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -15,6 +15,13 @@ elif [[ $MACHINE_ID = hera* ]] ; then fi module purge +elif [[ $MACHINE_ID = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = orion* ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/sub_hercules b/ush/sub_hercules new file mode 100755 index 0000000000..de5a415603 --- /dev/null +++ b/ush/sub_hercules @@ -0,0 +1,170 @@ +#!/bin/sh --login +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +export jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +exp=${jobname} + +DATA=${ptmp:-/work/noaa/da/stmp/$LOGNAME/tmp} +mkdir -p $DATA + +#partition=${partition:-c1ms} +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${task_node:-$procs} +size=$((nodes*task_node)) +envars=$envars +threads=${rcpu:-1} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile +#echo "#PBS -S /bin/sh" >> $cfile +echo "#!/bin/sh --login" >> $cfile +echo "" >> $cfile +echo "#SBATCH --output=$output" >> $cfile +echo "#SBATCH --job-name=$jobname" >> $cfile +echo "#SBATCH --qos=$queue" >> $cfile +echo "#SBATCH --partition=$partition" >> $cfile +echo "#SBATCH --time=$timew" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile +echo "#SBATCH --account=$accnt" >> $cfile + +echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile +echo "export OMP_NUM_THREADS=$threads" >> $cfile +##echo "export OMP_STACKSIZE=2048M" >> $cfile +echo "ulimit -s unlimited" >> $cfile + +echo "" >>$cfile +echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile +echo "" >>$cfile + +echo ". /apps/lmod/lmod/init/sh" >> $cfile +echo "module purge" >> $cfile +echo "module use $modulefiles" >> $cfile +echo "module load gsi_hercules" >> $cfile +echo "module list" >> $cfile +echo "" >> $cfile +cat $exec >> $cfile + +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +sbatch=${sbatch:-sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch --export=ALL $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc From 5e4a738cefd8f7a0d857488256250a703ce9e479 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 9 Nov 2023 13:58:41 -0600 Subject: [PATCH 19/30] Clean up hercules debug #574 --- ush/build.sh | 2 +- ush/sub_hercules | 4 ++-- ush/sub_orion | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ush/build.sh b/ush/build.sh index f0cab6ffe5..9a280c4e55 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -29,7 +29,7 @@ module list set -x # Set CONTROLPATH variable to user develop installation -CONTROLPATH="$DIR_ROOT/install/bin" +CONTROLPATH="$DIR_ROOT/../develop/install/bin" # Collect BUILD Options CMAKE_OPTS+=" -DCMAKE_BUILD_TYPE=$BUILD_TYPE" diff --git a/ush/sub_hercules b/ush/sub_hercules index de5a415603..3466c5a79d 100755 --- a/ush/sub_hercules +++ b/ush/sub_hercules @@ -165,6 +165,6 @@ if [[ -w $SUBLOG ]];then jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG fi -#rm $cfile $ofile -#[[ $MKDATA = YES ]] && rmdir $DATA +rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA exit $rc diff --git a/ush/sub_orion b/ush/sub_orion index 1bcce5cc4f..e5844474db 100755 --- a/ush/sub_orion +++ b/ush/sub_orion @@ -87,6 +87,7 @@ export jobname=${jobname:-$bn} output=${output:-$jobname.out} myuser=$LOGNAME myhost=$(hostname) +exp=${jobname} DATA=${ptmp:-/work/noaa/da/stmp/$LOGNAME/tmp} mkdir -p $DATA @@ -108,6 +109,7 @@ echo "" echo "#SBATCH --output=$output" >> $cfile echo "#SBATCH --job-name=$jobname" >> $cfile echo "#SBATCH --qos=$queue" >> $cfile +echo "#SBATCH --partition=$partition" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile echo "#SBATCH --account=$accnt" >> $cfile From 46c7beb54de25227232a4061c0f66567dd55fb28 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 9 Nov 2023 20:52:47 +0000 Subject: [PATCH 20/30] Upgraded S4 to spack-stack/1.5.1. #589 --- modulefiles/gsi_s4.lua | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua index 7480eb210f..a60ea3c16e 100644 --- a/modulefiles/gsi_s4.lua +++ b/modulefiles/gsi_s4.lua @@ -1,10 +1,9 @@ help([[ ]]) -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local python_ver=os.getenv("python_ver") or "3.10.8" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" @@ -12,11 +11,11 @@ local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod-util", prod_util_ver)) +load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-march=ivybridge") pushenv("FFLAGS", "-march=ivybridge") From 68e699be97518f58d0ad4c310279e1ee20680a63 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 9 Nov 2023 15:05:07 -0600 Subject: [PATCH 21/30] Tidy up hercules #574 --- regression/regression_var.sh | 4 ++-- ush/module-setup.sh | 2 +- ush/sub_hercules | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 5d865a7cc8..02ffb24b12 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -42,9 +42,9 @@ elif [[ -d /sw/gaea ]]; then # Gaea export machine="Gaea" elif [[ -d /data/prod ]]; then # S4 export machine="S4" -elif [[ -d /work && $(hostname -d) =~ "Orion" ]]; then # Orion +elif [[ -d /work && $(hostname) =~ "Orion" ]]; then # Orion export machine="Orion" -elif [[ -d /work && $(hostname -d) =~ "Hercules" ]]; then # Hercules +elif [[ -d /work && $(hostname) =~ "hercules" ]]; then # Hercules export machine="Hercules" elif [[ -d /lfs/h2 ]]; then # wcoss2 export machine="wcoss2" diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 192a21509c..d13da1efa3 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -18,7 +18,7 @@ elif [[ $MACHINE_ID = hera* ]] ; then elif [[ $MACHINE_ID = hercules* ]] ; then # We are on Hercules if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/bash + source /apps/other/lmod/lmod/init/bash fi module purge diff --git a/ush/sub_hercules b/ush/sub_hercules index 3466c5a79d..573378fdb6 100755 --- a/ush/sub_hercules +++ b/ush/sub_hercules @@ -126,7 +126,7 @@ echo "" >>$cfile echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile echo "" >>$cfile -echo ". /apps/lmod/lmod/init/sh" >> $cfile +echo ". /apps/other/lmod/lmod/init/sh" >> $cfile echo "module purge" >> $cfile echo "module use $modulefiles" >> $cfile echo "module load gsi_hercules" >> $cfile From 4b5024f38505a10dc0034d9bc6243370cab2b925 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 13 Nov 2023 20:03:34 +0000 Subject: [PATCH 22/30] Updated WCOSS2 modulefiles to the UFS WM modules. #589 This include netcdf/4.9.2 and hdf5/1.14.0. However, it does not include the upgrades in spack-stack on the other systems for ncdiag/1.1.2, ip/4.3.0, and w3emc/2.10.0. --- modulefiles/gsi_wcoss2.lua | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index e5f4c7b812..aba4956479 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -3,12 +3,27 @@ help([[ local PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" local intel_ver=os.getenv("intel_ver") or "19.1.3.304" -local craype_ver=os.getenv("craype_ver") or "2.7.8" -local cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" +local craype_ver=os.getenv("craype_ver") or "2.7.13" +local cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.12" local cmake_ver= os.getenv("cmake_ver") or "3.20.2" local python_ver=os.getenv("python_ver") or "3.8.6" local prod_util_ver=os.getenv("prod_util_ver") or "2.0.10" +local netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" +local hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" +local bufr_ver=os.getenv("bufr_ver") or "11.7.0" +local bacio_ver=os.getenv("bacio_ver") or "2.4.1" +local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +local sp_ver=os.getenv("sp_ver") or "2.3.3" +local ip_ver=os.getenv("ip_ver") or "3.3.3" +local sigio_ver=os.getenv("sigio_ver") or "2.3.2" +local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" +local ncio_ver=os.getenv("ncio_ver") or "1.1.2" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" + load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) load(pathJoin("intel", intel_ver)) load(pathJoin("craype", craype_ver)) @@ -18,7 +33,24 @@ load(pathJoin("python", python_ver)) load(pathJoin("prod_util", prod_util_ver)) -load("gsi_common") +prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/compiler/intel/19.1.3.304") +prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.12") + +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf", netcdf_ver)) + +load(pathJoin("bufr", bufr_ver)) +load(pathJoin("bacio", bacio_ver)) +load(pathJoin("w3emc", w3emc_ver)) +load(pathJoin("sp", sp_ver)) +load(pathJoin("ip", ip_ver)) +load(pathJoin("sigio", sigio_ver)) +load(pathJoin("sfcio", sfcio_ver)) +load(pathJoin("nemsio", nemsio_ver)) +load(pathJoin("wrf_io", wrf_io_ver)) +load(pathJoin("ncio", ncio_ver)) +load(pathJoin("crtm", crtm_ver)) +load(pathJoin("ncdiag",ncdiag_ver)) pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230911") From 7f4255fb25580a368909852d13796701819d298e Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 14 Nov 2023 16:58:23 +0000 Subject: [PATCH 23/30] Revert WCOSS2 modulefiles. --- modulefiles/gsi_wcoss2.lua | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index aba4956479..8dde986e58 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -3,14 +3,13 @@ help([[ local PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" local intel_ver=os.getenv("intel_ver") or "19.1.3.304" -local craype_ver=os.getenv("craype_ver") or "2.7.13" -local cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.12" +local craype_ver=os.getenv("craype_ver") or "2.7.8" +local cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" local cmake_ver= os.getenv("cmake_ver") or "3.20.2" local python_ver=os.getenv("python_ver") or "3.8.6" local prod_util_ver=os.getenv("prod_util_ver") or "2.0.10" -local netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" -local hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" +local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" @@ -33,12 +32,7 @@ load(pathJoin("python", python_ver)) load(pathJoin("prod_util", prod_util_ver)) -prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/compiler/intel/19.1.3.304") -prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.12") - -load(pathJoin("hdf5", hdf5_ver)) load(pathJoin("netcdf", netcdf_ver)) - load(pathJoin("bufr", bufr_ver)) load(pathJoin("bacio", bacio_ver)) load(pathJoin("w3emc", w3emc_ver)) From 28ebd5f67bfe6fb6088024e3d7f527546678b6be Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 Nov 2023 17:04:12 +0000 Subject: [PATCH 24/30] Update Hera GNU modulefile to spack-stack/1.5.1. #589 --- modulefiles/gsi_hera.gnu.lua | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index 7dcbc2e0c8..550b01ee7b 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -1,10 +1,11 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +--Needed for openmpi build prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local python_ver=os.getenv("python_ver") or "3.10.8" local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" @@ -13,12 +14,12 @@ local openblas_ver=os.getenv("openblas_ver") or "0.3.19" load(pathJoin("stack-gcc", stack_gnu_ver)) load(pathJoin("stack-openmpi", stack_openmpi_ver)) -load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod-util", prod_util_ver)) +load(pathJoin("prod_util", prod_util_ver)) load(pathJoin("openblas", openblas_ver)) pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230911") From e2e8f6eab2e84d4bca8a712d5392a2d4628418f9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 14 Nov 2023 17:11:45 +0000 Subject: [PATCH 25/30] Update Jet spack-stack to 1.5.1 #589 --- modulefiles/gsi_jet.lua | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index 967a03fd0d..20b80ff61a 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -1,10 +1,9 @@ help([[ ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local python_ver=os.getenv("python_ver") or "3.10.8" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" @@ -12,11 +11,11 @@ local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod-util", prod_util_ver)) +load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2") pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2") From 4deb40fddf29e3d986e7504944c895b5f42c22c4 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 Nov 2023 17:53:40 +0000 Subject: [PATCH 26/30] Update Hera/intel spack-stack to 1.5.1. #589 --- modulefiles/gsi_hera.intel.lua | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index bdc82db365..abdc6e5623 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -1,10 +1,9 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local python_ver=os.getenv("python_ver") or "3.10.8" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" @@ -12,11 +11,11 @@ local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod-util", prod_util_ver)) +load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") From b4a59790dc5eb99967e7ad4d22b27ace35dbf820 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 Nov 2023 18:06:11 +0000 Subject: [PATCH 27/30] Reverted Orion to use higher time/cpu requests. --- regression/regression_param.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/regression/regression_param.sh b/regression/regression_param.sh index fd93e1024a..46d2647ac0 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -63,10 +63,10 @@ case $regtest in topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "Orion" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" + topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2" elif [[ "$machine" = "Hercules" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" + topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -163,11 +163,11 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Hercules" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" From a36040aba432263625a8db0dc88a8f15180e4218 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 12:57:44 +0000 Subject: [PATCH 28/30] Update CI steps (avoid using icx/ifx). --- .github/workflows/gcc.yml | 35 ++++++++++++++-------------- .github/workflows/intel.yml | 46 ++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index 6ba8ef3295..a54c05e083 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -15,9 +15,9 @@ env: # The jobs are split into: # 1. a dependency build step (setup), and -# 2. a GSI build step (build) +# 2. a GSI build step (gsi) # The setup is run once and the environment is cached, -# so each build of GSI can reuse the cached dependencies to save time (and compute). +# so each subsequent build of GSI can reuse the cached dependencies to save time (and compute). jobs: setup: @@ -25,62 +25,63 @@ jobs: steps: # Checkout the GSI to get the ci/spack.yaml file - - name: checkout-gsi + - name: checkout if: steps.cache-env.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: GSI + path: gsi # Cache spack, compiler and dependencies - name: cache-env id: cache-env - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | spack ~/.spack - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} # Install dependencies using Spack - name: install-dependencies-with-spack if: steps.cache-env.outputs.cache-hit != 'true' run: | sudo apt-get install cmake - git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git + git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git source spack/share/spack/setup-env.sh - spack env create gsi-env GSI/ci/spack.yaml + spack env create gsi-env gsi/ci/spack.yaml spack env activate gsi-env spack compiler find + sudo apt install cmake spack external find spack add mpich@3.4.2 spack concretize spack install -v --fail-fast --dirty spack clean -a - build: + gsi: needs: setup runs-on: ubuntu-latest steps: - - name: checkout-gsi - uses: actions/checkout@v3 + - name: checkout + uses: actions/checkout@v4 with: - path: GSI + path: gsi - name: cache-env id: cache-env - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | spack ~/.spack - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} - - name: build-gsi + - name: build run: | source spack/share/spack/setup-env.sh spack env activate gsi-env - cd GSI + cd gsi mkdir -p build && cd build cmake -DCMAKE_INSTALL_PREFIX=../install -DGSI_MODE=Regional -DENKF_MODE=GFS -DBUILD_REG_TESTING=OFF .. make -j2 VERBOSE=1 diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index b30d24c1f8..b81615e033 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -18,13 +18,13 @@ env: # The jobs are split into: # 1. a dependency build step (setup), and -# 2. a GSI build step (build) +# 2. a GSI build step (gsi) # The setup is run once and the environment is cached, -# so each build of GSI can reuse the cached dependencies to save time (and compute). +# so each subsequent build of GSI can reuse the cached dependencies to save time (and compute). jobs: setup: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: # Free up disk space @@ -37,31 +37,30 @@ jobs: docker rmi $(docker image ls -aq) # Checkout the GSI to get the ci/spack.yaml file - - name: checkout-gsi + - name: checkout if: steps.cache-env.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: GSI + path: gsi # Cache spack, compiler and dependencies - name: cache-env id: cache-env - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | spack ~/.spack /opt/intel - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} - name: install-intel-compilers - if: steps.cache-env.outputs.cache-hit != 'true' run: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1 sudo apt-get clean echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile @@ -71,46 +70,47 @@ jobs: run: | sudo mv /usr/local/ /usr_local_mv sudo apt-get install cmake - git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git + git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git source spack/share/spack/setup-env.sh - spack env create gsi-env GSI/ci/spack.yaml + spack env create gsi-env gsi/ci/spack.yaml spack env activate gsi-env spack compiler find + sudo apt install cmake spack external find spack add intel-oneapi-mpi spack concretize spack install -v --fail-fast --dirty spack clean -a - build: + gsi: needs: setup - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - - name: checkout-gsi - uses: actions/checkout@v3 - with: - path: GSI - - name: install-intel run: | echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile + - name: checkout + uses: actions/checkout@v4 + with: + path: gsi + - name: cache-env id: cache-env - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | spack ~/.spack /opt/intel - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} - - name: build-gsi + - name: build run: | source spack/share/spack/setup-env.sh spack env activate gsi-env - cd GSI + cd gsi mkdir -p build && cd build cmake -DCMAKE_INSTALL_PREFIX=../install -DGSI_MODE=Regional -DENKF_MODE=GFS -DBUILD_REG_TESTING=OFF .. make -j2 VERBOSE=1 From 344f457805c4d87abbfdb0947a40b473b682ea94 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 13:12:21 +0000 Subject: [PATCH 29/30] Revert cache versions. --- .github/workflows/gcc.yml | 4 ++-- .github/workflows/intel.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index a54c05e083..1f6fa3afcd 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -34,7 +34,7 @@ jobs: # Cache spack, compiler and dependencies - name: cache-env id: cache-env - uses: actions/cache@v4 + uses: actions/cache@v3 with: path: | spack @@ -70,7 +70,7 @@ jobs: - name: cache-env id: cache-env - uses: actions/cache@v4 + uses: actions/cache@v3 with: path: | spack diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index b81615e033..e44a8a4416 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -46,7 +46,7 @@ jobs: # Cache spack, compiler and dependencies - name: cache-env id: cache-env - uses: actions/cache@v4 + uses: actions/cache@v3 with: path: | spack @@ -98,7 +98,7 @@ jobs: - name: cache-env id: cache-env - uses: actions/cache@v4 + uses: actions/cache@v3 with: path: | spack From cbb806757a924e6052bcc2813135ff7f3fefc460 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 13:28:38 +0000 Subject: [PATCH 30/30] Re-upgrade ubuntu in CI to latest. --- .github/workflows/intel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index e44a8a4416..d21420687a 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -24,7 +24,7 @@ env: jobs: setup: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: # Free up disk space @@ -84,7 +84,7 @@ jobs: gsi: needs: setup - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: install-intel