From c94bc72ff410b48c325abbfe92c9fcb601d89aed Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Thu, 30 Nov 2023 11:52:16 -0500 Subject: [PATCH] Upgrade to spack-stack libraries on non-production machines (#624) Co-authored-by: Natalie Perlin Co-authored-by: DavidHuber-NOAA Co-authored-by: RussTreadon-NOAA --- .github/workflows/gcc.yml | 31 +++--- .github/workflows/intel.yml | 47 ++++---- ci/spack.yaml | 4 +- modulefiles/gsi_cheyenne.gnu.lua | 36 +++--- modulefiles/gsi_cheyenne.intel.lua | 32 +++--- modulefiles/gsi_common.lua | 16 +-- modulefiles/gsi_gaea.lua | 24 ++-- modulefiles/gsi_hera.gnu.lua | 22 ++-- modulefiles/gsi_hera.intel.lua | 21 ++-- modulefiles/gsi_hercules.lua | 26 +++++ modulefiles/gsi_jet.lua | 22 ++-- modulefiles/gsi_orion.lua | 21 ++-- modulefiles/gsi_s4.lua | 23 ++-- modulefiles/gsi_wcoss2.lua | 28 ++++- regression/regression_param.sh | 137 ++++++++++++++--------- regression/regression_var.sh | 19 +++- ush/detect_machine.sh | 2 + ush/module-setup.sh | 7 ++ ush/sub_hercules | 170 +++++++++++++++++++++++++++++ ush/sub_orion | 2 + 20 files changed, 485 insertions(+), 205 deletions(-) create mode 100644 modulefiles/gsi_hercules.lua create mode 100755 ush/sub_hercules diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index 6ba8ef3295..1f6fa3afcd 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -15,9 +15,9 @@ env: # The jobs are split into: # 1. a dependency build step (setup), and -# 2. a GSI build step (build) +# 2. a GSI build step (gsi) # The setup is run once and the environment is cached, -# so each build of GSI can reuse the cached dependencies to save time (and compute). +# so each subsequent build of GSI can reuse the cached dependencies to save time (and compute). jobs: setup: @@ -25,11 +25,11 @@ jobs: steps: # Checkout the GSI to get the ci/spack.yaml file - - name: checkout-gsi + - name: checkout if: steps.cache-env.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: GSI + path: gsi # Cache spack, compiler and dependencies - name: cache-env @@ -39,33 +39,34 @@ jobs: path: | spack ~/.spack - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} # Install dependencies using Spack - name: install-dependencies-with-spack if: steps.cache-env.outputs.cache-hit != 'true' run: | sudo apt-get install cmake - git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git + git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git source spack/share/spack/setup-env.sh - spack env create gsi-env GSI/ci/spack.yaml + spack env create gsi-env gsi/ci/spack.yaml spack env activate gsi-env spack compiler find + sudo apt install cmake spack external find spack add mpich@3.4.2 spack concretize spack install -v --fail-fast --dirty spack clean -a - build: + gsi: needs: setup runs-on: ubuntu-latest steps: - - name: checkout-gsi - uses: actions/checkout@v3 + - name: checkout + uses: actions/checkout@v4 with: - path: GSI + path: gsi - name: cache-env id: cache-env @@ -74,13 +75,13 @@ jobs: path: | spack ~/.spack - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} - - name: build-gsi + - name: build run: | source spack/share/spack/setup-env.sh spack env activate gsi-env - cd GSI + cd gsi mkdir -p build && cd build cmake -DCMAKE_INSTALL_PREFIX=../install -DGSI_MODE=Regional -DENKF_MODE=GFS -DBUILD_REG_TESTING=OFF .. make -j2 VERBOSE=1 diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index d48c00a21e..d21420687a 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -18,21 +18,30 @@ env: # The jobs are split into: # 1. a dependency build step (setup), and -# 2. a GSI build step (build) +# 2. a GSI build step (gsi) # The setup is run once and the environment is cached, -# so each build of GSI can reuse the cached dependencies to save time (and compute). +# so each subsequent build of GSI can reuse the cached dependencies to save time (and compute). jobs: setup: runs-on: ubuntu-latest steps: + # Free up disk space + - name: free-disk-spack + run: | + df -h + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + docker rmi $(docker image ls -aq) + # Checkout the GSI to get the ci/spack.yaml file - - name: checkout-gsi + - name: checkout if: steps.cache-env.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: GSI + path: gsi # Cache spack, compiler and dependencies - name: cache-env @@ -43,16 +52,15 @@ jobs: spack ~/.spack /opt/intel - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} - name: install-intel-compilers - if: steps.cache-env.outputs.cache-hit != 'true' run: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1 sudo apt-get clean echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile @@ -62,31 +70,32 @@ jobs: run: | sudo mv /usr/local/ /usr_local_mv sudo apt-get install cmake - git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git + git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git source spack/share/spack/setup-env.sh - spack env create gsi-env GSI/ci/spack.yaml + spack env create gsi-env gsi/ci/spack.yaml spack env activate gsi-env spack compiler find + sudo apt install cmake spack external find spack add intel-oneapi-mpi spack concretize spack install -v --fail-fast --dirty spack clean -a - build: + gsi: needs: setup runs-on: ubuntu-latest steps: - - name: checkout-gsi - uses: actions/checkout@v3 - with: - path: GSI - - name: install-intel run: | echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile + - name: checkout + uses: actions/checkout@v4 + with: + path: gsi + - name: cache-env id: cache-env uses: actions/cache@v3 @@ -95,13 +104,13 @@ jobs: spack ~/.spack /opt/intel - key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('GSI/ci/spack.yaml') }} + key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('gsi/ci/spack.yaml') }} - - name: build-gsi + - name: build run: | source spack/share/spack/setup-env.sh spack env activate gsi-env - cd GSI + cd gsi mkdir -p build && cd build cmake -DCMAKE_INSTALL_PREFIX=../install -DGSI_MODE=Regional -DENKF_MODE=GFS -DBUILD_REG_TESTING=OFF .. make -j2 VERBOSE=1 diff --git a/ci/spack.yaml b/ci/spack.yaml index 0fc66547e5..deacdff0b5 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -6,8 +6,8 @@ spack: - intel - gcc@10:10 specs: - - netcdf-c@4.7.4 - - netcdf-fortran@4.5.3 + - netcdf-c@4.9.2 + - netcdf-fortran@4.6.0 - bufr@11.7.0 - bacio@2.4.1 - w3emc@2.9.2 diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua index 43e6aaf02c..1d903082a8 100644 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ b/modulefiles/gsi_cheyenne.gnu.lua @@ -1,30 +1,32 @@ help([[ ]]) -load("cmake/3.22.0") -load("python/3.7.9") -load("ncarenv/1.3") -load("gnu/11.2.0") -load("mpt/2.25") -load("ncarcompilers/0.5.0") -unload("intel") -unload("netcdf") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack") +unload("ncarenv/1.3") +unload("intel/19.1.1") +unload("ncarcompilers/0.5.0") +unload("mpt/2.25") +unload("netcdf/4.8.1") -load("hpc/1.2.0") -load("hpc-gnu/11.2.0") -load("hpc-mpt/2.25") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") -load("gsi_common") +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_gnu_ver=os.getenv("stack_gnu_ver") or "10.1.0" +local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.1" +local cmake_ver=os.getenv("cmake_ver") or "3.22.0" +load(pathJoin("stack-gcc", stack_gnu_ver)) +load(pathJoin("stack-openmpi", stack_openmpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_fix/fix") +load("gsi_common") -pushenv("CC", "mpicc") -pushenv("FC", "mpif90") -pushenv("CXX", "mpicxx") +pushenv("CFLAGS", "-xHOST") +pushenv("FFLAGS", "-xHOST") +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") whatis("Description: GSI environment on Cheyenne with GNU Compilers") diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua index 26ed666695..8c328e2b34 100644 --- a/modulefiles/gsi_cheyenne.intel.lua +++ b/modulefiles/gsi_cheyenne.intel.lua @@ -1,26 +1,32 @@ help([[ ]]) -load("cmake/3.22.0") -load("python/3.7.9") -load("ncarenv/1.3") -load("intel/2022.1") -load("mpt/2.25") -load("ncarcompilers/0.5.0") +unload("ncarenv/1.3") +unload("intel/19.1.1") +unload("ncarcompilers/0.5.0") +unload("mpt/2.25") +unload("netcdf/4.8.1") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") -load("hpc/1.2.0") -load("hpc-intel/2022.1") -load("hpc-mpt/2.25") -load("mkl/2022.1") +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "19.1.1.217" +local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2019.7.217" +local cmake_ver=os.getenv("cmake_ver") or "3.22.0" -load("gsi_common") +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-mpi", stack_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) -load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) +load("gsi_common") +load(pathJoin("prod-util", os.getenv("prod_util_ver") or "1.2.2")) pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230911") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") + whatis("Description: GSI environment on Cheyenne with Intel Compilers") diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index c54f6ddb92..d3365a98dc 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -2,22 +2,24 @@ help([[ Load common modules to build GSI on all machines ]]) -local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" +local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" +local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" -local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" local sp_ver=os.getenv("sp_ver") or "2.3.3" -local ip_ver=os.getenv("ip_ver") or "3.3.3" +local ip_ver=os.getenv("ip_ver") or "4.3.0" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" local crtm_ver=os.getenv("crtm_ver") or "2.4.0" -local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" +local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2" -load(pathJoin("netcdf", netcdf_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) load(pathJoin("bufr", bufr_ver)) load(pathJoin("bacio", bacio_ver)) @@ -27,7 +29,7 @@ load(pathJoin("ip", ip_ver)) load(pathJoin("sigio", sigio_ver)) load(pathJoin("sfcio", sfcio_ver)) load(pathJoin("nemsio", nemsio_ver)) -load(pathJoin("wrf_io", wrf_io_ver)) +load(pathJoin("wrf-io", wrf_io_ver)) load(pathJoin("ncio", ncio_ver)) load(pathJoin("crtm", crtm_ver)) -load(pathJoin("ncdiag",ncdiag_ver)) +load(pathJoin("gsi-ncdiag",ncdiag_ver)) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index a7a2454eff..ef6b9ddba7 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -1,20 +1,28 @@ help([[ ]]) -load("cmake/3.20.1") +unload("intel") +unload("cray-mpich") +unload("cray-python") +unload("darshan") -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles") -load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) -load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) -load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) -load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) load("gsi_common") local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("prod-util", prod_util_ver)) -- Needed at runtime: load("alps") diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index 37504485e3..550b01ee7b 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -1,20 +1,20 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +--Needed for openmpi build +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local gnu_ver=os.getenv("gnu_ver") or "9.2.0" -local hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2" -local hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2" -local cmake_ver=os.getenv("cmake_ver") or "3.20.1" +local python_ver=os.getenv("python_ver") or "3.10.8" +local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" +local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -local openblas_ver=os.getenv("openblas_ver") or "0.3.23" +local openblas_ver=os.getenv("openblas_ver") or "0.3.19" -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("gnu", gnu_ver)) -load(pathJoin("hpc-gnu", hpc_gnu_ver)) -load(pathJoin("hpc-mpich", hpc_mpich_ver)) +load(pathJoin("stack-gcc", stack_gnu_ver)) +load(pathJoin("stack-openmpi", stack_openmpi_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index 619d0e76c9..abdc6e5623 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -1,25 +1,20 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/miniconda3/modulefiles") -miniconda3_ver=os.getenv("miniconda3_ver") or "4.12.0" -load(pathJoin("miniconda3", miniconda3_ver)) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") - -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -local cmake_ver=os.getenv("cmake_ver") or "3.20.1" +local python_ver=os.getenv("python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") - load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") diff --git a/modulefiles/gsi_hercules.lua b/modulefiles/gsi_hercules.lua new file mode 100644 index 0000000000..bf29bc21db --- /dev/null +++ b/modulefiles/gsi_hercules.lua @@ -0,0 +1,26 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gsi_common") +load(pathJoin("prod_util", prod_util_ver)) +load("intel-oneapi-mkl/2022.2.1") + +pushenv("CFLAGS", "-xHOST") +pushenv("FFLAGS", "-xHOST") + +pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230911") + +whatis("Description: GSI environment on Hercules with Intel Compilers") diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index c9e5e90680..20b80ff61a 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -1,31 +1,25 @@ help([[ ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles") -miniconda3_ver=os.getenv("miniconda3_ver") or "4.12.0" -load(pathJoin("miniconda3", miniconda3_ver)) +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") - -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -local cmake_ver=os.getenv("cmake_ver") or "3.20.1" +local python_ver=os.getenv("python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("python", python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") - load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2") pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2") - pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230911") whatis("Description: GSI environment on Jet with Intel Compilers") diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index e75a01ef5e..80ec342c93 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -1,25 +1,20 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") -miniconda3_ver=os.getenv("miniconda3_ver") or "4.12.0" -load(pathJoin("miniconda3", miniconda3_ver)) +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack") - -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -local cmake_ver=os.getenv("cmake_ver") or "3.22.1" +local stack_python_ver=os.getenv("python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") - load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua index 03c21e708d..a60ea3c16e 100644 --- a/modulefiles/gsi_s4.lua +++ b/modulefiles/gsi_s4.lua @@ -1,23 +1,20 @@ help([[ ]]) -local hpc_ver=os.getenv("hpc_ver") or "1.2.0" -local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1" -local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1" -local miniconda_ver=os.getenv("miniconda_ver") or "3.8-s4" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" - -prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -load("license_intel/S4") -load(pathJoin("hpc", hpc_ver)) -load(pathJoin("hpc-intel", hpc_intel_ver)) -load(pathJoin("hpc-impi", hpc_impi_ver)) +local python_ver=os.getenv("python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("miniconda", miniconda_ver)) +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("python", python_ver)) +load(pathJoin("cmake", cmake_ver)) load("gsi_common") - load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-march=ivybridge") diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index e5f4c7b812..8dde986e58 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -9,6 +9,20 @@ local cmake_ver= os.getenv("cmake_ver") or "3.20.2" local python_ver=os.getenv("python_ver") or "3.8.6" local prod_util_ver=os.getenv("prod_util_ver") or "2.0.10" +local netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" +local bufr_ver=os.getenv("bufr_ver") or "11.7.0" +local bacio_ver=os.getenv("bacio_ver") or "2.4.1" +local w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +local sp_ver=os.getenv("sp_ver") or "2.3.3" +local ip_ver=os.getenv("ip_ver") or "3.3.3" +local sigio_ver=os.getenv("sigio_ver") or "2.3.2" +local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" +local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" +local ncio_ver=os.getenv("ncio_ver") or "1.1.2" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" + load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) load(pathJoin("intel", intel_ver)) load(pathJoin("craype", craype_ver)) @@ -18,7 +32,19 @@ load(pathJoin("python", python_ver)) load(pathJoin("prod_util", prod_util_ver)) -load("gsi_common") +load(pathJoin("netcdf", netcdf_ver)) +load(pathJoin("bufr", bufr_ver)) +load(pathJoin("bacio", bacio_ver)) +load(pathJoin("w3emc", w3emc_ver)) +load(pathJoin("sp", sp_ver)) +load(pathJoin("ip", ip_ver)) +load(pathJoin("sigio", sigio_ver)) +load(pathJoin("sfcio", sfcio_ver)) +load(pathJoin("nemsio", nemsio_ver)) +load(pathJoin("wrf_io", wrf_io_ver)) +load(pathJoin("ncio", ncio_ver)) +load(pathJoin("crtm", crtm_ver)) +load(pathJoin("ncdiag",ncdiag_ver)) pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230911") diff --git a/regression/regression_param.sh b/regression/regression_param.sh index 87a21dc0f1..46d2647ac0 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -9,10 +9,15 @@ case $machine in memnode=96 numcore=40 ;; - Orion) + Orion) sub_cmd="sub_orion" memnode=192 numcore=40 + ;; + Hercules) + sub_cmd="sub_hercules" + memnode=512 + numcore=40 ;; Jet) sub_cmd="sub_jet" @@ -59,6 +64,9 @@ case $regtest in elif [[ "$machine" = "Orion" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -87,20 +95,23 @@ case $regtest in rrfs_3denvar_glbens) if [[ "$machine" = "Hera" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -117,20 +128,23 @@ case $regtest in hafs_3denvar_hybens) if [[ "$machine" = "Hera" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -146,20 +160,23 @@ case $regtest in hafs_4denvar_glbens) if [[ "$machine" = "Hera" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:20:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:20:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -176,20 +193,23 @@ case $regtest in netcdf_fv3_regional) if [[ "$machine" = "Hera" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" @@ -206,20 +226,23 @@ case $regtest in rtma) if [[ "$machine" = "Hera" ]]; then - topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Jet" ]]; then - topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2" @@ -236,20 +259,23 @@ case $regtest in global_enkf) if [[ "$machine" = "Hera" ]]; then - topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Orion" ]]; then - topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + elif [[ "$machine" = "Hercules" ]]; then + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Jet" ]]; then - topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" + topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" @@ -300,7 +326,10 @@ if [[ "$machine" = "Hera" ]]; then export APRUN="srun" elif [[ "$machine" = "Orion" ]]; then export OMP_STACKSIZE=2048M - export APRUN="srun -n \$ntasks" + export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" +elif [[ "$machine" = "Hercules" ]]; then + export OMP_STACKSIZE=2048M + export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" elif [[ "$machine" = "Jet" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 7403d89ec0..02ffb24b12 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -42,8 +42,10 @@ elif [[ -d /sw/gaea ]]; then # Gaea export machine="Gaea" elif [[ -d /data/prod ]]; then # S4 export machine="S4" -elif [[ -d /work ]]; then # Orion +elif [[ -d /work && $(hostname) =~ "Orion" ]]; then # Orion export machine="Orion" +elif [[ -d /work && $(hostname) =~ "hercules" ]]; then # Hercules + export machine="Hercules" elif [[ -d /lfs/h2 ]]; then # wcoss2 export machine="wcoss2" fi @@ -98,18 +100,25 @@ case $machine in export check_resource="no" export accnt="${accnt:-GFS-DEV}" ;; - Orion) + Orion | Hercules) export local_or_default="${local_or_default:-/work/noaa/da/$LOGNAME}" if [ -d $local_or_default ]; then - export noscrub="$local_or_default/noscrub" + export noscrub="$local_or_default/noscrub" elif [ -d /work/noaa/global/$LOGNAME ]; then - export noscrub="/work/noaa/global/$LOGNAME/noscrub" + export noscrub="/work/noaa/global/$LOGNAME/noscrub" fi export queue="${queue:-batch}" + + if [[ "${machine}" == "Orion" ]]; then + export partition="${partition:-orion}" + else + export partition="${partition:-hercules}" + fi + export group="${group:-global}" if [[ "$cmaketest" = "false" ]]; then - export basedir="/work/noaa/da/$LOGNAME/gsi" + export basedir="/work/noaa/da/$LOGNAME/gsi" fi export ptmp="${ptmp:-/work/noaa/stmp/$LOGNAME/$ptmpName}" diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 6f0673ce29..ac6c7f58d1 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -25,6 +25,8 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index ab92477a56..d13da1efa3 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -15,6 +15,13 @@ elif [[ $MACHINE_ID = hera* ]] ; then fi module purge +elif [[ $MACHINE_ID = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/other/lmod/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = orion* ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/sub_hercules b/ush/sub_hercules new file mode 100755 index 0000000000..573378fdb6 --- /dev/null +++ b/ush/sub_hercules @@ -0,0 +1,170 @@ +#!/bin/sh --login +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +export jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +exp=${jobname} + +DATA=${ptmp:-/work/noaa/da/stmp/$LOGNAME/tmp} +mkdir -p $DATA + +#partition=${partition:-c1ms} +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${task_node:-$procs} +size=$((nodes*task_node)) +envars=$envars +threads=${rcpu:-1} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile +#echo "#PBS -S /bin/sh" >> $cfile +echo "#!/bin/sh --login" >> $cfile +echo "" >> $cfile +echo "#SBATCH --output=$output" >> $cfile +echo "#SBATCH --job-name=$jobname" >> $cfile +echo "#SBATCH --qos=$queue" >> $cfile +echo "#SBATCH --partition=$partition" >> $cfile +echo "#SBATCH --time=$timew" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile +echo "#SBATCH --account=$accnt" >> $cfile + +echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile +echo "export OMP_NUM_THREADS=$threads" >> $cfile +##echo "export OMP_STACKSIZE=2048M" >> $cfile +echo "ulimit -s unlimited" >> $cfile + +echo "" >>$cfile +echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile +echo "" >>$cfile + +echo ". /apps/other/lmod/lmod/init/sh" >> $cfile +echo "module purge" >> $cfile +echo "module use $modulefiles" >> $cfile +echo "module load gsi_hercules" >> $cfile +echo "module list" >> $cfile +echo "" >> $cfile +cat $exec >> $cfile + +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +sbatch=${sbatch:-sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch --export=ALL $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/ush/sub_orion b/ush/sub_orion index 1bcce5cc4f..e5844474db 100755 --- a/ush/sub_orion +++ b/ush/sub_orion @@ -87,6 +87,7 @@ export jobname=${jobname:-$bn} output=${output:-$jobname.out} myuser=$LOGNAME myhost=$(hostname) +exp=${jobname} DATA=${ptmp:-/work/noaa/da/stmp/$LOGNAME/tmp} mkdir -p $DATA @@ -108,6 +109,7 @@ echo "" echo "#SBATCH --output=$output" >> $cfile echo "#SBATCH --job-name=$jobname" >> $cfile echo "#SBATCH --qos=$queue" >> $cfile +echo "#SBATCH --partition=$partition" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile echo "#SBATCH --account=$accnt" >> $cfile