From 922e1e240c1b1e07f40f19b2ea3e4c46e022d326 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 23 Aug 2023 03:19:45 -0600 Subject: [PATCH 01/13] Changes for Derecho, a new platform --- etc/lmod-setup.csh | 10 +++- etc/lmod-setup.sh | 11 +++- modulefiles/build_derecho_intel.lua | 34 ++++++++++++ modulefiles/srw_common.lua | 14 ++--- modulefiles/tasks/derecho/aqm_ics.local.lua | 3 + modulefiles/tasks/derecho/aqm_lbcs.local.lua | 3 + .../tasks/derecho/fire_emission.local.lua | 2 + .../miniconda_regional_workflow_cmaq.lua | 6 ++ .../tasks/derecho/nexus_emission.local.lua | 4 ++ .../tasks/derecho/nexus_gfs_sfc.local.lua | 2 + .../tasks/derecho/nexus_post_split.local.lua | 3 + .../tasks/derecho/plot_allvars.local.lua | 5 ++ .../tasks/derecho/point_source.local.lua | 2 + .../tasks/derecho/pre_post_stat.local.lua | 2 + modulefiles/tasks/derecho/python_srw.lua | 5 ++ modulefiles/tasks/derecho/run_vx.local.lua | 6 ++ modulefiles/wflow_derecho.lua | 24 ++++++++ tests/README.md | 6 +- .../WE2E/machine_suites/comprehensive.derecho | 55 +++++++++++++++++++ tests/WE2E/machine_suites/coverage.derecho | 10 ++++ tests/WE2E/setup_WE2E_tests.sh | 2 +- tests/build.sh | 2 +- ush/get_crontab_contents.py | 2 +- ush/machine/derecho.yaml | 35 ++++++++++++ ush/valid_param_vals.yaml | 2 +- 25 files changed, 234 insertions(+), 16 deletions(-) create mode 100644 modulefiles/build_derecho_intel.lua create mode 100644 modulefiles/tasks/derecho/aqm_ics.local.lua create mode 100644 modulefiles/tasks/derecho/aqm_lbcs.local.lua create mode 100644 modulefiles/tasks/derecho/fire_emission.local.lua create mode 100644 modulefiles/tasks/derecho/miniconda_regional_workflow_cmaq.lua create mode 100644 modulefiles/tasks/derecho/nexus_emission.local.lua create mode 100644 modulefiles/tasks/derecho/nexus_gfs_sfc.local.lua create mode 100644 modulefiles/tasks/derecho/nexus_post_split.local.lua create mode 100644 modulefiles/tasks/derecho/plot_allvars.local.lua create mode 100644 modulefiles/tasks/derecho/point_source.local.lua create mode 100644 modulefiles/tasks/derecho/pre_post_stat.local.lua create mode 100644 modulefiles/tasks/derecho/python_srw.lua create mode 100644 modulefiles/tasks/derecho/run_vx.local.lua create mode 100644 modulefiles/wflow_derecho.lua create mode 100644 tests/WE2E/machine_suites/comprehensive.derecho create mode 100644 tests/WE2E/machine_suites/coverage.derecho create mode 100644 ush/machine/derecho.yaml diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 3b587f1532..85ea8d073b 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -6,7 +6,7 @@ Usage: source etc/lmod-setup.csh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | derecho | hera | jet | orion | wcoss2 ) EOF_USAGE exit 1 else @@ -40,6 +40,14 @@ else if ( "$L_MACHINE" == singularity ) then else if ( "$L_MACHINE" == gaea ) then source /lustre/f2/dev/role.epic/contrib/Lmod_init.csh +else if ( "$L_MACHINE" == derecho ) then + module purge > /dev/null + setenv ENV "/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/csh" + source $ENV + setenv BASH_ENV "/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/bash" + setenv PATH "$MODULESHOME/libexec:$MODULESHOME/init/ksh_funcs:$PATH" + module --initial_load --no_redirect restore + else if ( "$L_MACHINE" == odin ) then module unload modules unset -f module diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 9c330d910c..e22d11c3dc 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -7,7 +7,7 @@ Usage: source etc/lmod-setup.sh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss2 ) + (e.g. derecho | hera | jet | orion | wcoss2 ) EOF_USAGE exit 1 else @@ -47,6 +47,15 @@ elif [ "$L_MACHINE" = singularity ]; then elif [ "$L_MACHINE" = gaea ]; then source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh +elif [ "$L_MACHINE" = derecho ]; then + module purge 2>/dev/null + + export BASH_ENV=/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/bash + source $BASH_ENV + export LMOD_SYSTEM_DEFAULT_MODULES="ncarenv:craype/2.7.20" + export PATH=$MODULESHOME/libexec:$MODULESHOME/init/ksh_funcs:$PATH + module --initial_load --no_redirect restore + elif [ "$L_MACHINE" = odin ]; then module unload modules unset -f module diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua new file mode 100644 index 0000000000..e8ba15fdc5 --- /dev/null +++ b/modulefiles/build_derecho_intel.lua @@ -0,0 +1,34 @@ +help([[ +This module loads libraries for building the UFS SRW App on +the CISL machine Cheyenne using Intel-2022.1 +]]) + +whatis([===[Loads libraries needed for building the UFS SRW App on Cheyenne ]===]) + +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3")) +load(pathJoin("ncarenv", os.getenv("ncarenv_ver") or "23.06")) +load(pathJoin("craype", os.getenv("craype_ver") or "2.7.20")) + +unload("netcdf") +unload("hdf5") +unload("intel") + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/hpc-stack/intel-classic-2023.0.0/modulefiles/stack") +load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) +load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2023.0.0")) +load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "8.1.25")) + +load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "1.0.0")) +load(pathJoin("mkl", os.getenv("mkl_ver") or "2023.0.0")) + +load("srw_common") + +setenv("CC","cc") +setenv("FC","ftn") +setenv("CXX","CC") + +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_Platform","derecho.intel") + diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 12b5122c18..96541c9960 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -1,12 +1,12 @@ load("jasper/2.0.25") load("zlib/1.2.11") -load_any("png/1.6.35", "libpng/1.6.37") +load("libpng/1.6.37") -load_any("netcdf/4.7.4", "netcdf-c/4.7.4") -load_any("netcdf/4.7.4", "netcdf-fortran/4.5.4") -load_any("pio/2.5.7", "parallelio/2.5.2") -load_any("esmf/8.3.0b09", "esmf/8.2.0") -load("fms/2022.04") +load_any("netcdf/4.9.2", "netcdf-c/4.9.2") +load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") +load_any("pio/2.5.10", "parallelio/2.5.10") +load_any("esmf/8.4.2", "esmf/8.5.0") +load("fms/2023.01") load("bufr/11.7.0") load("bacio/2.4.1") @@ -19,7 +19,7 @@ load("w3emc/2.9.2") load_any("gftl-shared/v1.5.0", "gftl-shared/1.5.0") load_any("yafyaml/v0.5.1", "yafyaml/0.5.1") -load_any("mapl/2.22.0-esmf-8.3.0b09", "mapl/2.11.0-esmf-8.2.0") +load_any("mapl/2.35.2-esmf-8.4.2") load("nemsio/2.5.4") load("sfcio/1.4.1") diff --git a/modulefiles/tasks/derecho/aqm_ics.local.lua b/modulefiles/tasks/derecho/aqm_ics.local.lua new file mode 100644 index 0000000000..de700dbbc6 --- /dev/null +++ b/modulefiles/tasks/derecho/aqm_ics.local.lua @@ -0,0 +1,3 @@ +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.22.0")) +load("nco/4.9.5") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/aqm_lbcs.local.lua b/modulefiles/tasks/derecho/aqm_lbcs.local.lua new file mode 100644 index 0000000000..de700dbbc6 --- /dev/null +++ b/modulefiles/tasks/derecho/aqm_lbcs.local.lua @@ -0,0 +1,3 @@ +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.22.0")) +load("nco/4.9.5") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/fire_emission.local.lua b/modulefiles/tasks/derecho/fire_emission.local.lua new file mode 100644 index 0000000000..f8a53ee074 --- /dev/null +++ b/modulefiles/tasks/derecho/fire_emission.local.lua @@ -0,0 +1,2 @@ +load("ncarenv") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/derecho/miniconda_regional_workflow_cmaq.lua new file mode 100644 index 0000000000..1ecbec291f --- /dev/null +++ b/modulefiles/tasks/derecho/miniconda_regional_workflow_cmaq.lua @@ -0,0 +1,6 @@ +unload("python") +load("conda") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/nexus_emission.local.lua b/modulefiles/tasks/derecho/nexus_emission.local.lua new file mode 100644 index 0000000000..b94435c230 --- /dev/null +++ b/modulefiles/tasks/derecho/nexus_emission.local.lua @@ -0,0 +1,4 @@ +load("nco/5.0.6") + +load("ncarenv") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/nexus_gfs_sfc.local.lua b/modulefiles/tasks/derecho/nexus_gfs_sfc.local.lua new file mode 100644 index 0000000000..f8a53ee074 --- /dev/null +++ b/modulefiles/tasks/derecho/nexus_gfs_sfc.local.lua @@ -0,0 +1,2 @@ +load("ncarenv") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/nexus_post_split.local.lua b/modulefiles/tasks/derecho/nexus_post_split.local.lua new file mode 100644 index 0000000000..8874da32a8 --- /dev/null +++ b/modulefiles/tasks/derecho/nexus_post_split.local.lua @@ -0,0 +1,3 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) +load("ncarenv") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/plot_allvars.local.lua b/modulefiles/tasks/derecho/plot_allvars.local.lua new file mode 100644 index 0000000000..2263141a0a --- /dev/null +++ b/modulefiles/tasks/derecho/plot_allvars.local.lua @@ -0,0 +1,5 @@ +unload("python") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/derecho/point_source.local.lua b/modulefiles/tasks/derecho/point_source.local.lua new file mode 100644 index 0000000000..f8a53ee074 --- /dev/null +++ b/modulefiles/tasks/derecho/point_source.local.lua @@ -0,0 +1,2 @@ +load("ncarenv") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/pre_post_stat.local.lua b/modulefiles/tasks/derecho/pre_post_stat.local.lua new file mode 100644 index 0000000000..262919b475 --- /dev/null +++ b/modulefiles/tasks/derecho/pre_post_stat.local.lua @@ -0,0 +1,2 @@ +load("nco/4.9.5") +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/derecho/python_srw.lua b/modulefiles/tasks/derecho/python_srw.lua new file mode 100644 index 0000000000..29c290813e --- /dev/null +++ b/modulefiles/tasks/derecho/python_srw.lua @@ -0,0 +1,5 @@ +unload("python") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/derecho/run_vx.local.lua b/modulefiles/tasks/derecho/run_vx.local.lua new file mode 100644 index 0000000000..5979a8db96 --- /dev/null +++ b/modulefiles/tasks/derecho/run_vx.local.lua @@ -0,0 +1,6 @@ +--[[ +Compiler-specific modules are used for met and metplus libraries +--]] +load(pathJoin("met", os.getenv("met_ver") or "10.1.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3")) +load("python_srw") diff --git a/modulefiles/wflow_derecho.lua b/modulefiles/wflow_derecho.lua new file mode 100644 index 0000000000..d6e1cf2f17 --- /dev/null +++ b/modulefiles/wflow_derecho.lua @@ -0,0 +1,24 @@ +help([[ +This module loads python environement for running the UFS SRW App on +on the CISL machine Cheyenne +]]) + +whatis([===[Loads libraries needed for running the UFS SRW App on Cheyenne ]===]) + +load("ncarenv") + +append_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/rocoto/modulefiles") +load("rocoto") + +unload("python") + +load("set_pythonpath") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate workflow_tools +]===]) +end + diff --git a/tests/README.md b/tests/README.md index 0a0c58c842..3670d50635 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,9 +15,9 @@ for the current code in the users ufs-srweather-app directory. It consists of t Currently, the following configurations are supported: -Machine | Cheyenne | Hera | Jet | Orion | wcoss2 | -------------| ------------|--------|--------|--------|---------| -Compiler(s) | Intel, GNU | Intel | Intel | Intel | Intel | +Machine | Derecho | Cheyenne | Hera | Jet | Orion | wcoss2 | +------------|---------|-------------|--------|--------|--------|---------| +Compiler(s) | Intel | Intel, GNU | Intel | Intel | Intel | Intel | The CMake build is done in the ``build_${compiler}`` directory. The executables for each build are installed under the ``bin_${compiler}`` directory. diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho new file mode 100644 index 0000000000..239ad2a1f4 --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.derecho @@ -0,0 +1,55 @@ +community +custom_ESGgrid +custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE +custom_GFDLgrid +deactivate_tasks +grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 +grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot +GST_release_public_v1 +MET_ensemble_verification_only_vx +MET_verification_only_vx +nco +nco_ensemble +nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16 +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR +pregen_grid_orog_sfc_climo +specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS +specify_template_filenames diff --git a/tests/WE2E/machine_suites/coverage.derecho b/tests/WE2E/machine_suites/coverage.derecho new file mode 100644 index 0000000000..25194b2c7d --- /dev/null +++ b/tests/WE2E/machine_suites/coverage.derecho @@ -0,0 +1,10 @@ +community +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot +nco_ensemble +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 682334952e..8d63b5f8dd 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -45,7 +45,7 @@ function usage { } -machines=( hera jet cheyenne orion wcoss2 gaea odin singularity macos noaacloud ) +machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud ) if [ "$1" = "-h" ] ; then usage ; fi [[ $# -le 2 ]] && usage diff --git a/tests/build.sh b/tests/build.sh index eccfca8830..76aaa8a6e1 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne orion wcoss2 gaea odin singularity macos noaacloud ) +machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud ) [[ $# -gt 4 ]] && usage diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index acc222822a..9f2516ed46 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -58,7 +58,7 @@ def get_crontab_contents(called_from_cron): # themselves being called as cron jobs. In that case, we must instead # call the system version of crontab at /usr/bin/crontab. # - if MACHINE == "CHEYENNE": + if MACHINE == "CHEYENNE" or "DERECHO": if called_from_cron: __crontab_cmd__ = "/usr/bin/crontab" diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml new file mode 100644 index 0000000000..9879323be4 --- /dev/null +++ b/ush/machine/derecho.yaml @@ -0,0 +1,35 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 128 + SCHED: pbspro + TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + DOMAIN_PREGEN_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/FV3LAM_pregen + QUEUE_DEFAULT: main + QUEUE_FCST: main + QUEUE_HPSS: main + RUN_CMD_FCST: mpiexec -n ${PE_MEMBER01} --hostfile ${PBS_NODEFILE} + RUN_CMD_POST: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} + RUN_CMD_PRDGEN: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} + RUN_CMD_NEXUS: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} + RUN_CMD_AQMLBC: mpiexec -n ${NUMTS} --hostfile ${PBS_NODEFILE} + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/aqm_data + TEST_PREGEN_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/FV3LAM_pregen + TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir + TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir + TEST_VX_FCST_INPUT_BASEDIR: '{{ "/glade/work/epicufsrt/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' + FIXaer: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_aer + FIXgsi: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_gsi + FIXgsm: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_am + FIXlut: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_lut + FIXorg: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_orog + FIXsfc: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo + FIXshp: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/NaturalEarth +data: + ics_lbcs: + FV3GFS: /glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh} diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 81eed41853..559fea70a0 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR: ["nco", "community"] valid_vals_VERBOSE: [True, False] valid_vals_DEBUG: [True, False] -valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] +valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] valid_vals_FCST_MODEL: ["ufs-weather-model"] valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"] From b88b9cc87a3cbd68f943ffd5745c94927eea3d5d Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 30 Aug 2023 13:57:12 -0600 Subject: [PATCH 02/13] Modulefiles and configurations for Derecho system --- etc/lmod-setup.csh | 7 +-- etc/lmod-setup.sh | 8 +-- etc/lmod-setup.sh_orig | 75 +++++++++++++++++++++++++++++ modulefiles/build_derecho_intel.lua | 5 +- modulefiles/wflow_derecho.lua | 4 +- ush/machine/derecho.yaml | 7 +-- 6 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 etc/lmod-setup.sh_orig diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 85ea8d073b..a4f3083c97 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -41,12 +41,7 @@ else if ( "$L_MACHINE" == gaea ) then source /lustre/f2/dev/role.epic/contrib/Lmod_init.csh else if ( "$L_MACHINE" == derecho ) then - module purge > /dev/null - setenv ENV "/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/csh" - source $ENV - setenv BASH_ENV "/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/bash" - setenv PATH "$MODULESHOME/libexec:$MODULESHOME/init/ksh_funcs:$PATH" - module --initial_load --no_redirect restore + module reset else if ( "$L_MACHINE" == odin ) then module unload modules diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index e22d11c3dc..62e9f292be 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -48,13 +48,7 @@ elif [ "$L_MACHINE" = gaea ]; then source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh elif [ "$L_MACHINE" = derecho ]; then - module purge 2>/dev/null - - export BASH_ENV=/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/bash - source $BASH_ENV - export LMOD_SYSTEM_DEFAULT_MODULES="ncarenv:craype/2.7.20" - export PATH=$MODULESHOME/libexec:$MODULESHOME/init/ksh_funcs:$PATH - module --initial_load --no_redirect restore + module reset elif [ "$L_MACHINE" = odin ]; then module unload modules diff --git a/etc/lmod-setup.sh_orig b/etc/lmod-setup.sh_orig new file mode 100644 index 0000000000..e22d11c3dc --- /dev/null +++ b/etc/lmod-setup.sh_orig @@ -0,0 +1,75 @@ +#!/bin/sh + +if [ $# = 0 ]; then + L_MACHINE=${MACHINE} + cat << EOF_USAGE +Usage: source etc/lmod-setup.sh PLATFORM + +OPTIONS: + PLATFORM - name of machine you are building on + (e.g. derecho | hera | jet | orion | wcoss2 ) +EOF_USAGE + exit 1 +else + L_MACHINE=$1 +fi + +if [ "$L_MACHINE" != wcoss2 ]; then + [[ ${SHELLOPTS} =~ nounset ]] && has_mu=true || has_mu=false + [[ ${SHELLOPTS} =~ errexit ]] && has_me=true || has_me=false + $has_mu && set +u + $has_me && set +e + source /etc/profile + $has_mu && set -u + $has_me && set -e +fi + +if [ "$L_MACHINE" = macos ]; then + arch=$(uname -m) + [[ "$arch" = arm64 ]] && export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" + [[ "$arch" = x86_64 ]] && export BASH_ENV="/usr/local/opt/lmod/init/bash" + source $BASH_ENV + + module purge + +elif [ "$L_MACHINE" = linux ]; then + export BASH_ENV="/usr/share/lmod/lmod/init/bash" + source $BASH_ENV + + module purge + +elif [ "$L_MACHINE" = singularity ]; then + export BASH_ENV="/usr/share/lmod/lmod/init/bash" + source $BASH_ENV + + module purge + +elif [ "$L_MACHINE" = gaea ]; then + source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh + +elif [ "$L_MACHINE" = derecho ]; then + module purge 2>/dev/null + + export BASH_ENV=/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/bash + source $BASH_ENV + export LMOD_SYSTEM_DEFAULT_MODULES="ncarenv:craype/2.7.20" + export PATH=$MODULESHOME/libexec:$MODULESHOME/init/ksh_funcs:$PATH + module --initial_load --no_redirect restore + +elif [ "$L_MACHINE" = odin ]; then + module unload modules + unset -f module + + export BASH_ENV="/usr/local/lmod/8.3.1/init/bash" + source $BASH_ENV + + export LMOD_SYSTEM_DEFAULT_MODULES="PrgEnv-intel:cray-mpich:intel:craype" + module --initial_load --no_redirect restore + export MODULEPATH="/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" + +elif [ "$L_MACHINE" = wcoss2 ]; then + module reset + +else + module purge +fi diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua index e8ba15fdc5..7c7f3c771e 100644 --- a/modulefiles/build_derecho_intel.lua +++ b/modulefiles/build_derecho_intel.lua @@ -1,6 +1,6 @@ help([[ This module loads libraries for building the UFS SRW App on -the CISL machine Cheyenne using Intel-2022.1 +the CISL machine Derecho (Cray) using Intel-classic-2023.0.0 ]]) whatis([===[Loads libraries needed for building the UFS SRW App on Cheyenne ]===]) @@ -11,7 +11,8 @@ load(pathJoin("craype", os.getenv("craype_ver") or "2.7.20")) unload("netcdf") unload("hdf5") -unload("intel") +load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2023.0.0")) +load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "8.1.25")) prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/hpc-stack/intel-classic-2023.0.0/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) diff --git a/modulefiles/wflow_derecho.lua b/modulefiles/wflow_derecho.lua index d6e1cf2f17..8a71ed6cd7 100644 --- a/modulefiles/wflow_derecho.lua +++ b/modulefiles/wflow_derecho.lua @@ -1,9 +1,9 @@ help([[ This module loads python environement for running the UFS SRW App on -on the CISL machine Cheyenne +on the CISL machine Derecho (Cray) ]]) -whatis([===[Loads libraries needed for running the UFS SRW App on Cheyenne ]===]) +whatis([===[Loads libraries for running the UFS SRW Workflow on Derecho ]===]) load("ncarenv") diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 9879323be4..481b4ba2cc 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro - TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc - TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc - TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/FV3LAM_pregen QUEUE_DEFAULT: main QUEUE_FCST: main @@ -30,6 +30,7 @@ platform: FIXorg: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_orog FIXsfc: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/NaturalEarth + EXTRN_MDL_DATA_STORES: disk aws nomads data: ics_lbcs: FV3GFS: /glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh} From fd690f3b7f74327b86b63e21c95ed4a6c6032ed9 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 30 Aug 2023 15:23:32 -0600 Subject: [PATCH 03/13] remove lmod-setup.sh_orig --- etc/lmod-setup.sh_orig | 75 ------------------------------------------ 1 file changed, 75 deletions(-) delete mode 100644 etc/lmod-setup.sh_orig diff --git a/etc/lmod-setup.sh_orig b/etc/lmod-setup.sh_orig deleted file mode 100644 index e22d11c3dc..0000000000 --- a/etc/lmod-setup.sh_orig +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh - -if [ $# = 0 ]; then - L_MACHINE=${MACHINE} - cat << EOF_USAGE -Usage: source etc/lmod-setup.sh PLATFORM - -OPTIONS: - PLATFORM - name of machine you are building on - (e.g. derecho | hera | jet | orion | wcoss2 ) -EOF_USAGE - exit 1 -else - L_MACHINE=$1 -fi - -if [ "$L_MACHINE" != wcoss2 ]; then - [[ ${SHELLOPTS} =~ nounset ]] && has_mu=true || has_mu=false - [[ ${SHELLOPTS} =~ errexit ]] && has_me=true || has_me=false - $has_mu && set +u - $has_me && set +e - source /etc/profile - $has_mu && set -u - $has_me && set -e -fi - -if [ "$L_MACHINE" = macos ]; then - arch=$(uname -m) - [[ "$arch" = arm64 ]] && export BASH_ENV="/opt/homebrew/opt/lmod/init/bash" - [[ "$arch" = x86_64 ]] && export BASH_ENV="/usr/local/opt/lmod/init/bash" - source $BASH_ENV - - module purge - -elif [ "$L_MACHINE" = linux ]; then - export BASH_ENV="/usr/share/lmod/lmod/init/bash" - source $BASH_ENV - - module purge - -elif [ "$L_MACHINE" = singularity ]; then - export BASH_ENV="/usr/share/lmod/lmod/init/bash" - source $BASH_ENV - - module purge - -elif [ "$L_MACHINE" = gaea ]; then - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - -elif [ "$L_MACHINE" = derecho ]; then - module purge 2>/dev/null - - export BASH_ENV=/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/bash - source $BASH_ENV - export LMOD_SYSTEM_DEFAULT_MODULES="ncarenv:craype/2.7.20" - export PATH=$MODULESHOME/libexec:$MODULESHOME/init/ksh_funcs:$PATH - module --initial_load --no_redirect restore - -elif [ "$L_MACHINE" = odin ]; then - module unload modules - unset -f module - - export BASH_ENV="/usr/local/lmod/8.3.1/init/bash" - source $BASH_ENV - - export LMOD_SYSTEM_DEFAULT_MODULES="PrgEnv-intel:cray-mpich:intel:craype" - module --initial_load --no_redirect restore - export MODULEPATH="/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" - -elif [ "$L_MACHINE" = wcoss2 ]; then - module reset - -else - module purge -fi From 18e87b8ac25ed353a0001d8b011bd4eb5e9258e3 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 30 Aug 2023 15:29:31 -0600 Subject: [PATCH 04/13] update srw_common.lua --- modulefiles/srw_common.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 96541c9960..5ee8ee7797 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -5,7 +5,7 @@ load("libpng/1.6.37") load_any("netcdf/4.9.2", "netcdf-c/4.9.2") load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") load_any("pio/2.5.10", "parallelio/2.5.10") -load_any("esmf/8.4.2", "esmf/8.5.0") +load("esmf/8.4.2") load("fms/2023.01") load("bufr/11.7.0") From 4fb57dcfda255a1fe54b4e55fe670672b55e675d Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 30 Aug 2023 15:33:47 -0600 Subject: [PATCH 05/13] update lmod-setup.* for Derecho --- etc/lmod-setup.csh | 2 +- etc/lmod-setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index a4f3083c97..fd1f7f8c9c 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -6,7 +6,7 @@ Usage: source etc/lmod-setup.csh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | derecho | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | hera | jet | orion | wcoss2 ) EOF_USAGE exit 1 else diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 62e9f292be..0c922abcdb 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -7,7 +7,7 @@ Usage: source etc/lmod-setup.sh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. derecho | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | hera | jet | orion | wcoss2 ) EOF_USAGE exit 1 else From 787863211e19966b517a693f91f01677d0f29f74 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:58:20 -0400 Subject: [PATCH 06/13] Update ush/get_crontab_contents.py Thank you, @mkavulich! Co-authored-by: Michael Kavulich --- ush/get_crontab_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 9f2516ed46..a9aa2b092d 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -58,7 +58,7 @@ def get_crontab_contents(called_from_cron): # themselves being called as cron jobs. In that case, we must instead # call the system version of crontab at /usr/bin/crontab. # - if MACHINE == "CHEYENNE" or "DERECHO": + if MACHINE == "CHEYENNE" or MACHINE == "DERECHO": if called_from_cron: __crontab_cmd__ = "/usr/bin/crontab" From 76c3e6ff03ce962508a6db0ddfa99a58bf6e5da5 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 13 Sep 2023 18:18:48 -0600 Subject: [PATCH 07/13] update coverage.derecho --- tests/WE2E/machine_suites/coverage.derecho | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/WE2E/machine_suites/coverage.derecho b/tests/WE2E/machine_suites/coverage.derecho index 25194b2c7d..9c47da1030 100644 --- a/tests/WE2E/machine_suites/coverage.derecho +++ b/tests/WE2E/machine_suites/coverage.derecho @@ -1,10 +1,9 @@ -community -grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot -nco_ensemble -nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR +#nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16 +pregen_grid_orog_sfc_climo +specify_template_filenames From 81af6a9bb6ab3e44e202cb4ec80759f13e2c27a4 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 13 Sep 2023 19:16:43 -0600 Subject: [PATCH 08/13] changes for derecho --- tests/WE2E/machine_suites/comprehensive.derecho | 3 +++ ush/machine/derecho.yaml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho index 239ad2a1f4..952a7de642 100644 --- a/tests/WE2E/machine_suites/comprehensive.derecho +++ b/tests/WE2E/machine_suites/comprehensive.derecho @@ -1,8 +1,11 @@ +2020_CAD community custom_ESGgrid custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE custom_GFDLgrid deactivate_tasks +get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems +get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 481b4ba2cc..acd5c82dca 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro - CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/FV3LAM_pregen QUEUE_DEFAULT: main QUEUE_FCST: main @@ -33,4 +33,10 @@ platform: EXTRN_MDL_DATA_STORES: disk aws nomads data: ics_lbcs: - FV3GFS: /glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh} + FV3GFS: + nemsio: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh} + grib2: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} + NAM: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/NAM/${yyyymmdd}${hh} + HRRR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} + RAP: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} + GSMGFS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} From 4b21569ffd9600ded530d0308e9dc75cb789c57a Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 13 Sep 2023 19:41:31 -0600 Subject: [PATCH 09/13] changes for derecho machine_suites --- .../WE2E/machine_suites/comprehensive.derecho | 59 +------------------ tests/WE2E/machine_suites/coverage.derecho | 4 +- 2 files changed, 3 insertions(+), 60 deletions(-) mode change 100644 => 120000 tests/WE2E/machine_suites/comprehensive.derecho diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho deleted file mode 100644 index 952a7de642..0000000000 --- a/tests/WE2E/machine_suites/comprehensive.derecho +++ /dev/null @@ -1,58 +0,0 @@ -2020_CAD -community -custom_ESGgrid -custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE -custom_GFDLgrid -deactivate_tasks -get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems -get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS -grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot -grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP -grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 -grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta -grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP -grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 -grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 -grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 -grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot -GST_release_public_v1 -MET_ensemble_verification_only_vx -MET_verification_only_vx -nco -nco_ensemble -nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16 -nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km -nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR -pregen_grid_orog_sfc_climo -specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS -specify_template_filenames diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho new file mode 120000 index 0000000000..18a56d9739 --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.derecho @@ -0,0 +1 @@ +comprehensive.orion \ No newline at end of file diff --git a/tests/WE2E/machine_suites/coverage.derecho b/tests/WE2E/machine_suites/coverage.derecho index 9c47da1030..19bbc623c7 100644 --- a/tests/WE2E/machine_suites/coverage.derecho +++ b/tests/WE2E/machine_suites/coverage.derecho @@ -1,9 +1,9 @@ -custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE +custom_ESGgrid_IndianOcean_6km grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR -#nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16 +nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16 pregen_grid_orog_sfc_climo specify_template_filenames From a0baf5181300d2868c4c9274d96020ec29f12382 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 13 Sep 2023 19:50:22 -0600 Subject: [PATCH 10/13] updated files under wflow/ to remove nodesize parameter --- parm/wflow/aqm_post.yaml | 1 - parm/wflow/aqm_prep.yaml | 1 - parm/wflow/coldstart.yaml | 2 -- parm/wflow/da_data_preproc.yaml | 1 - parm/wflow/plot.yaml | 1 - parm/wflow/post.yaml | 1 - parm/wflow/prdgen.yaml | 1 - parm/wflow/prep.yaml | 1 - parm/wflow/verify_det.yaml | 1 - parm/wflow/verify_ens.yaml | 1 - parm/wflow/verify_pre.yaml | 1 - 11 files changed, 12 deletions(-) diff --git a/parm/wflow/aqm_post.yaml b/parm/wflow/aqm_post.yaml index 4016c8190c..31b7b34848 100644 --- a/parm/wflow/aqm_post.yaml +++ b/parm/wflow/aqm_post.yaml @@ -15,7 +15,6 @@ default_aqm_task: &default_aqm native: '{{ platform.SCHED_NATIVE_CMD }}' nnodes: 1 nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: "&NCORES_PER_NODE;" partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/aqm_prep.yaml b/parm/wflow/aqm_prep.yaml index 43c1e81aa1..36d2c8af2a 100644 --- a/parm/wflow/aqm_prep.yaml +++ b/parm/wflow/aqm_prep.yaml @@ -15,7 +15,6 @@ default_aqm_task: &default_aqm native: '{{ platform.SCHED_NATIVE_CMD }}' nnodes: 1 nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: "&NCORES_PER_NODE;" partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index c1935f2160..1c4beba0b7 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -14,7 +14,6 @@ default_task: &default_task ENSMEM_INDX: '#mem#' native: '{{ platform.SCHED_NATIVE_CMD }}' nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: "&NCORES_PER_NODE;" partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' queue: '&QUEUE_DEFAULT;' walltime: 00:30:00 @@ -145,7 +144,6 @@ metatask_run_ensemble: SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;' nprocs: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - nodesize: '&NCORES_PER_NODE;' nnodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}' partition: '{% if platform.get("PARTITION_FCST") %}&PARTITION_FCST;{% else %}None{% endif %}' ppn: '{{ task_run_fcst.PPN_RUN_FCST // 1 }}' diff --git a/parm/wflow/da_data_preproc.yaml b/parm/wflow/da_data_preproc.yaml index c609fb967a..4010b8a016 100644 --- a/parm/wflow/da_data_preproc.yaml +++ b/parm/wflow/da_data_preproc.yaml @@ -18,7 +18,6 @@ default_data_preproc_task: &default_preproc native: '{{ platform.SCHED_NATIVE_CMD }}' nodes: '{{ nnodes }}:ppn={{ ppn }}' nnodes: 1 - nodesize: "&NCORES_PER_NODE;" ppn: 1 partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/plot.yaml b/parm/wflow/plot.yaml index 7b3eeff15d..6dad3e0dfa 100644 --- a/parm/wflow/plot.yaml +++ b/parm/wflow/plot.yaml @@ -19,7 +19,6 @@ default_task_plot: &default_task native: '{{ platform.SCHED_NATIVE_CMD }}' nnodes: 1 nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: "&NCORES_PER_NODE;" partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 24 queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/post.yaml b/parm/wflow/post.yaml index ae02b4025b..5672e7343f 100644 --- a/parm/wflow/post.yaml +++ b/parm/wflow/post.yaml @@ -18,7 +18,6 @@ default_task_post: &default_task nodes: '{{ nnodes }}:ppn={{ ppn }}' nnodes: 2 ppn: 24 - nodesize: "&NCORES_PER_NODE;" partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' queue: '&QUEUE_DEFAULT;' walltime: 00:15:00 diff --git a/parm/wflow/prdgen.yaml b/parm/wflow/prdgen.yaml index c88455ad4c..6b9f7cd4f6 100644 --- a/parm/wflow/prdgen.yaml +++ b/parm/wflow/prdgen.yaml @@ -23,7 +23,6 @@ metatask_run_prdgen: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' native: '{{ platform.SCHED_NATIVE_CMD }}' nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: "&NCORES_PER_NODE;" nnodes: 1 partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 22 diff --git a/parm/wflow/prep.yaml b/parm/wflow/prep.yaml index bd0915f87a..c9d5549909 100644 --- a/parm/wflow/prep.yaml +++ b/parm/wflow/prep.yaml @@ -17,7 +17,6 @@ default_task_prep: &default_task native: '{{ platform.SCHED_NATIVE_CMD }}' nodes: '{{ nnodes }}:ppn={{ ppn }}' nnodes: 1 - nodesize: "&NCORES_PER_NODE;" ppn: 24 partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index c39a872d5e..3c477e88f9 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -16,7 +16,6 @@ default_task_verify_det: &default_task_verify_det native: '{{ platform.SCHED_NATIVE_CMD }}' nnodes: 1 nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: '&NCORES_PER_NODE;' partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index c4d455461c..e3b1a6455b 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -16,7 +16,6 @@ default_task_verify_ens: &default_task_verify_ens native: '{{ platform.SCHED_NATIVE_CMD }}' nnodes: 1 nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: '&NCORES_PER_NODE;' partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_DEFAULT;' diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml index e46474c361..ccc8e0d11e 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/verify_pre.yaml @@ -16,7 +16,6 @@ default_task_verify_pre: &default_task_verify_pre native: '{{ platform.SCHED_NATIVE_CMD }}' nnodes: 1 nodes: '{{ nnodes }}:ppn={{ ppn }}' - nodesize: '&NCORES_PER_NODE;' partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_DEFAULT;' From fa080fd505996bab4e9dc574e44765601302c72d Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 13 Sep 2023 20:04:09 -0600 Subject: [PATCH 11/13] correcting get_crontab_contents.py --- ush/get_crontab_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 86741bb13d..5e9ccd6fa8 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -35,7 +35,7 @@ def get_crontab_contents(called_from_cron, machine, debug): # themselves being called as cron jobs. In that case, we must instead # call the system version of crontab at /usr/bin/crontab. # - if MACHINE == "CHEYENNE" or MACHINE == "DERECHO": + if machine == "CHEYENNE" or machine == "DERECHO": if called_from_cron: crontab_cmd = "/usr/bin/crontab" else: From d6ce2d3e2d55631c606a77a6ed437d3b806fe499 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 15 Sep 2023 11:58:26 -0600 Subject: [PATCH 12/13] update ush/machine/derecho.yaml --- ush/machine/derecho.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index acd5c82dca..7109a40274 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -30,7 +30,7 @@ platform: FIXorg: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_orog FIXsfc: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/NaturalEarth - EXTRN_MDL_DATA_STORES: disk aws nomads + EXTRN_MDL_DATA_STORES: aws data: ics_lbcs: FV3GFS: From 559c5a60db4aef35f284801b4a2cc30eab66274e Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Sun, 17 Sep 2023 07:09:53 -0600 Subject: [PATCH 13/13] update derecho.yaml for staged data locations --- ush/machine/derecho.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 7109a40274..c7e144b201 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -9,13 +9,13 @@ platform: QUEUE_DEFAULT: main QUEUE_FCST: main QUEUE_HPSS: main - RUN_CMD_FCST: mpiexec -n ${PE_MEMBER01} --hostfile ${PBS_NODEFILE} - RUN_CMD_POST: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} - RUN_CMD_PRDGEN: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} + RUN_CMD_FCST: mpiexec -n ${PE_MEMBER01} + RUN_CMD_POST: mpiexec -n $nprocs + RUN_CMD_PRDGEN: mpiexec -n $nprocs RUN_CMD_SERIAL: time - RUN_CMD_UTILS: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} - RUN_CMD_NEXUS: mpiexec -n $nprocs --hostfile ${PBS_NODEFILE} - RUN_CMD_AQMLBC: mpiexec -n ${NUMTS} --hostfile ${PBS_NODEFILE} + RUN_CMD_UTILS: mpiexec -n $nprocs + RUN_CMD_NEXUS: mpiexec -n $nprocs + RUN_CMD_AQMLBC: mpiexec -n ${NUMTS} PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data TEST_AQM_INPUT_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/aqm_data @@ -36,6 +36,7 @@ data: FV3GFS: nemsio: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh} grib2: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} + netcdf: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh} NAM: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/NAM/${yyyymmdd}${hh} HRRR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} RAP: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}