diff --git a/env/S4.env b/env/S4.env index 25e335e334..0d11cea02e 100755 --- a/env/S4.env +++ b/env/S4.env @@ -1,10 +1,11 @@ -#!/usr/bin/env bash +#! /usr/bin/env bash if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" echo "atmanalrun atmensanalrun" + echo "aeroanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -30,9 +31,8 @@ export NTHSTACK=1024000000 ulimit -s unlimited ulimit -a -if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then +if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then - npe_node_prep=${npe_node_prep:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_prep)) export POE="NO" @@ -40,199 +40,195 @@ if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then export sys_tp="S4" export launcher_PREP="srun" -elif [[ ${step} = "waveinit" || ${step} = "waveprep" || ${step} = "wavepostsbs" || ${step} = "wavepostbndpnt" || ${step} = "wavepostbndpntbll" || ${step} = "wavepostpnt" ]]; then +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then export CFP_MP="YES" - if [[ ${step} = "waveprep" ]]; then export MP_PULSE=0 ; fi + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} -elif [[ ${step} = "atmanalrun" ]]; then +elif [[ "${step}" = "atmanalrun" ]]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - npe_node_atmanalrun=${npe_node_atmanalrun:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_atmanalrun)) export NTHREADS_ATMANAL=${nth_atmanalrun:-${nth_max}} [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} - export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun:-0}" + export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}" -elif [[ ${step} = "atmensanalrun" ]]; then +elif [[ "${step}" = "atmensanalrun" ]]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - npe_node_atmensanalrun=${npe_node_atmensanalrun:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_atmensanalrun)) export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} - export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun:-0}" + export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}" -elif [[ ${step} = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlrun" ]]; then - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - npe_node_aeroanlrun=${npe_node_aeroanlrun:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_aeroanlrun)) export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun:-0}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" -elif [[ ${step} = "anal" || ${step} = "analcalc" ]]; then +elif [[ "${step}" = "ocnanalbmat" ]]; then + echo "WARNING: ${step} is not enabled on S4!" + +elif [[ "${step}" = "ocnanalrun" ]]; then + echo "WARNING: ${step} is not enabled on S4!" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - npe_node_anal=${npe_node_anal:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_anal)) export NTHREADS_GSI=${nth_anal:-${nth_max}} [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max} - export APRUN_GSI=${launcher} + export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max} - export APRUN_CALCINC=${launcher} + export APRUN_CALCINC="${launcher} \$ncmd" export NTHREADS_CYCLE=${nth_cycle:-12} [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} npe_cycle=${ntiles:-6} - export APRUN_CYCLE="${launcher} -n ${npe_cycle:-0}" + export APRUN_CYCLE="${launcher} -n ${npe_cycle}" export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl:-0}" + export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}" -elif [[ ${step} = "sfcanl" ]]; then - npe_node_sfcanl=${npe_node_sfcanl:-${npe_node_max}} +elif [[ "${step}" = "sfcanl" ]]; then nth_max=$((npe_node_max / npe_node_sfcanl)) export NTHREADS_CYCLE=${nth_sfcanl:-14} [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} npe_sfcanl=${ntiles:-6} - export APRUN_CYCLE="${launcher} -n ${npe_sfcanl:-0}" + export APRUN_CYCLE="${launcher} -n ${npe_sfcanl}" -elif [[ ${step} = "gldas" ]]; then +elif [[ "${step}" = "gldas" ]]; then + + export USE_CFP="NO" + export CFP_MP="YES" - npe_node_gldas=${npe_node_gldas:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_gldas)) export NTHREADS_GLDAS=${nth_gldas:-${nth_max}} [[ ${NTHREADS_GLDAS} -gt ${nth_max} ]] && export NTHREADS_GLDAS=${nth_max} - export APRUN_GLDAS="${launcher} -n ${npe_gldas:-0}" + export APRUN_GLDAS="${launcher} -n ${npe_gldas}" export NTHREADS_GAUSSIAN=${nth_gaussian:-1} [[ ${NTHREADS_GAUSSIAN} -gt ${nth_max} ]] && export NTHREADS_GAUSSIAN=${nth_max} - export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian:-0}" + export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian}" # Must run data processing with exactly the number of tasks as time # periods being processed. - gldas_spinup_hours=${gldas_spinup_hours:-0} npe_gldas_data_proc=$((gldas_spinup_hours + 12)) - export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} --multi-prog" + export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} ${mpmd_opt}" -elif [[ ${step} = "eobs" ]]; then +elif [[ "${step}" = "eobs" ]]; then export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - - npe_node_eobs=${npe_node_eobs:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_eobs)) export NTHREADS_GSI=${nth_eobs:-${nth_max}} [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max} - export APRUN_GSI=${launcher} - -elif [[ ${step} = "eupd" ]]; then + export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}}" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "eupd" ]]; then - npe_node_eupd=${npe_node_eupd:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_eupd)) export NTHREADS_ENKF=${nth_eupd:-${nth_max}} [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max} - export APRUN_ENKF=${launcher} + export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" -elif [[ ${step} = "fcst" ]]; then +elif [[ "${step}" = "fcst" ]]; then #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs - if [[ ${CDUMP:-gdas} == "gfs" ]]; then - npe_fcst=${npe_fcst_gfs:-0} - npe_node_fcst=${npe_node_fcst_gfs:-${npe_node_max}} - nth_fv3=${nth_fv3_gfs:-1} + if [[ ${CDUMP} == "gfs" ]]; then + npe_fcst=${npe_fcst_gfs} + npe_node_fcst=${npe_node_fcst_gfs} + nth_fv3=${nth_fv3_gfs} fi - npe_node_fcst=${npe_node_fcst:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-${nth_max}} [[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max} export cores_per_node=${npe_node_max} - export APRUN_FV3="${launcher} -n ${npe_fcst:-0}" + export APRUN_FV3="${launcher} -n ${npe_fcst}" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max} - export APRUN_REGRID_NEMSIO=${launcher} + export APRUN_REGRID_NEMSIO="${launcher} -n ${LEVS}" export NTHREADS_REMAP=${nth_remap:-2} [[ ${NTHREADS_REMAP} -gt ${nth_max} ]] && export NTHREADS_REMAP=${nth_max} - export APRUN_REMAP=${launcher} + export APRUN_REMAP="${launcher} -n ${npe_remap:-${npe_fcst}}" export I_MPI_DAPL_UD="enable" -elif [[ ${step} = "efcs" ]]; then +elif [[ "${step}" = "efcs" ]]; then - npe_node_efcs=${npe_node_efcs:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_efcs)) export NTHREADS_FV3=${nth_efcs:-${nth_max}} [[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max} export cores_per_node=${npe_node_max} - export APRUN_FV3="${launcher} -n ${npe_efcs:-0}" + export APRUN_FV3="${launcher} -n ${npe_efcs}" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max} - export APRUN_REGRID_NEMSIO="${launcher} ${LEVS:-128}" + export APRUN_REGRID_NEMSIO="${launcher} -n ${LEVS}" -elif [[ ${step} = "post" ]]; then +elif [[ "${step}" = "post" ]]; then - npe_node_post=${npe_node_post:-npe_node_max} nth_max=$((npe_node_max / npe_node_post)) export NTHREADS_NP=${nth_np:-1} [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} - export APRUN_NP=${launcher} + export APRUN_NP="${launcher} -n ${npe_post}" export NTHREADS_DWN=${nth_dwn:-1} [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} - export APRUN_DWN=${launcher} + export APRUN_DWN="${launcher} -n ${npe_dwn}" -elif [[ ${step} = "ecen" ]]; then +elif [[ "${step}" = "ecen" ]]; then - npe_node_ecen=${npe_node_ecen:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_ecen)) export NTHREADS_ECEN=${nth_ecen:-${nth_max}} [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max} - export APRUN_ECEN=${launcher} + export APRUN_ECEN="${launcher} -n ${npe_ecen}" export NTHREADS_CHGRES=${nth_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max} @@ -240,54 +236,51 @@ elif [[ ${step} = "ecen" ]]; then export NTHREADS_CALCINC=${nth_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max} - export APRUN_CALCINC=${launcher} + export APRUN_CALCINC="${launcher} -n ${npe_ecen}" -elif [[ ${step} = "esfc" ]]; then +elif [[ "${step}" = "esfc" ]]; then - npe_node_esfc=${npe_node_esfc:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_esfc)) export NTHREADS_ESFC=${nth_esfc:-${nth_max}} [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max} - export APRUN_ESFC="${launcher} -n ${npe_esfc:-0}" + export APRUN_ESFC="${launcher} -n ${npe_esfc}" export NTHREADS_CYCLE=${nth_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} - export APRUN_CYCLE="${launcher} -n ${npe_esfc:-0}" + export APRUN_CYCLE="${launcher} -n ${npe_esfc}" -elif [[ ${step} = "epos" ]]; then +elif [[ "${step}" = "epos" ]]; then - npe_node_epos=${npe_node_epos:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_epos)) export NTHREADS_EPOS=${nth_epos:-${nth_max}} [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max} - export APRUN_EPOS=${launcher} + export APRUN_EPOS="${launcher} -n ${npe_epos}" -elif [[ ${step} = "init" ]]; then +elif [[ "${step}" = "init" ]]; then - export APRUN=${launcher} + export APRUN="${launcher} -n ${npe_init}" -elif [[ ${step} = "postsnd" ]]; then +elif [[ "${step}" = "postsnd" ]]; then export CFP_MP="YES" - npe_node_postsnd=${npe_node_postsnd:-${npe_node_max}} nth_max=$((npe_node_max / npe_node_postsnd)) export NTHREADS_POSTSND=${nth_postsnd:-1} [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max} - export APRUN_POSTSND=${launcher} + export APRUN_POSTSND="${launcher} -n ${npe_postsnd}" export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max} - export APRUN_POSTSNDCFP=${launcher} + export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}" -elif [[ ${step} = "awips" ]]; then +elif [[ "${step}" = "awips" ]]; then echo "WARNING: ${step} is not enabled on S4!" -elif [[ ${step} = "gempak" ]]; then +elif [[ "${step}" = "gempak" ]]; then echo "WARNING: ${step} is not enabled on S4!" fi diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua index 93d95de6b9..dc1683ad42 100644 --- a/modulefiles/module_base.s4.lua +++ b/modulefiles/module_base.s4.lua @@ -24,8 +24,8 @@ load(pathJoin("png", "1.6.35")) load(pathJoin("hdf5", "1.10.6")) load(pathJoin("netcdf", "4.7.4")) -load(pathJoin("pio", "2.5.2")) -load(pathJoin("esmf", "8.2.1b04")) +load(pathJoin("pio", "2.5.7")) +load(pathJoin("esmf", "8.3.0b09")) load(pathJoin("fms", "2021.03")) load(pathJoin("bacio", "2.4.1")) diff --git a/parm/config/config.resources b/parm/config/config.resources index ac2e407d12..c6f7d2ddaa 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -704,7 +704,7 @@ elif [ ${step} = "ediag" ]; then export npe_ediag=48 export nth_ediag=1 export npe_node_ediag=$(echo "${npe_node_max} / ${nth_ediag}" | bc) - export memory_ediag="24GB" + export memory_ediag="30GB" elif [ ${step} = "eupd" ]; then diff --git a/scripts/exgdas_enkf_earc.sh b/scripts/exgdas_enkf_earc.sh index 4df4dc82c7..7ebc4a3db0 100755 --- a/scripts/exgdas_enkf_earc.sh +++ b/scripts/exgdas_enkf_earc.sh @@ -48,7 +48,7 @@ if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]]; SAVEWARMICB="NO" mm=$(echo "${CDATE}"|cut -c 5-6) dd=$(echo "${CDATE}"|cut -c 7-8) - nday=$(( (mm-1)*30+dd )) + nday=$(( (10#${mm}-1)*30+10#${dd} )) mod=$((nday % ARCH_WARMICFREQ)) if [ "${CDATE}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCINC_CYC}" ]; then SAVEWARMICA="YES" ; fi if [ "${CDATE}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ]; then SAVEWARMICB="YES" ; fi