Port the global workflow to Jet

env/JET.env

@@ -18,11 +18,13 @@ if [[ "${PARTITION_BATCH}" = "xjet" ]]; then
   export npe_node_max=24
 elif [[ "${PARTITION_BATCH}" = "vjet" ]]; then
   export npe_node_max=16
+elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then
+  export npe_node_max=40
 fi
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --epilog=/apps/local/bin/report-mem --export=ALL"
 export mpmd_opt="--multi-prog"
 
-# Configure STACK
+# Configure MPI environment
 export OMP_STACKSIZE=2048000
 export NTHSTACK=1024000000
 
@@ -34,46 +36,85 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     nth_max=$((npe_node_max / npe_node_prep))
 
     export POE="NO"
-    export BACK=${BACK:-"NO"}
+    export BACK="NO"
     export sys_tp="JET"
+    export launcher_PREP="srun"
 
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
+    export CFP_MP="YES"
     if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
 elif [[ "${step}" = "atmanalrun" ]]; then
 
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
     nth_max=$((npe_node_max / npe_node_atmanalrun))
 
     export NTHREADS_ATMANAL=${nth_atmanalrun:-${nth_max}}
     [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max}
-    export APRUN_ATMANAL="${launcher} ${npe_atmanalrun}"
+    export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}"
 
 elif [[ "${step}" = "atmensanalrun" ]]; then
 
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
     nth_max=$((npe_node_max / npe_node_atmensanalrun))
 
     export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}}
     [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max}
-    export APRUN_ATMENSANAL="${launcher} ${npe_atmensanalrun}"
+    export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
     nth_max=$((npe_node_max / npe_node_aeroanlrun))
 
     export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
-    export APRUN_AEROANL="${launcher} ${npe_aeroanlrun}"
+    export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
+
+elif [[ "${step}" = "ocnanalbmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}"
+
+elif [[ "${step}" = "ocnanalrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalrun))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
 
-elif [[ "${step}" = "anal" ]]; then
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     nth_max=$((npe_node_max / npe_node_anal))
 
-    export NTHREADS_GSI=${nth_gsi:-${nth_max}}
+    export NTHREADS_GSI=${nth_anal:-${nth_max}}
     [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
-    export APRUN_GSI="${launcher} ${npe_gsi:-${npe_anal:-${PBS_NP}}}"
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}"
 
     export NTHREADS_CALCINC=${nth_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
@@ -82,47 +123,67 @@ elif [[ "${step}" = "anal" ]]; then
     export NTHREADS_CYCLE=${nth_cycle:-12}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
     npe_cycle=${ntiles:-6}
-    export APRUN_CYCLE="${launcher} ${npe_cycle}"
+    export APRUN_CYCLE="${launcher} -n ${npe_cycle}"
 
     export NTHREADS_GAUSFCANL=1
     npe_gausfcanl=${npe_gausfcanl:-1}
-    export APRUN_GAUSFCANL="${launcher} ${npe_gausfcanl}"
+    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
 
 elif [[ "${step}" = "sfcanl" ]]; then
     nth_max=$((npe_node_max / npe_node_sfcanl))
 
     export NTHREADS_CYCLE=${nth_sfcanl:-14}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
     npe_sfcanl=${ntiles:-6}
-    export APRUN_CYCLE="${launcher} ${npe_sfcanl}"
+    export APRUN_CYCLE="${launcher} -n ${npe_sfcanl}"
 
 elif [[ "${step}" = "gldas" ]]; then
 
+    export USE_CFP="NO"
+    export CFP_MP="YES"
+
     nth_max=$((npe_node_max / npe_node_gldas))
 
     export NTHREADS_GLDAS=${nth_gldas:-${nth_max}}
     [[ ${NTHREADS_GLDAS} -gt ${nth_max} ]] && export NTHREADS_GLDAS=${nth_max}
-    export APRUN_GLDAS="${launcher} ${npe_gldas}"
+    export APRUN_GLDAS="${launcher} -n ${npe_gldas}"
 
     export NTHREADS_GAUSSIAN=${nth_gaussian:-1}
     [[ ${NTHREADS_GAUSSIAN} -gt ${nth_max} ]] && export NTHREADS_GAUSSIAN=${nth_max}
-    export APRUN_GAUSSIAN="${launcher} ${npe_gaussian}"
+    export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian}"
+
+# Must run data processing with exactly the number of tasks as time
+# periods being processed.
+
+    npe_gldas_data_proc=$((gldas_spinup_hours + 12))
+    export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} ${mpmd_opt}"
 
 elif [[ "${step}" = "eobs" ]]; then
 
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
     nth_max=$((npe_node_max / npe_node_eobs))
 
-    export NTHREADS_GSI=${nth_gsi:-${nth_max}}
+    export NTHREADS_GSI=${nth_eobs:-${nth_max}}
     [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
-    export APRUN_GSI="${launcher} ${npe_gsi:-${npe_eobs:-${PBS_NP}}}"
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
 elif [[ "${step}" = "eupd" ]]; then
 
     nth_max=$((npe_node_max / npe_node_eupd))
 
-    export NTHREADS_ENKF=${nth_enkf:-${nth_max}}
+    export NTHREADS_ENKF=${nth_eupd:-${nth_max}}
     [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max}
-    export APRUN_ENKF="${launcher} ${npe_enkf:-${npe_eupd:-${PBS_NP}}}"
+    export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
@@ -145,64 +206,65 @@ elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${nth_np:-1}
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP="${launcher}"
+    export APRUN_NP="${launcher} -n ${npe_post}"
 
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_NP="${launcher}"
+    export APRUN_DWN="${launcher} -n ${npe_dwn}"
 
 elif [[ "${step}" = "ecen" ]]; then
 
     nth_max=$((npe_node_max / npe_node_ecen))
 
     export NTHREADS_ECEN=${nth_ecen:-${nth_max}}
     [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max}
-    export APRUN_ECEN="${launcher} ${npe_ecen:-${PBS_NP}}"
+    export APRUN_ECEN="${launcher} -n ${npe_ecen}"
 
     export NTHREADS_CHGRES=${nth_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${nth_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
-    export APRUN_CALCINC="${launcher} ${npe_ecen:-${PBS_NP}}"
+    export APRUN_CALCINC="${launcher} -n ${npe_ecen}"
 
 elif [[ "${step}" = "esfc" ]]; then
 
     nth_max=$((npe_node_max / npe_node_esfc))
 
     export NTHREADS_ESFC=${nth_esfc:-${nth_max}}
     [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max}
-    export APRUN_ESFC="${launcher} ${npe_esfc:-${PBS_NP}}"
+    export APRUN_ESFC="${launcher} -n ${npe_esfc}"
 
-    export NTHREADS_CYCLE=${nth_cycle:-12}
+    export NTHREADS_CYCLE=${nth_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
-    export APRUN_CYCLE="${launcher} ${npe_esfc}"
-
+    export APRUN_CYCLE="${launcher} -n ${npe_esfc}"
 
 elif [[ "${step}" = "epos" ]]; then
 
     nth_max=$((npe_node_max / npe_node_epos))
 
     export NTHREADS_EPOS=${nth_epos:-${nth_max}}
     [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max}
-    export APRUN_EPOS="${launcher} ${npe_epos:-${PBS_NP}}"
+    export APRUN_EPOS="${launcher} -n ${npe_epos}"
 
 elif [[ "${step}" = "init" ]]; then
 
-    export APRUN="${launcher}"
+    export APRUN="${launcher} -n ${npe_init}"
 
 elif [[ "${step}" = "postsnd" ]]; then
 
+    export CFP_MP="YES"
+
     nth_max=$((npe_node_max / npe_node_postsnd))
 
     export NTHREADS_POSTSND=${nth_postsnd:-1}
     [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max}
-    export APRUN_POSTSND="${launcher} ${npe_postsnd}"
+    export APRUN_POSTSND="${launcher} -n ${npe_postsnd}"
 
     export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
     [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max}
-    export APRUN_POSTSNDCFP="${launcher} ${npe_postsndcfp}"
+    export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}"
 
 elif [[ "${step}" = "awips" ]]; then
 
@@ -218,6 +280,6 @@ elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
     [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
-    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+    export MPIRUN="${launcher} -n ${npe_fit2obs} --mem=0"
 
 fi

jobs/rocoto/vrfy.sh

@@ -17,33 +17,36 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
 
 ###############################################################
 export COMPONENT="atmos"
-export CDATEm1=$(${NDATE} -24 ${CDATE})
-export PDYm1=$(echo ${CDATEm1} | cut -c1-8)
+CDATEm1=$(${NDATE} -24 "${CDATE}")
+export CDATEm1
+PDYm1=$(echo "${CDATEm1}" | cut -c1-8)
+export PDYm1
 
-CDATEm1c=$(${NDATE} -06 ${CDATE})
-PDYm1c=$(echo ${CDATEm1c} | cut -c1-8)
-pcyc=$(echo ${CDATEm1c} | cut -c9-10)
+CDATEm1c=$(${NDATE} -06 "${CDATE}")
+PDYm1c=$(echo "${CDATEm1c}" | cut -c1-8)
+pcyc=$(echo "${CDATEm1c}" | cut -c9-10)
 
 export COMIN="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}"
 
 
 ###############################################################
 echo
 echo "=============== START TO GENERATE QUARTER DEGREE GRIB1 FILES ==============="
-if [ ${MKPGB4PRCP} = "YES" -a ${CDUMP} = "gfs" ]; then
-    if [ ! -d ${ARCDIR} ]; then mkdir -p ${ARCDIR} ; fi
+if [[ ${MKPGB4PRCP} = "YES" && ${CDUMP} = "gfs" ]]; then
+    if [[ ! -d "${ARCDIR}" ]]; then mkdir -p "${ARCDIR}" ; fi
     nthreads_env=${OMP_NUM_THREADS:-1} # get threads set in env
     export OMP_NUM_THREADS=1
-    cd ${COMIN}
+    set -e
+    cd "${COMIN}"
     fhmax=${vhr_rain:-${FHMAX_GFS}}
     fhr=0
-    while [ ${fhr} -le ${fhmax} ]; do
-       fhr2=$(printf %02i ${fhr})
-       fhr3=$(printf %03i ${fhr})
+    while [[ ${fhr} -le ${fhmax} ]]; do
+       fhr2=$(printf %02i "${fhr}")
+       fhr3=$(printf %03i "${fhr}")
        fname=${CDUMP}.t${cyc}z.sfluxgrbf${fhr3}.grib2
        fileout=${ARCDIR}/pgbq${fhr2}.${CDUMP}.${CDATE}.grib2
-       ${WGRIB2} ${fname} -match "(:PRATE:surface:)|(:TMP:2 m above ground:)" -grib ${fileout}
-       (( fhr = ${fhr} + 6 ))
+       ${WGRIB2} "${fname}" -match "(:PRATE:surface:)|(:TMP:2 m above ground:)" -grib "${fileout}"
+       (( fhr = fhr + 6 ))
     done
     export OMP_NUM_THREADS=${nthreads_env} # revert to threads set in env
 fi
@@ -60,7 +63,7 @@ fi
 ###############################################################
 echo
 echo "=============== START TO RUN RADMON DATA EXTRACTION ==============="
-if [ ${VRFYRAD} = "YES" -a "${CDUMP}" = "${CDFNL}" -a "${CDATE}" != "${SDATE}" ]; then
+if [[ ${VRFYRAD} = "YES" && ${CDUMP} = "${CDFNL}" && ${CDATE} != "${SDATE}" ]]; then
 
     export EXP=${PSLOT}
     export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}"
@@ -76,7 +79,7 @@ fi
 ###############################################################
 echo
 echo "=============== START TO RUN OZMON DATA EXTRACTION ==============="
-if [ "${VRFYOZN}" = "YES" -a "${CDUMP}" = "${CDFNL}" -a "${CDATE}" != "${SDATE}" ]; then
+if [[ ${VRFYOZN} = "YES" && ${CDUMP} = "${CDFNL}" && ${CDATE} != "${SDATE}" ]]; then
 
     export EXP=${PSLOT}
     export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}"
@@ -92,7 +95,7 @@ fi
 ###############################################################
 echo
 echo "=============== START TO RUN MINMON ==============="
-if [ "${VRFYMINMON}" = "YES" -a "${CDATE}" != "${SDATE}" ]; then
+if [[ ${VRFYMINMON} = "YES" && ${CDATE} != "${SDATE}" ]]; then
 
     export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}"
     export M_TANKverfM0="${M_TANKverf}/stats/${PSLOT}/${CDUMP}.${PDY}/${cyc}"
@@ -107,9 +110,10 @@ fi
 ################################################################################
 echo
 echo "=============== START TO RUN CYCLONE TRACK VERIFICATION ==============="
-if [ ${VRFYTRAK} = "YES" ]; then
+if [[ ${VRFYTRAK} = "YES" ]]; then
 
-    export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat}
+    COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat}
+    export COMINsyn
 
     ${TRACKERSH}
 fi
@@ -118,23 +122,23 @@ fi
 ################################################################################
 echo
 echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION ==============="
-if [ ${VRFYGENESIS} = "YES" -a "${CDUMP}" = "gfs" ]; then
+if [[ ${VRFYGENESIS} = "YES" && "${CDUMP}" = "gfs" ]]; then
     ${GENESISSH}
 fi
 
 
 ################################################################################
 echo
 echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION (FSU) ==============="
-if [ ${VRFYFSU} = "YES" -a "${CDUMP}" = "gfs" ]; then
+if [[ ${VRFYFSU} = "YES" && "${CDUMP}" = "gfs" ]]; then
     ${GENESISFSU}
 fi
 
 
 ###############################################################
 # Force Exit out cleanly
-cd ${DATAROOT}
-if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf ${DATA} ; fi
+cd "${DATAROOT}"
+if [[ ${KEEPDATA:-"NO"} = "NO" ]] ; then rm -rf "${DATA}" ; fi
 
 
 exit 0

modulefiles/module-setup.sh.inc

@@ -22,7 +22,7 @@ if [[ -d /lfs/f1 ]] ; then
         source /usr/share/lmod/lmod/init/$__ms_shell
     fi
     module reset
-elif [[ -d /lfs3 ]] ; then
+elif [[ -d /mnt/lfs1 ]] ; then
     # We are on NOAA Jet
     if ( ! eval module help > /dev/null 2>&1 ) ; then
         source /apps/lmod/lmod/init/$__ms_shell