Merge remote-tracking branch 'upstream/feature/gfsv16b-wave' into fea…

…ture/gfsv16b-wave

* upstream/feature/gfsv16b-wave:
  adding pnt jobs as seperate jobs for env
  moving definitions of wavempexec and wave_mpmd from jobs to env
  lowering the resource requirement for wave prep job
  update module for cdo
  updates to resources for wave jobs
  Fixing wavepostbndpnt dependency in setup_workflow_fcstonly.py
  cleaning up wave point post scripts
  rename wave post script
  remove gridded so can rename
  updates to boundary point
  Remove extra space from line
  Fix wavepostbndpnt dependency and increase wavepostbntpnt and wavepostpnt walltimes
  fix typos
  updating WCOSS work around for CDO, CDO_ROOT is missing from module file
  adding module use for cdo module on wcoss dell
  updates for new weights file and adding cdo module for wave prep
  Adding new wavepostbndpnt and wavepostpnt jobs
  updating wave post scripts for restructured format
  updates to JJOBs for wave post point for new structure
  changing exit 0 to exit number for FATAL errors in ice prep for waves
  updating error message and exit if there is no current input file
  update WAVE_PREP so that currents do not check for previous 24 hour so that parallels will be reproducible also deleted unused variable
  renaming wave job and scripts
  update jobs for waves
  Issue #94 - pull in two fixes for wave job setup in free-forecast mode
  updates for wave scripts to split them, also added updates from GEFS branch to add extra error checking
  updates from gefs post workflow: updates for re-run case as well as extra error messages

env/HERA.env

@@ -38,12 +38,12 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK="NO"
     export sys_tp="HERA"
 
-elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then
-
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt"]; then
     export mpmd="--multi-prog"
     export CFP_MP="YES"
     if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
-
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

env/JET.env

@@ -37,9 +37,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK=${BACK:-"YES"}
     export sys_tp="Cray-CS400"
 
-elif [ $step = "waveprep" ]; then
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt"]; then
 
-    export MP_PULSE=0
+    if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

env/WCOSS_C.env

@@ -37,9 +37,10 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export POE=${POE:-"YES"}
     export sys_tp="Cray-XC40"
 
-elif [ $step = "waveprep" ]; then
-
-    export MP_PULSE=0
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt"]; then
+    if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

env/WCOSS_DELL_P3.env

@@ -45,9 +45,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK=${BACK:-"YES"}
     export sys_tp="Dell-p3"
 
-elif [ $step = "waveprep" ]; then
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt"]; then
 
-    export MP_PULSE=0
+    if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

jobs/JGLOBAL_WAVE_POST_BNDPNT

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+date
+export RUN_ENVIR=${RUN_ENVIR:-"nco"}
+export PS4=' $SECONDS + '
+set -x -e
+
+#############################
+# Source relevant config files
+#############################
+configs="base wave wavepostsbs wavepostbndpnt"
+export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config}
+config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config}
+for config in $configs; do
+    . $config_path/config.$config
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+##########################################
+# Source machine runtime environment
+##########################################
+. $HOMEgfs/env/${machine}.env wavepostbndpnt
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+# PATH for working directory
+export NET=${NET:-gfs}
+export RUN=${RUN:-gfs}
+export COMPONENT=${COMPONENT:-wave}
+
+export HOMEgefs=${HOMEgefs:-$NWROOT/$NET.${gefs_ver}}
+export HOMEgfs=${HOMEgfs:-$NWROOT/$NET.${gfs_ver}}
+
+# Add default errchk = err_chk
+export errchk=${errchk:-err_chk}
+
+# Create and go to DATA directory
+export DATA=${DATA:-${DATAROOT:?}/${jobid}}
+mkdir -p $DATA
+cd $DATA
+
+export cyc=${cyc:-00}
+export cycle=${cycle:-t${cyc}z}
+
+# Set PDY
+setpdy.sh
+sh ./PDY
+
+export pgmout=OUTPUT.$$
+
+export MP_PULSE=0
+
+# Path to HOME Directory
+export FIXwave=${FIXwave:-$HOMEgfs/fix/fix_wave_${NET}}
+export PARMwave=${PARMwave:-$HOMEgfs/parm/wave}
+export USHwave=${USHwave:-$HOMEgfs/ush}
+export EXECwave=${EXECwave:-$HOMEgfs/exec}
+
+# Set COM Paths and GETGES environment
+if [ $RUN_ENVIR = "nco" ]; then
+    export ROTDIR=${COMROOT:?}/$NET/$envir
+fi
+export COMIN=${COMIN:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+
+export COMINice=${COMINice:-${COMROOTp2}/omb/prod}
+export COMINwnd=${COMINwnd:-${COMROOT}/gfs/prod}
+export COMIN_WAV_CUR=${COMIN_WAV_CUR:-${COMROOTp2}/rtofs/prod}
+
+mkdir -p $COMOUT/station
+
+env | sort
+
+# Set wave model ID tag to include member number 
+# if ensemble; waveMEMB var empty in deterministic
+# Set wave model ID tag to include member number
+# if ensemble; waveMEMB var empty in deterministic
+membTAG='p'
+if [ "${waveMEMB}" == "00" ]; then membTAG='c'; fi
+export membTAG
+export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
+
+export CFP_VERBOSE=1
+
+# Execute the Script  
+$HOMEgfs/scripts/exgfs_wave_post_bndpnt.sh
+err=$?
+if [ $err -ne 0 ]; then
+  msg="FATAL ERROR: ex-script of GWES_POST failed!"
+else
+  msg="$job completed normally!"
+fi
+postmsg "$jlogfile" "$msg"
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd $DATAROOT
+[[ $KEEPDATA = "NO" ]] && rm -rf $DATA
+
+date
+exit 0

jobs/JGLOBAL_WAVE_POST_PNT

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+date
+export RUN_ENVIR=${RUN_ENVIR:-"nco"}
+export PS4=' $SECONDS + '
+set -x -e
+
+#############################
+# Source relevant config files
+#############################
+configs="base wave wavepostsbs wavepostpnt"
+export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config}
+config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config}
+for config in $configs; do
+    . $config_path/config.$config
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+##########################################
+# Source machine runtime environment
+##########################################
+. $HOMEgfs/env/${machine}.env wavepostpnt
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+# PATH for working directory
+export NET=${NET:-gfs}
+export RUN=${RUN:-gfs}
+export COMPONENT=${COMPONENT:-wave}
+
+export HOMEgefs=${HOMEgefs:-$NWROOT/$NET.${gefs_ver}}
+export HOMEgfs=${HOMEgfs:-$NWROOT/$NET.${gfs_ver}}
+
+# Add default errchk = err_chk
+export errchk=${errchk:-err_chk}
+
+# Create and go to DATA directory
+export DATA=${DATA:-${DATAROOT:?}/${jobid}}
+mkdir -p $DATA
+cd $DATA
+
+export cyc=${cyc:-00}
+export cycle=${cycle:-t${cyc}z}
+
+# Set PDY
+setpdy.sh
+sh ./PDY
+
+export pgmout=OUTPUT.$$
+
+export MP_PULSE=0
+
+# Path to HOME Directory
+export FIXwave=${FIXwave:-$HOMEgfs/fix/fix_wave_${NET}}
+export PARMwave=${PARMwave:-$HOMEgfs/parm/wave}
+export USHwave=${USHwave:-$HOMEgfs/ush}
+export EXECwave=${EXECwave:-$HOMEgfs/exec}
+
+# Set COM Paths and GETGES environment
+if [ $RUN_ENVIR = "nco" ]; then
+    export ROTDIR=${COMROOT:?}/$NET/$envir
+fi
+export COMIN=${COMIN:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+
+export COMINice=${COMINice:-${COMROOTp2}/omb/prod}
+export COMINwnd=${COMINwnd:-${COMROOT}/gfs/prod}
+export COMIN_WAV_CUR=${COMIN_WAV_CUR:-${COMROOTp2}/rtofs/prod}
+
+mkdir -p $COMOUT/station
+
+env | sort
+
+# Set wave model ID tag to include member number 
+# if ensemble; waveMEMB var empty in deterministic
+# Set wave model ID tag to include member number
+# if ensemble; waveMEMB var empty in deterministic
+membTAG='p'
+if [ "${waveMEMB}" == "00" ]; then membTAG='c'; fi
+export membTAG
+export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
+
+export CFP_VERBOSE=1
+
+# Execute the Script  
+$HOMEgfs/scripts/exgfs_wave_post_pnt.sh
+err=$?
+if [ $err -ne 0 ]; then
+  msg="FATAL ERROR: ex-script of GWES_POST failed!"
+else
+  msg="$job completed normally!"
+fi
+postmsg "$jlogfile" "$msg"
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd $DATAROOT
+[[ $KEEPDATA = "NO" ]] && rm -rf $DATA
+
+date
+exit 0

jobs/JGLOBAL_WAVE_POST_SBS

@@ -69,12 +69,6 @@ export COMINwnd=${COMINwnd:-${COMROOT}/gfs/prod}
 export COMIN_WAV_CUR=${COMIN_WAV_CUR:-${COMROOTp2}/rtofs/prod}
 
 mkdir -p $COMOUT/gridded
-mkdir -p $COMOUT/station
-mkdir -p $COMOUT/stats
-
-# Set mpi serial command
-export wavempexec=${launcher:-"mpirun -n"}
-export wave_mpmd=${mpmd:-"cfp"}
 
 env | sort
 
@@ -90,7 +84,7 @@ export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
 export CFP_VERBOSE=1
 
 # Execute the Script  
-$HOMEgfs/scripts/exgfs_wave_post_sbs.sh
+$HOMEgfs/scripts/exgfs_wave_post_gridded_sbs.sh
 err=$?
 if [ $err -ne 0 ]; then
   msg="FATAL ERROR: ex-script of GWES_POST failed!"

jobs/JGLOBAL_WAVE_PREP

@@ -71,30 +71,22 @@ export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
 
 if [ $RUN_ENVIR = "nco" ]; then
   export COMIN_WAV_ICE=${COMIN_WAV_ICE:-$(compath.py gfs/prod)}/${CDUMP}.${PDY}/${cyc}/atmos
-  export COMIN_WAV_WND=${COMIN_WAV_WND:-$(compath.py gfs/prod)}/${CDUMP}.${PDY}/${cyc}/atmos
   export COMIN_WAV_CUR=${COMIN_WAV_CUR:-$(compath.py ${WAVECUR_DID}/prod)}/${WAVECUR_DID}.${RPDY}
   if [ ! -d $COMIN_WAV_CUR ]; then
     export RPDY=`$NDATE -24 ${PDY}00 | cut -c1-8`
     export COMIN_WAV_CUR=$(compath.py ${WAVECUR_DID}/prod)/${WAVECUR_DID}.${RPDY}
   fi
 else 
-  if [ ! -d $DMPDIR/${WAVECUR_DID}.${RPDY} ]; then export RPDY=`$NDATE -24 ${PDY}00 | cut -c1-8`; fi
   if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY}
   fi
   if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/atmos/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/atmos/${WAVICEFILE}
   fi
-  export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc/atmos}
-  export COMIN_WAV_ICE=${COMIN_OBS}
-  export COMIN_WAV_WND=${COMIN_OBS}
+  export COMIN_WAV_ICE=${COMIN_WAV_ICE:-$ROTDIR/$RUN.$PDY/$cyc/atmos}
   export COMIN_WAV_CUR=${ROTDIR}/${WAVECUR_DID}.${RPDY}
 fi
 
-# Set mpi serial command
-export wavempexec=${launcher:-"mpirun -n"}
-export wave_mpmd=${mpmd:-"cfp"}
-
 # Execute the Script  
 $HOMEgfs/scripts/exgfs_wave_prep.sh
 

jobs/rocoto/wavepostbndpnt.sh

@@ -0,0 +1,21 @@
+#!/bin/ksh -x
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo
+echo "=============== START TO RUN WAVE_POST_BNDPNT ==============="
+# Execute the JJOB
+$HOMEgfs/jobs/JGLOBAL_WAVE_POST_BNDPNT
+status=$?
+exit $status
+
+###############################################################
+# Force Exit out cleanly
+if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf $DATAROOT ; fi
+exit 0

jobs/rocoto/wavepostpnt.sh

@@ -0,0 +1,21 @@
+#!/bin/ksh -x
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo
+echo "=============== START TO RUN WAVE_POST_PNT ==============="
+# Execute the JJOB
+$HOMEgfs/jobs/JGLOBAL_WAVE_POST_PNT
+status=$?
+exit $status
+
+###############################################################
+# Force Exit out cleanly
+if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf $DATAROOT ; fi
+exit 0

modulefiles/module_base.wcoss_dell_p3

@@ -43,3 +43,6 @@ module load g2tmpl/1.6.0
 # Load FSU Genesis module
 module use -a /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/tracker/ens_tracker.v1.1.15.3/ush/FSUgenesisPY/modulefiles
 module load tclogg
+
+# waveprep
+module load cdo/1.9.8