Merge pull request #124 from JessicaMeixner-NOAA/feature/gfsv16b-wave

Splitting post jobs for waves
NOAA-EMC · Sep 11, 2020 · 674cd84 · 674cd84
2 parents 46c2404 + ad1a9d9
commit 674cd84
Show file tree

Hide file tree

Showing 24 changed files with 1,503 additions and 288 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -38,12 +38,12 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK="NO"
     export sys_tp="HERA"
 
-elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then
-
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then
     export mpmd="--multi-prog"
     export CFP_MP="YES"
     if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
-
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

diff --git a/env/JET.env b/env/JET.env
@@ -37,9 +37,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK=${BACK:-"YES"}
     export sys_tp="Cray-CS400"
 
-elif [ $step = "waveprep" ]; then
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then
 
-    export MP_PULSE=0
+    if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env
@@ -37,9 +37,10 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export POE=${POE:-"YES"}
     export sys_tp="Cray-XC40"
 
-elif [ $step = "waveprep" ]; then
-
-    export MP_PULSE=0
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then
+    if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env
@@ -45,9 +45,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK=${BACK:-"YES"}
     export sys_tp="Dell-p3"
 
-elif [ $step = "waveprep" ]; then
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then
 
-    export MP_PULSE=0
+    if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd}
 
 elif [ $step = "anal" ]; then
 

diff --git a/jobs/JGLOBAL_WAVE_POST_BNDPNT b/jobs/JGLOBAL_WAVE_POST_BNDPNT
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+date
+export RUN_ENVIR=${RUN_ENVIR:-"nco"}
+export PS4=' $SECONDS + '
+set -x -e
+
+#############################
+# Source relevant config files
+#############################
+configs="base wave wavepostsbs wavepostbndpnt"
+export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config}
+config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config}
+for config in $configs; do
+    . $config_path/config.$config
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+##########################################
+# Source machine runtime environment
+##########################################
+. $HOMEgfs/env/${machine}.env wavepostbndpnt
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+# PATH for working directory
+export NET=${NET:-gfs}
+export RUN=${RUN:-gfs}
+export COMPONENT=${COMPONENT:-wave}
+
+export HOMEgefs=${HOMEgefs:-$NWROOT/$NET.${gefs_ver}}
+export HOMEgfs=${HOMEgfs:-$NWROOT/$NET.${gfs_ver}}
+
+# Add default errchk = err_chk
+export errchk=${errchk:-err_chk}
+
+# Create and go to DATA directory
+export DATA=${DATA:-${DATAROOT:?}/${jobid}}
+mkdir -p $DATA
+cd $DATA
+
+export cyc=${cyc:-00}
+export cycle=${cycle:-t${cyc}z}
+
+# Set PDY
+setpdy.sh
+sh ./PDY
+
+export pgmout=OUTPUT.$$
+
+export MP_PULSE=0
+
+# Path to HOME Directory
+export FIXwave=${FIXwave:-$HOMEgfs/fix/fix_wave_${NET}}
+export PARMwave=${PARMwave:-$HOMEgfs/parm/wave}
+export USHwave=${USHwave:-$HOMEgfs/ush}
+export EXECwave=${EXECwave:-$HOMEgfs/exec}
+
+# Set COM Paths and GETGES environment
+if [ $RUN_ENVIR = "nco" ]; then
+    export ROTDIR=${COMROOT:?}/$NET/$envir
+fi
+export COMIN=${COMIN:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+
+export COMINice=${COMINice:-${COMROOTp2}/omb/prod}
+export COMINwnd=${COMINwnd:-${COMROOT}/gfs/prod}
+export COMIN_WAV_CUR=${COMIN_WAV_CUR:-${COMROOTp2}/rtofs/prod}
+
+mkdir -p $COMOUT/station
+
+env | sort
+
+# Set wave model ID tag to include member number 
+# if ensemble; waveMEMB var empty in deterministic
+# Set wave model ID tag to include member number
+# if ensemble; waveMEMB var empty in deterministic
+membTAG='p'
+if [ "${waveMEMB}" == "00" ]; then membTAG='c'; fi
+export membTAG
+export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
+
+export CFP_VERBOSE=1
+
+# Execute the Script  
+$HOMEgfs/scripts/exgfs_wave_post_bndpnt.sh
+err=$?
+if [ $err -ne 0 ]; then
+  msg="FATAL ERROR: ex-script of GWES_POST failed!"
+else
+  msg="$job completed normally!"
+fi
+postmsg "$jlogfile" "$msg"
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd $DATAROOT
+[[ $KEEPDATA = "NO" ]] && rm -rf $DATA
+
+date
+exit 0
diff --git a/jobs/JGLOBAL_WAVE_POST_PNT b/jobs/JGLOBAL_WAVE_POST_PNT
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+date
+export RUN_ENVIR=${RUN_ENVIR:-"nco"}
+export PS4=' $SECONDS + '
+set -x -e
+
+#############################
+# Source relevant config files
+#############################
+configs="base wave wavepostsbs wavepostpnt"
+export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config}
+config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config}
+for config in $configs; do
+    . $config_path/config.$config
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+##########################################
+# Source machine runtime environment
+##########################################
+. $HOMEgfs/env/${machine}.env wavepostpnt
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+# PATH for working directory
+export NET=${NET:-gfs}
+export RUN=${RUN:-gfs}
+export COMPONENT=${COMPONENT:-wave}
+
+export HOMEgefs=${HOMEgefs:-$NWROOT/$NET.${gefs_ver}}
+export HOMEgfs=${HOMEgfs:-$NWROOT/$NET.${gfs_ver}}
+
+# Add default errchk = err_chk
+export errchk=${errchk:-err_chk}
+
+# Create and go to DATA directory
+export DATA=${DATA:-${DATAROOT:?}/${jobid}}
+mkdir -p $DATA
+cd $DATA
+
+export cyc=${cyc:-00}
+export cycle=${cycle:-t${cyc}z}
+
+# Set PDY
+setpdy.sh
+sh ./PDY
+
+export pgmout=OUTPUT.$$
+
+export MP_PULSE=0
+
+# Path to HOME Directory
+export FIXwave=${FIXwave:-$HOMEgfs/fix/fix_wave_${NET}}
+export PARMwave=${PARMwave:-$HOMEgfs/parm/wave}
+export USHwave=${USHwave:-$HOMEgfs/ush}
+export EXECwave=${EXECwave:-$HOMEgfs/exec}
+
+# Set COM Paths and GETGES environment
+if [ $RUN_ENVIR = "nco" ]; then
+    export ROTDIR=${COMROOT:?}/$NET/$envir
+fi
+export COMIN=${COMIN:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
+
+export COMINice=${COMINice:-${COMROOTp2}/omb/prod}
+export COMINwnd=${COMINwnd:-${COMROOT}/gfs/prod}
+export COMIN_WAV_CUR=${COMIN_WAV_CUR:-${COMROOTp2}/rtofs/prod}
+
+mkdir -p $COMOUT/station
+
+env | sort
+
+# Set wave model ID tag to include member number 
+# if ensemble; waveMEMB var empty in deterministic
+# Set wave model ID tag to include member number
+# if ensemble; waveMEMB var empty in deterministic
+membTAG='p'
+if [ "${waveMEMB}" == "00" ]; then membTAG='c'; fi
+export membTAG
+export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
+
+export CFP_VERBOSE=1
+
+# Execute the Script  
+$HOMEgfs/scripts/exgfs_wave_post_pnt.sh
+err=$?
+if [ $err -ne 0 ]; then
+  msg="FATAL ERROR: ex-script of GWES_POST failed!"
+else
+  msg="$job completed normally!"
+fi
+postmsg "$jlogfile" "$msg"
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd $DATAROOT
+[[ $KEEPDATA = "NO" ]] && rm -rf $DATA
+
+date
+exit 0
diff --git a/jobs/JGLOBAL_WAVE_POST_SBS b/jobs/JGLOBAL_WAVE_POST_SBS
@@ -69,12 +69,6 @@ export COMINwnd=${COMINwnd:-${COMROOT}/gfs/prod}
 export COMIN_WAV_CUR=${COMIN_WAV_CUR:-${COMROOTp2}/rtofs/prod}
 
 mkdir -p $COMOUT/gridded
-mkdir -p $COMOUT/station
-mkdir -p $COMOUT/stats
-
-# Set mpi serial command
-export wavempexec=${launcher:-"mpirun -n"}
-export wave_mpmd=${mpmd:-"cfp"}
 
 env | sort
 
@@ -90,7 +84,7 @@ export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
 export CFP_VERBOSE=1
 
 # Execute the Script  
-$HOMEgfs/scripts/exgfs_wave_post_sbs.sh
+$HOMEgfs/scripts/exgfs_wave_post_gridded_sbs.sh
 err=$?
 if [ $err -ne 0 ]; then
   msg="FATAL ERROR: ex-script of GWES_POST failed!"

diff --git a/jobs/JGLOBAL_WAVE_PREP b/jobs/JGLOBAL_WAVE_PREP
@@ -71,30 +71,22 @@ export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/$COMPONENT}
 
 if [ $RUN_ENVIR = "nco" ]; then
   export COMIN_WAV_ICE=${COMIN_WAV_ICE:-$(compath.py gfs/prod)}/${CDUMP}.${PDY}/${cyc}/atmos
-  export COMIN_WAV_WND=${COMIN_WAV_WND:-$(compath.py gfs/prod)}/${CDUMP}.${PDY}/${cyc}/atmos
   export COMIN_WAV_CUR=${COMIN_WAV_CUR:-$(compath.py ${WAVECUR_DID}/prod)}/${WAVECUR_DID}.${RPDY}
   if [ ! -d $COMIN_WAV_CUR ]; then
     export RPDY=`$NDATE -24 ${PDY}00 | cut -c1-8`
     export COMIN_WAV_CUR=$(compath.py ${WAVECUR_DID}/prod)/${WAVECUR_DID}.${RPDY}
   fi
 else 
-  if [ ! -d $DMPDIR/${WAVECUR_DID}.${RPDY} ]; then export RPDY=`$NDATE -24 ${PDY}00 | cut -c1-8`; fi
   if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY}
   fi
   if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/atmos/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/atmos/${WAVICEFILE}
   fi
-  export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc/atmos}
-  export COMIN_WAV_ICE=${COMIN_OBS}
-  export COMIN_WAV_WND=${COMIN_OBS}
+  export COMIN_WAV_ICE=${COMIN_WAV_ICE:-$ROTDIR/$RUN.$PDY/$cyc/atmos}
   export COMIN_WAV_CUR=${ROTDIR}/${WAVECUR_DID}.${RPDY}
 fi
 
-# Set mpi serial command
-export wavempexec=${launcher:-"mpirun -n"}
-export wave_mpmd=${mpmd:-"cfp"}
-
 # Execute the Script  
 $HOMEgfs/scripts/exgfs_wave_prep.sh
 

diff --git a/jobs/rocoto/wavepostbndpnt.sh b/jobs/rocoto/wavepostbndpnt.sh
@@ -0,0 +1,21 @@
+#!/bin/ksh -x
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo
+echo "=============== START TO RUN WAVE_POST_BNDPNT ==============="
+# Execute the JJOB
+$HOMEgfs/jobs/JGLOBAL_WAVE_POST_BNDPNT
+status=$?
+exit $status
+
+###############################################################
+# Force Exit out cleanly
+if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf $DATAROOT ; fi
+exit 0
diff --git a/jobs/rocoto/wavepostpnt.sh b/jobs/rocoto/wavepostpnt.sh
@@ -0,0 +1,21 @@
+#!/bin/ksh -x
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+echo
+echo "=============== START TO RUN WAVE_POST_PNT ==============="
+# Execute the JJOB
+$HOMEgfs/jobs/JGLOBAL_WAVE_POST_PNT
+status=$?
+exit $status
+
+###############################################################
+# Force Exit out cleanly
+if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf $DATAROOT ; fi
+exit 0
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
@@ -43,3 +43,6 @@ module load g2tmpl/1.6.0
 # Load FSU Genesis module
 module use -a /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/tracker/ens_tracker.v1.1.15.3/ush/FSUgenesisPY/modulefiles
 module load tclogg
+
+# waveprep
+module load cdo/1.9.8