Merge pull request #135 from KateFriedman-NOAA/feature/gfsv16b-down

Small updates to downstream jobs and tag updates

Externals.cfg

@@ -15,14 +15,14 @@ protocol = git
 required = True
 
 [GLDAS]
-tag = gldas_gfsv16_release.v1.5.0
+tag = gldas_gfsv16_release.v1.7.0
 local_path = sorc/gldas.fd
 repo_url = https://github.com/NOAA-EMC/GLDAS.git
 protocol = git
 required = True
 
 [EMC_post]
-tag = upp_gfsv16_release.v1.0.13
+tag = upp_gfsv16_release.v1.0.14
 local_path = sorc/gfs_post.fd
 repo_url = https://github.com/NOAA-EMC/EMC_post.git
 protocol = git
@@ -43,7 +43,7 @@ protocol = git
 required = True
 
 [EMC_gfs_wafs]
-tag = gfs_wafs.v6.0.4
+tag = gfs_wafs.v6.0.6
 local_path = sorc/gfs_wafs.fd
 repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git
 protocol = git

env/HERA.env

@@ -38,6 +38,9 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK="NO"
     export sys_tp="HERA"
 
+    module unload netcdfp/4.7.4
+    module load netcdf/4.7.0
+
 elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then
     export mpmd="--multi-prog"
     export CFP_MP="YES"

jobs/rocoto/arch.sh

@@ -213,15 +213,25 @@ if [ $CDUMP = "gfs" ]; then
         fi
     fi
 
-   #--save mdl gfsmos output from all cycles in the 18Z archive directory
-   if [ -d gfsmos.$PDY_MOS -a $cyc -eq 18 ]; then
+    #for downstream products
+    if [ $DO_BUFRSND = "YES" -o $WAFSF = "YES" ]; then
+        htar -P -cvf $ATARDIR/$CDATE/gfs_downstream.tar `cat $ARCH_LIST/gfs_downstream.txt`
+        status=$?
+        if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
+            echo "HTAR $CDATE gfs_downstream.tar failed"
+            exit $status
+        fi
+    fi
+
+    #--save mdl gfsmos output from all cycles in the 18Z archive directory
+    if [ -d gfsmos.$PDY_MOS -a $cyc -eq 18 ]; then
         htar -P -cvf $ATARDIR/$CDATE_MOS/gfsmos.tar ./gfsmos.$PDY_MOS
         status=$?
         if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
             echo "HTAR $CDATE gfsmos.tar failed"
             exit $status
         fi
-   fi
+    fi
 
 fi
 

modulefiles/module_base.wcoss_dell_p3

@@ -31,6 +31,7 @@ module load gempak/7.3.1
 # Load for WAFS
 module load bufr_dumplist/2.0.0
 module load dumpjb/5.0.3
+module load util_shared/1.1.2
 
 # python
 module load python/3.6.3

scripts/exgfs_awips_20km_1p0deg.sh

@@ -100,7 +100,7 @@ cat tmpfile2${fcsthrs}   tmpfile2b${fcsthrs}  >  tmpfile${fcsthrs}
 $WGRIB2 tmpfile${fcsthrs} | grep  -F -f $PARMproduct/gfs_awips_parmlist_g2 | $WGRIB2 -i -grib masterfile  tmpfile${fcsthrs}
 export err=$?
 if [[ $err -ne 0 ]] ; then
-   echo " File: masterfile does not exist."
+   echo " FATAL ERROR: masterfile does not exist."
    exit $err
 fi
 

scripts/exgfs_grib_awips.sh

@@ -38,7 +38,7 @@ then
    echo ""
 else
    echo ""
-   echo " Number of arguments were not passed "
+   echo " FATAL ERROR: Number of arguments were not passed."
    echo ""
    echo ""
    echo "Usage: $0  \$fcsthrs  (3-digit) "

sorc/checkout.sh

@@ -50,7 +50,7 @@ if [[ ! -d gldas.fd ]] ; then
     rm -f ${topdir}/checkout-gldas.log
     git clone https://github.com/NOAA-EMC/GLDAS  gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1
     cd gldas.fd
-    git checkout gldas_gfsv16_release.v1.6.0
+    git checkout gldas_gfsv16_release.v1.7.0
     cd ${topdir}
 else
     echo 'Skip.  Directory gldas.fd already exists.'
@@ -72,7 +72,7 @@ if [[ ! -d gfs_post.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_post.log
     git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
     cd gfs_post.fd
-    git checkout upp_gfsv16_release.v1.0.13
+    git checkout upp_gfsv16_release.v1.0.14
     ################################################################################
     # checkout_gtg
     ## yes: The gtg code at NCAR private repository is available for ops. GFS only.
@@ -95,7 +95,7 @@ if [[ ! -d gfs_wafs.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_wafs.log
     git clone --recursive https://github.com/NOAA-EMC/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1
     cd gfs_wafs.fd
-    git checkout gfs_wafs.v6.0.4
+    git checkout gfs_wafs.v6.0.6
     cd ${topdir}
 else
     echo 'Skip.  Directory gfs_wafs.fd already exists.'

ush/hpssarch_gen.sh

@@ -21,6 +21,12 @@ else
   format="nemsio"
 fi
 
+# Set whether to archive downstream products
+DO_DOWN=${DO_DOWN:-"NO"}
+if [ $DO_BUFRSND = "YES" -o $WAFSF = "YES" ]; then
+  export DO_DOWN="YES"
+fi
+
 #-----------------------------------------------------
 if [ $type = "gfs" ]; then
 #-----------------------------------------------------
@@ -46,6 +52,11 @@ if [ $type = "gfs" ]; then
   touch gfs_${format}b.txt
   touch gfs_restarta.txt
 
+  if [ $DO_DOWN = "YES" ]; then
+    rm -f gfs_downstream.txt
+    touch gfs_downstream.txt
+  fi
+
   dirpath="gfs.${PDY}/${cyc}/atmos/"
   dirname="./${dirpath}"
 
@@ -57,6 +68,7 @@ if [ $type = "gfs" ]; then
   echo  "${dirname}${head}pgrb2b.0p50.anl                  " >>gfs_pgrb2b.txt
   echo  "${dirname}${head}pgrb2b.0p50.anl.idx              " >>gfs_pgrb2b.txt
 
+  echo  "./logs/${CDATE}/gfs*.log                          " >>gfsa.txt
   echo  "${dirname}${head}gsistat                          " >>gfsa.txt
   echo  "${dirname}${head}nsstbufr                         " >>gfsa.txt
   echo  "${dirname}${head}prepbufr                         " >>gfsa.txt
@@ -70,10 +82,26 @@ if [ $type = "gfs" ]; then
   echo  "${dirname}trak.gfso.atcfunix.altg.${PDY}${cyc}    " >>gfsa.txt
   echo  "${dirname}storms.gfso.atcf_gen.${PDY}${cyc}       " >>gfsa.txt
   echo  "${dirname}storms.gfso.atcf_gen.altg.${PDY}${cyc}  " >>gfsa.txt
-  echo  "${dirname}gempak/gfs_${PDY}${cyc}.sfc             " >>gfsa.txt
-  echo  "${dirname}gempak/gfs_${PDY}${cyc}.snd             " >>gfsa.txt
-  echo  "${dirname}bufr.t${cyc}z                           " >>gfsa.txt
-  echo  "./logs/${CDATE}/gfs*.log                          " >>gfsa.txt
+
+  if [ $DO_DOWN = "YES" ]; then
+   if [ $DO_BUFRSND = "YES" ]; then
+    echo  "${dirname}gempak/gfs_${PDY}${cyc}.sfc              " >>gfs_downstream.txt
+    echo  "${dirname}gempak/gfs_${PDY}${cyc}.snd              " >>gfs_downstream.txt
+    echo  "${dirname}wmo/gfs_collective*.postsnd_${cyc}       " >>gfs_downstream.txt
+    echo  "${dirname}bufr.t${cyc}z                            " >>gfs_downstream.txt
+    echo  "${dirname}gfs.t${cyc}z.bufrsnd.tar.gz              " >>gfs_downstream.txt
+   fi
+   if [ $WAFSF = "YES" ]; then
+    echo  "${dirname}wafsgfs*.t${cyc}z.gribf*.grib2           " >>gfs_downstream.txt
+    echo  "${dirname}gfs.t${cyc}z.wafs_grb45f*.grib2          " >>gfs_downstream.txt
+    echo  "${dirname}gfs.t${cyc}z.wafs_grb45f*.nouswafs.grib2 " >>gfs_downstream.txt
+    echo  "${dirname}WAFS_blended_${PDY}${cyc}f*.grib2        " >>gfs_downstream.txt
+    echo  "${dirname}gfs.t*z.gcip.f*.grib2                    " >>gfs_downstream.txt
+    echo  "${dirname}gfs.t${cyc}z.wafs_0p25.f*.grib2          " >>gfs_downstream.txt
+    echo  "${dirname}gfs.t${cyc}z.wafs_0p25_unblended.f*.grib2" >>gfs_downstream.txt
+    echo  "${dirname}WAFS_0p25_blended_${PDY}${cyc}f*.grib2   " >>gfs_downstream.txt
+   fi
+  fi
 
   echo  "${dirname}${head}pgrb2.0p50.anl                   " >>gfsb.txt
   echo  "${dirname}${head}pgrb2.0p50.anl.idx               " >>gfsb.txt

ush/rocoto/setup_workflow.py

@@ -238,7 +238,6 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'):
     do_metp = base.get('DO_METP', 'NO').upper()
     do_gldas = base.get('DO_GLDAS', 'NO').upper()
     do_wave = base.get('DO_WAVE', 'NO').upper()
-    do_waveprdgen = base.get('DO_WAVEPRDGEN', 'NO').upper()
     do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper()
     reservation = base.get('RESERVATION', 'NONE').upper()
 
@@ -761,6 +760,14 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'):
     deps.append(rocoto.add_dependency(dep_dict))
     dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
     deps.append(rocoto.add_dependency(dep_dict))
+    if do_wave in ['Y', 'YES']:
+      dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
+      deps.append(rocoto.add_dependency(dep_dict))
+      dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump}
+      deps.append(rocoto.add_dependency(dep_dict))
+      if cdump in ['gfs']:
+        dep_dict = {'type': 'task', 'name': '%swavepostbndpnt' % cdump}
+        deps.append(rocoto.add_dependency(dep_dict))
     dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
     task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies)