Merge branch 'feature/gfsv16b' of https://github.com/NOAA-EMC/global-…

…workflow into feature/gfsv16b_restart
NOAA-EMC · Jun 4, 2020 · bf5a5c4 · bf5a5c4
2 parents b27a01d + 968b986
commit bf5a5c4
Show file tree

Hide file tree

Showing 13 changed files with 176 additions and 269 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -29,6 +29,7 @@ export NTHSTACK=1024000000
 export job=${PBS_JOBNAME:-$step}
 export jobid=${job}.${PBS_JOBID:-$$}
 
+
 if [ $step = "prep" -o $step = "prepbufr" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_prep))
@@ -37,6 +38,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK="NO"
     export sys_tp="HERA"
 
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then
+
+    export mpmd="--multi-prog"
+    export CFP_MP="YES"
+
 elif [ $step = "anal" ]; then
 
     export MKL_NUM_THREADS=4

diff --git a/jobs/JWAVE_PREP b/jobs/JWAVE_PREP
@@ -91,7 +91,9 @@ else
   if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY}
   fi
-  $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+  if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR
+    $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+  fi
   export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc}
   export COMIN_WAV_ICE=${COMIN_OBS}
   export COMIN_WAV_WND=${COMIN_OBS}

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
@@ -25,3 +25,6 @@ module load prod_util/1.1.0
 # python
 module use -a /contrib/modulefiles
 module load anaconda/2.3.0
+
+# waveprep
+module load cdo/1.9.5
diff --git a/parm/config/config.ediag b/parm/config/config.ediag
@@ -8,6 +8,4 @@ echo "BEGIN: config.ediag"
 # Get task specific resources
 . $EXPDIR/config.resources ediag
 
-export ANALDIAGSH="$HOMEgfs/scripts/exglobal_analdiag_fv3gfs.sh.ecf"
-
 echo "END: config.ediag"
diff --git a/parm/config/config.waveprep b/parm/config/config.waveprep
@@ -32,8 +32,8 @@ export WAV_WND_HOUR_INC=1      # This value should match with the one used in
                          # the wind update script
 # Intake currents settings
 export WAV_CUR_DT=${WAV_CUR_DT:-3}
-export WAV_CUR_HF_DT=${WAV_CUR_HF_DT:-3}
-export WAV_CUR_HF_FH=${WAV_CUR_HF_FH:-72}
+export WAV_CUR_HF_DT=${WAV_CUR_HF_DT:-3} # Defaults to 3h for GEFSv12
+export WAV_CUR_HF_FH=${WAV_CUR_HF_FH:-0} # Constant DT for GFSv16 from getgo
 export WAV_CUR_CDO_SMOOTH="NO"
 
 # Location of CDO module

diff --git a/scripts/exwave_init.sh b/scripts/exwave_init.sh
@@ -129,7 +129,11 @@
       fi
 
       [[ ! -d $COMOUT/rundata ]] && mkdir -m 775 -p $COMOUT/rundata
-      echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      else
+        echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      fi
 
       nmoddef=`expr $nmoddef + 1`
 
@@ -162,7 +166,11 @@
 
     if [ "$NTASKS" -gt '1' ]
     then
-      ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile
+      else
+        ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+      fi
       exit=$?
     else
       ./cmdfile

diff --git a/scripts/exwave_prep.sh b/scripts/exwave_prep.sh
@@ -638,55 +638,55 @@
       touch cmdfile
       chmod 744 cmdfile
 
-    ymdh_rtofs=${PDY}00 # RTOFS runs once daily
-    ymdh_end=`$NDATE ${FHMAX_WAV_CUR} ${ymdh_rtofs}`
-    NDATE_DT=${WAV_CUR_HF_DT}
-    FLGHF='T'
+      ymdh_rtofs=${PDY}00 # RTOFS runs once daily use ${PDY}00
+      ymdh_end=`$NDATE ${FHMAX_WAV_CUR} ${PDY}00`
+      NDATE_DT=${WAV_CUR_HF_DT}
+      FLGHF='T'
 
-    if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP
-    while [ "$ymdh_rtofs" -le "$ymdh_end" ]
-    do
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP
+      while [ "$ymdh_rtofs" -le "$ymdh_end" ]
+      do
 # Timing has to be made relative to the single 00z RTOFS cycle for that PDY
-      fhr_rtofs=`${NHOUR} ${ymdh_rtofs} ${PDY}00`
-      fext='f'
+        fhr_rtofs=`${NHOUR} ${ymdh_rtofs} ${PDY}00`
+        fext='f'
 
-      fh3_rtofs=`printf "%03d" "${fhr_rtofs#0}"`
+        fh3_rtofs=`printf "%03d" "${fhr_rtofs#0}"`
 
-      curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_1hrly_prog.nc
-      curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_3hrly_prog.nc
+        curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_1hrly_prog.nc
+        curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_3hrly_prog.nc
 
-      if [ -s ${curfile1h} ]  && [ "${FLGHF}" = "T" ] ; then
-        curfile=${curfile1h}
-      elif [ -s ${curfile3h} ]; then
-        curfile=${curfile3h}
-        FLGHF='F'
-      else
-        echo ' '
-        set $setoff
-        echo ' '
-        echo '************************************** '
-        echo "*** FATAL ERROR: NO CUR FILE $curfile ***  "
-        echo '************************************** '
-        echo ' '
-        set $seton
-        postmsg "$jlogfile" "FATAL ERROR - NO CURRENT FILE (RTOFS)"
-        err=11;export err;${errchk}
-        exit 0
-        echo ' '
-      fi
+        if [ -s ${curfile1h} ]  && [ "${FLGHF}" = "T" ] ; then
+          curfile=${curfile1h}
+        elif [ -s ${curfile3h} ]; then
+          curfile=${curfile3h}
+          FLGHF='F'
+        else
+          echo ' '
+          set $setoff
+          echo ' '
+          echo '************************************** '
+          echo "*** FATAL ERROR: NO CUR FILE $curfile ***  "
+          echo '************************************** '
+          echo ' '
+          set $seton
+          postmsg "$jlogfile" "FATAL ERROR - NO CURRENT FILE (RTOFS)"
+          err=11;export err;${errchk}
+          exit 0
+          echo ' '
+        fi
 
-      if [ ${CFP_MP:-"NO"} = "YES" ]; then
-        echo "$nm $USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
-        nm=`expr $nm + 1`
-      else
-        echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
-      fi
+        if [ ${CFP_MP:-"NO"} = "YES" ]; then
+          echo "$nm $USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+          nm=`expr $nm + 1`
+        else
+          echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+        fi
 
-      if [ $fhr_rtofs -ge ${WAV_CUR_HF_FH} ] ; then
-        NDATE_DT=${WAV_CUR_DT}
-      fi
-      ymdh_rtofs=`$NDATE $NDATE_DT $ymdh_rtofs`
-  done
+        if [ $fhr_rtofs -ge ${WAV_CUR_HF_FH} ] ; then
+          NDATE_DT=${WAV_CUR_DT}
+        fi
+        ymdh_rtofs=`$NDATE $NDATE_DT $ymdh_rtofs`
+      done
 
 # Set number of processes for mpmd
       wavenproc=`wc -l cmdfile | awk '{print $1}'`
@@ -747,7 +747,6 @@
       do
         echo $file
         cat $file >> cur.${WAVECUR_FID}
-        rm -f $file
       done
 
       cp -f cur.${WAVECUR_FID} ${COMOUT}/rundata/${COMPONENTwave}.${WAVECUR_FID}.$cycle.cur 

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
@@ -9,7 +9,7 @@ if [[ ! -d fv3gfs.fd ]] ; then
     rm -f ${topdir}/checkout-fv3gfs.log
     git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    git checkout  GFS.v16.0.4
+    git checkout  GFS.v16.0.5
     git submodule update --init --recursive
     cd ${topdir}
 else
@@ -19,10 +19,9 @@ fi
 echo gsi checkout ...
 if [[ ! -d gsi.fd ]] ; then
     rm -f ${topdir}/checkout-gsi.log
-    git clone --recursive gerrit:ProdGSI gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
+    git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
     cd gsi.fd
-#   git checkout gfsda.v16.0.0
-    git checkout feature/parallel_ncio
+    git checkout release/gfsda.v16.0.0
     git submodule update
     cd ${topdir}
 else
@@ -78,7 +77,7 @@ if [[ ! -d verif-global.fd ]] ; then
     rm -f ${topdir}/checkout-verif-global.log
     git clone --recursive https://github.com/NOAA-EMC/EMC_verif-global.git verif-global.fd >> ${topdir}/checkout-verif-global.log 2>&1
     cd verif-global.fd
-    git checkout verif_global_v1.7.2
+    git checkout verif_global_v1.8.0
     cd ${topdir}
 else
     echo 'Skip. Directory verif-global.fd already exist.'

diff --git a/sorc/gfs_bufr.fd/gfsbufr.f b/sorc/gfs_bufr.fd/gfsbufr.f
@@ -212,7 +212,7 @@ program meteormrf
           error=nf90_inq_dimid(ncid,"pfull",dimid)
           error=nf90_inquire_dimension(ncid,dimid,dim_nam,levsi)
           error=nf90_close(ncid)
-!!          print*,'NetCDF file im,jm,lm= ',im,jm,levs,levsi
+          print*,'NetCDF file im,jm,lm= ',im,jm,levs,levsi
 
            else
           call nemsio_init(iret=irets)
@@ -239,7 +239,7 @@ program meteormrf
 
           call meteorg(npoint,rlat,rlon,istat,cstat,elevstn,
      &             nf,nfile,fnsig,jdate,idate,
-     &      levs,levsi,im,jm,nsfc,
+     &      levsi,im,jm,nsfc,
      &      landwater,nend1, nint1, nint3, iidum,jjdum,
      &      fformat,iocomms(ntask),iope,ionproc)
        call mpi_barrier(iocomms(ntask), ierr)
@@ -248,7 +248,7 @@ program meteormrf
 !! For nemsio input
           call meteorg(npoint,rlat,rlon,istat,cstat,elevstn,
      &             nf,nfile,fnsig,jdate,idate,
-     &      levs,levsi,im,jm,nsfc,
+     &      levs,im,jm,nsfc,
      &      landwater,nend1, nint1, nint3, iidum,jjdum,
      &      fformat,iocomms(ntask),iope,ionproc)
         endif