Redefining mpmd command for working on Hera with slurm Adapting wave …

…scripts to execute mpmd command on Hera Adjusting wave_prnc_cur.sh for properly catting files on Hera

env/HERA.env

@@ -29,6 +29,7 @@ export NTHSTACK=1024000000
 export job=${PBS_JOBNAME:-$step}
 export jobid=${job}.${PBS_JOBID:-$$}
 
+
 if [ $step = "prep" -o $step = "prepbufr" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_prep))
@@ -37,6 +38,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK="NO"
     export sys_tp="HERA"
 
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then
+
+    export mpmd="--multi-prog"
+    export CFP_MP="YES"
+
 elif [ $step = "anal" ]; then
 
     export MKL_NUM_THREADS=4

jobs/JWAVE_PREP

@@ -60,8 +60,7 @@ export pgmout=OUTPUT.$$
 export MP_PULSE=0
 
 # CDO required for processing RTOFS currents
-# export CDO=${COMROOTp2}/nwprod/rtofs_glo.v1.2.0/bin/cdo
-export CDO=/gpfs/dell2/emc/verification/noscrub/Todd.Spindler/CDO/bin/cdo
+export CDO=${CDO_ROOT}/bin/cdo
 
 # Path to HOME Directory
 export FIXwave=${FIXwave:-$HOMEwave/fix/fix_wave_${NET}}
@@ -92,7 +91,12 @@ else
   if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY}
   fi
-  $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+
+    ls -l $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} 
+    ls -l $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+  if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR
+    $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+  fi
   export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc}
   export COMIN_WAV_ICE=${COMIN_OBS}
   export COMIN_WAV_WND=${COMIN_OBS}

modulefiles/module_base.hera

@@ -25,3 +25,6 @@ module load prod_util/1.1.0
 # python
 module use -a /contrib/modulefiles
 module load anaconda/2.3.0
+
+# waveprep
+module load cdo/1.9.5

scripts/exwave_init.sh

@@ -129,7 +129,11 @@
       fi
 
       [[ ! -d $COMOUT/rundata ]] && mkdir -m 775 -p $COMOUT/rundata
-      echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      else
+        echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      fi
 
       nmoddef=`expr $nmoddef + 1`
 
@@ -162,7 +166,11 @@
 
     if [ "$NTASKS" -gt '1' ]
     then
-      ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile
+      else
+        ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+      fi
       exit=$?
     else
       ./cmdfile

scripts/exwave_prep.sh

@@ -311,9 +311,15 @@
 # 3.a Gather and pre-process grib2 files 
       ymdh=$ymdh_beg
 
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP
       while [ "$ymdh" -le "$ymdh_end" ]
       do
-        echo "$USHwave/wave_g2ges.sh $ymdh > grb_$ymdh.out 2>&1" >> cmdfile
+        if [ ${CFP_MP:-"NO"} = "YES" ]; then
+          echo "$nm $USHwave/wave_g2ges.sh $ymdh > grb_$ymdh.out 2>&1" >> cmdfile
+          nm=`expr $nm + 1`
+        else
+          echo "$USHwave/wave_g2ges.sh $ymdh > grb_$ymdh.out 2>&1" >> cmdfile
+        fi
         ymdh=`$NDATE $WAV_WND_HOUR_INC $ymdh`
       done
 
@@ -334,7 +340,11 @@
 
       if [ "$wavenproc" -gt '1' ]
       then
-        ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+        if [ ${CFP_MP:-"NO"} = "YES" ]; then
+          ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile
+        else
+          ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+        fi
         exit=$?
       else
         ./cmdfile
@@ -626,30 +636,24 @@
 # Prepare files for cfp process
       rm -f cmdfile
       touch cmdfile
-      chmod 744 cmfile
+      chmod 744 cmdfile
 
     ymdh_rtofs=${PDY}00 # RTOFS runs once daily
     ymdh_end=`$NDATE ${FHMAX_WAV_CUR} ${ymdh_rtofs}`
     NDATE_DT=${WAV_CUR_HF_DT}
     FLGHF='T'
 
+    if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP
     while [ "$ymdh_rtofs" -le "$ymdh_end" ]
     do
 # Timing has to be made relative to the single 00z RTOFS cycle for that PDY
       fhr_rtofs=`${NHOUR} ${ymdh_rtofs} ${PDY}00`
       fext='f'
 
-      if [ ${fhr_rtofs} -lt 0 ]
-      then
-# Data from nowcast phase
-        fhr_rtofs=`expr 48 + ${fhr_rtofs}`
-        fext='n'
-      fi
-
-      fhr_rtofs=`printf "%03d\n" ${fhr_rtofs}`
+      fh3_rtofs=`printf "%03d" "${fhr_rtofs#0}"`
 
-      curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fhr_rtofs}_1hrly_prog.nc
-      curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fhr_rtofs}_3hrly_prog.nc
+      curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_1hrly_prog.nc
+      curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_3hrly_prog.nc
 
       if [ -s ${curfile1h} ]  && [ "${FLGHF}" = "T" ] ; then
         curfile=${curfile1h}
@@ -671,7 +675,13 @@
         echo ' '
       fi
 
-      echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        echo "$nm $USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+        nm=`expr $nm + 1`
+      else
+        echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+      fi
+
       if [ $fhr_rtofs -ge ${WAV_CUR_HF_FH} ] ; then
         NDATE_DT=${WAV_CUR_DT}
       fi
@@ -691,7 +701,11 @@
 
       if [ $wavenproc -gt '1' ]
       then
-        ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+        if [ ${CFP_MP:-"NO"} = "YES" ]; then
+          ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile
+        else
+          ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+        fi
         exit=$?
       else
         chmod 744 ./cmdfile
@@ -727,12 +741,13 @@
         err=11;export err;${errchk}
       fi
 
-      rm -f cur.${WAVECUR_FID}
+#      rm -f cur.${WAVECUR_FID}
 
       for file in $files
       do
-        cat $file >> cur.${WAVECUR_FID}
-        rm -f $file
+        echo $file
+        cat -s $file >> cur.${WAVECUR_FID}
+#        rm -f $file
       done
 
       cp -f cur.${WAVECUR_FID} ${COMOUT}/rundata/${COMPONENTwave}.${WAVECUR_FID}.$cycle.cur 

ush/wave_prnc_cur.sh

@@ -23,38 +23,40 @@ set -x
 
 ymdh_rtofs=$1
 curfile=$2
+fhr=$3
+fh3=`printf "%03d" "${fhr#0}"`
 
 # Timing has to be made relative to the single 00z RTOFS cycle for that PDY
 
 mkdir -p rtofs_${ymdh_rtofs}
 cd rtofs_${ymdh_rtofs}
 
-ncks -x -v sst,sss,layer_density  $curfile cur_uv_${PDY}_${fext}${fhr}.nc
-ncks -O -a -h -x -v Layer cur_uv_${PDY}_${fext}${fhr}.nc cur_temp1.nc
+ncks -x -v sst,sss,layer_density  $curfile cur_uv_${PDY}_${fext}${fh3}.nc
+ncks -O -a -h -x -v Layer cur_uv_${PDY}_${fext}${fh3}.nc cur_temp1.nc
 ncwa -h -O -a Layer cur_temp1.nc cur_temp2.nc
 ncrename -h -O -v MT,time cur_temp2.nc
 ncrename -h -O -d MT,time cur_temp2.nc
 ncks -v u_velocity,v_velocity cur_temp2.nc cur_temp3.nc
-mv -f cur_temp3.nc cur_uv_${PDY}_${fext}${fhr}_flat.nc
+mv -f cur_temp3.nc cur_uv_${PDY}_${fext}${fh3}_flat.nc
 
 # Convert to regular lat lon file
 
 cp ${FIXwave}/weights_rtofs_to_r4320x2160.nc ./weights.nc
 
 # Interpolate to regular 5 min grid
-$CDO remap,r4320x2160,weights.nc cur_uv_${PDY}_${fext}${fhr}_flat.nc cur_5min_01.nc
+$CDO remap,r4320x2160,weights.nc cur_uv_${PDY}_${fext}${fh3}_flat.nc cur_5min_01.nc
 
 # Perform 9-point smoothing twice to make RTOFS data less noisy when
 # interpolating from 1/12 deg RTOFS grid to 1/6 deg wave grid 
 if [ "WAV_CUR_CDO_SMOOTH" = "YES" ]; then
   $CDO -f nc -smooth9 cur_5min_01.nc cur_5min_02.nc
-  $CDO -f nc -smooth9 cur_5min_02.nc cur_glo_uv_${PDY}_${fext}${fhr}_5min.nc
+  $CDO -f nc -smooth9 cur_5min_02.nc cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc
 else
-  mv cur_5min_01.nc cur_glo_uv_${PDY}_${fext}${fhr}_5min.nc
+  mv cur_5min_01.nc cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc
 fi
 
 # Cleanup
-rm -f cur_temp[123].nc cur_5min_??.nc cur_glo_uv_${PDY}_${fext}${fhr}.nc weights.nc
+#### #rm -f cur_temp[123].nc cur_5min_??.nc cur_glo_uv_${PDY}_${fext}${fh3}.nc weights.nc
 
 if [ ${fhr} -gt ${WAVHINDH} ] 
 then
@@ -64,7 +66,7 @@ else
 fi
 
 rm -f cur.nc
-ln -s cur_glo_uv_${PDY}_${fext}${fhr}_5min.nc cur.nc
+ln -s cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc cur.nc
 ln -s ${DATA}/mod_def.${WAVECUR_FID} ./mod_def.ww3
 
 $EXECwave/ww3_prnc 1> prnc_${WAVECUR_FID}_${ymdh_rtofs}.out 2>&1