From 20ef779fd90a267affc0e80ae1f8a3f64afd2f8d Mon Sep 17 00:00:00 2001 From: "henrique.alves" Date: Fri, 22 May 2020 19:52:55 +0000 Subject: [PATCH] Redefining mpmd command for working on Hera with slurm Adapting wave scripts to execute mpmd command on Hera Adjusting wave_prnc_cur.sh for properly catting files on Hera --- env/HERA.env | 6 +++++ jobs/JWAVE_PREP | 10 ++++--- modulefiles/module_base.hera | 3 +++ scripts/exwave_init.sh | 12 +++++++-- scripts/exwave_prep.sh | 51 +++++++++++++++++++++++------------- ush/wave_prnc_cur.sh | 18 +++++++------ 6 files changed, 69 insertions(+), 31 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index 8bae2de719..dce975a383 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -29,6 +29,7 @@ export NTHSTACK=1024000000 export job=${PBS_JOBNAME:-$step} export jobid=${job}.${PBS_JOBID:-$$} + if [ $step = "prep" -o $step = "prepbufr" ]; then nth_max=$(($npe_node_max / $npe_node_prep)) @@ -37,6 +38,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then export BACK="NO" export sys_tp="HERA" +elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then + + export mpmd="--multi-prog" + export CFP_MP="YES" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 diff --git a/jobs/JWAVE_PREP b/jobs/JWAVE_PREP index 5505b0d52a..f8381bce9b 100755 --- a/jobs/JWAVE_PREP +++ b/jobs/JWAVE_PREP @@ -60,8 +60,7 @@ export pgmout=OUTPUT.$$ export MP_PULSE=0 # CDO required for processing RTOFS currents -# export CDO=${COMROOTp2}/nwprod/rtofs_glo.v1.2.0/bin/cdo -export CDO=/gpfs/dell2/emc/verification/noscrub/Todd.Spindler/CDO/bin/cdo +export CDO=${CDO_ROOT}/bin/cdo # Path to HOME Directory export FIXwave=${FIXwave:-$HOMEwave/fix/fix_wave_${NET}} @@ -92,7 +91,12 @@ else if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY} fi - $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} + + ls -l $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} + ls -l $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} + if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR + $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} + fi export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc} export COMIN_WAV_ICE=${COMIN_OBS} export COMIN_WAV_WND=${COMIN_OBS} diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index 0c461367d6..8d56832ab7 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -25,3 +25,6 @@ module load prod_util/1.1.0 # python module use -a /contrib/modulefiles module load anaconda/2.3.0 + +# waveprep +module load cdo/1.9.5 diff --git a/scripts/exwave_init.sh b/scripts/exwave_init.sh index 0704c9ea5d..848222d5ed 100755 --- a/scripts/exwave_init.sh +++ b/scripts/exwave_init.sh @@ -129,7 +129,11 @@ fi [[ ! -d $COMOUT/rundata ]] && mkdir -m 775 -p $COMOUT/rundata - echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile + if [ ${CFP_MP:-"NO"} = "YES" ]; then + echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile + else + echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile + fi nmoddef=`expr $nmoddef + 1` @@ -162,7 +166,11 @@ if [ "$NTASKS" -gt '1' ] then - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile + if [ ${CFP_MP:-"NO"} = "YES" ]; then + ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile + else + ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile + fi exit=$? else ./cmdfile diff --git a/scripts/exwave_prep.sh b/scripts/exwave_prep.sh index 97957e9f91..b37c3c0c63 100755 --- a/scripts/exwave_prep.sh +++ b/scripts/exwave_prep.sh @@ -311,9 +311,15 @@ # 3.a Gather and pre-process grib2 files ymdh=$ymdh_beg + if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP while [ "$ymdh" -le "$ymdh_end" ] do - echo "$USHwave/wave_g2ges.sh $ymdh > grb_$ymdh.out 2>&1" >> cmdfile + if [ ${CFP_MP:-"NO"} = "YES" ]; then + echo "$nm $USHwave/wave_g2ges.sh $ymdh > grb_$ymdh.out 2>&1" >> cmdfile + nm=`expr $nm + 1` + else + echo "$USHwave/wave_g2ges.sh $ymdh > grb_$ymdh.out 2>&1" >> cmdfile + fi ymdh=`$NDATE $WAV_WND_HOUR_INC $ymdh` done @@ -334,7 +340,11 @@ if [ "$wavenproc" -gt '1' ] then - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile + if [ ${CFP_MP:-"NO"} = "YES" ]; then + ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile + else + ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile + fi exit=$? else ./cmdfile @@ -626,30 +636,24 @@ # Prepare files for cfp process rm -f cmdfile touch cmdfile - chmod 744 cmfile + chmod 744 cmdfile ymdh_rtofs=${PDY}00 # RTOFS runs once daily ymdh_end=`$NDATE ${FHMAX_WAV_CUR} ${ymdh_rtofs}` NDATE_DT=${WAV_CUR_HF_DT} FLGHF='T' + if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP while [ "$ymdh_rtofs" -le "$ymdh_end" ] do # Timing has to be made relative to the single 00z RTOFS cycle for that PDY fhr_rtofs=`${NHOUR} ${ymdh_rtofs} ${PDY}00` fext='f' - if [ ${fhr_rtofs} -lt 0 ] - then -# Data from nowcast phase - fhr_rtofs=`expr 48 + ${fhr_rtofs}` - fext='n' - fi - - fhr_rtofs=`printf "%03d\n" ${fhr_rtofs}` + fh3_rtofs=`printf "%03d" "${fhr_rtofs#0}"` - curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fhr_rtofs}_1hrly_prog.nc - curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fhr_rtofs}_3hrly_prog.nc + curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_1hrly_prog.nc + curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_3hrly_prog.nc if [ -s ${curfile1h} ] && [ "${FLGHF}" = "T" ] ; then curfile=${curfile1h} @@ -671,7 +675,13 @@ echo ' ' fi - echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile > cur_$ymdh_rtofs.out 2>&1" >> cmdfile + if [ ${CFP_MP:-"NO"} = "YES" ]; then + echo "$nm $USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile + nm=`expr $nm + 1` + else + echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile + fi + if [ $fhr_rtofs -ge ${WAV_CUR_HF_FH} ] ; then NDATE_DT=${WAV_CUR_DT} fi @@ -691,7 +701,11 @@ if [ $wavenproc -gt '1' ] then - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile + if [ ${CFP_MP:-"NO"} = "YES" ]; then + ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile + else + ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile + fi exit=$? else chmod 744 ./cmdfile @@ -727,12 +741,13 @@ err=11;export err;${errchk} fi - rm -f cur.${WAVECUR_FID} +# rm -f cur.${WAVECUR_FID} for file in $files do - cat $file >> cur.${WAVECUR_FID} - rm -f $file + echo $file + cat -s $file >> cur.${WAVECUR_FID} +# rm -f $file done cp -f cur.${WAVECUR_FID} ${COMOUT}/rundata/${COMPONENTwave}.${WAVECUR_FID}.$cycle.cur diff --git a/ush/wave_prnc_cur.sh b/ush/wave_prnc_cur.sh index d94b78aff1..ba7656fc85 100755 --- a/ush/wave_prnc_cur.sh +++ b/ush/wave_prnc_cur.sh @@ -23,38 +23,40 @@ set -x ymdh_rtofs=$1 curfile=$2 +fhr=$3 +fh3=`printf "%03d" "${fhr#0}"` # Timing has to be made relative to the single 00z RTOFS cycle for that PDY mkdir -p rtofs_${ymdh_rtofs} cd rtofs_${ymdh_rtofs} -ncks -x -v sst,sss,layer_density $curfile cur_uv_${PDY}_${fext}${fhr}.nc -ncks -O -a -h -x -v Layer cur_uv_${PDY}_${fext}${fhr}.nc cur_temp1.nc +ncks -x -v sst,sss,layer_density $curfile cur_uv_${PDY}_${fext}${fh3}.nc +ncks -O -a -h -x -v Layer cur_uv_${PDY}_${fext}${fh3}.nc cur_temp1.nc ncwa -h -O -a Layer cur_temp1.nc cur_temp2.nc ncrename -h -O -v MT,time cur_temp2.nc ncrename -h -O -d MT,time cur_temp2.nc ncks -v u_velocity,v_velocity cur_temp2.nc cur_temp3.nc -mv -f cur_temp3.nc cur_uv_${PDY}_${fext}${fhr}_flat.nc +mv -f cur_temp3.nc cur_uv_${PDY}_${fext}${fh3}_flat.nc # Convert to regular lat lon file cp ${FIXwave}/weights_rtofs_to_r4320x2160.nc ./weights.nc # Interpolate to regular 5 min grid -$CDO remap,r4320x2160,weights.nc cur_uv_${PDY}_${fext}${fhr}_flat.nc cur_5min_01.nc +$CDO remap,r4320x2160,weights.nc cur_uv_${PDY}_${fext}${fh3}_flat.nc cur_5min_01.nc # Perform 9-point smoothing twice to make RTOFS data less noisy when # interpolating from 1/12 deg RTOFS grid to 1/6 deg wave grid if [ "WAV_CUR_CDO_SMOOTH" = "YES" ]; then $CDO -f nc -smooth9 cur_5min_01.nc cur_5min_02.nc - $CDO -f nc -smooth9 cur_5min_02.nc cur_glo_uv_${PDY}_${fext}${fhr}_5min.nc + $CDO -f nc -smooth9 cur_5min_02.nc cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc else - mv cur_5min_01.nc cur_glo_uv_${PDY}_${fext}${fhr}_5min.nc + mv cur_5min_01.nc cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc fi # Cleanup -rm -f cur_temp[123].nc cur_5min_??.nc cur_glo_uv_${PDY}_${fext}${fhr}.nc weights.nc +#### #rm -f cur_temp[123].nc cur_5min_??.nc cur_glo_uv_${PDY}_${fext}${fh3}.nc weights.nc if [ ${fhr} -gt ${WAVHINDH} ] then @@ -64,7 +66,7 @@ else fi rm -f cur.nc -ln -s cur_glo_uv_${PDY}_${fext}${fhr}_5min.nc cur.nc +ln -s cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc cur.nc ln -s ${DATA}/mod_def.${WAVECUR_FID} ./mod_def.ww3 $EXECwave/ww3_prnc 1> prnc_${WAVECUR_FID}_${ymdh_rtofs}.out 2>&1