From 1915aa921dfe2ef17799599e5a3084547caa3ca2 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 10 Jan 2020 18:36:32 +0000 Subject: [PATCH] Issue #8 - pulled in config.metp and modifications to two setup scripts --- parm/config/config.metp | 69 ++++++++++++++++ parm/config/config.vrfy | 114 ++++++-------------------- ush/rocoto/setup_expt.py | 12 ++- ush/rocoto/setup_workflow_fcstonly.py | 18 +++- 4 files changed, 116 insertions(+), 97 deletions(-) create mode 100755 parm/config/config.metp diff --git a/parm/config/config.metp b/parm/config/config.metp new file mode 100755 index 0000000000..68c16d5671 --- /dev/null +++ b/parm/config/config.metp @@ -0,0 +1,69 @@ +#!/bin/ksh -x + +########## config.metp ########## +# METplus verification step specific + +echo "BEGIN: config.metp" + +# Get task specific resources +. $EXPDIR/config.resources metp + +export RUN_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus +export RUN_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus +export RUN_PRECIP_STEP1="YES" # Run precip verification using METplus + + +#---------------------------------------------------------- +# METplus, Verify grid-to-grid, and/or grid-to-obs, and/or precipitation options +#---------------------------------------------------------- + +if [ "$CDUMP" = "gfs" ] ; then + if [ $RUN_GRID2GRID_STEP1 = "YES" -o $RUN_GRID2OBS_STEP1 = "YES" -o $RUN_PRECIP_STEP1 = "YES" ]; then + export HOMEverif_global=${HOMEgfs}/sorc/verif-global.fd + export VERIF_GLOBALSH=$HOMEverif_global/ush/run_verif_global_in_global_workflow.sh + ## INPUT DATA SETTINGS + export model_list=$PSLOT + export model_data_dir_list=$ARCDIR/.. + export model_fileformat_list="pgbf{lead?fmt=%H}.${CDUMP}.{init?fmt=%Y%m%d%H}.grib2" + export model_hpssdir_list=$ATARDIR/.. + export get_data_from_hpss="NO" + export hpss_walltime="10" + ## OUTPUT SETTINGS + export OUTPUTROOT=$RUNDIR/$CDUMP/$CDATE/vrfy/metplus_exp + export model_arch_dir_list=$ARCDIR/.. + export make_met_data_by="VALID" + export gather_by="VSDB" + ## DATE SETTINGS + export VRFYBACK_HRS="24" + ## METPLUS SETTINGS + export METplus_verbosity="INFO" + export MET_verbosity="2" + export log_MET_output_to_METplus="yes" + ## FORECAST VERIFICATION SETTINGS + export fhr_min=$FHMIN_GFS + export fhr_max=$FHMAX_GFS + # GRID-TO-GRID STEP 1 + export g2g1_type_list="anom pres sfc" + export g2g1_anl_name="self_anl" + export g2g1_anl_fileformat_list="pgbanl.gfs.{valid?fmt=%Y%m%d%H}.grib2" + export g2g1_grid="G002" + # GRID-TO-OBS STEP 1 + export g2o1_type_list="upper_air conus_sfc" + export g2o1_obtype_upper_air="ADPUPA" + export g2o1_grid_upper_air="G003" + export g2o1_fhr_out_upper_air="6" + export g2o1_obtype_conus_sfc="ONLYSF" + export g2o1_grid_conus_sfc="G104" + export g2o1_fhr_out_conus_sfc="3" + export g2o1_prepbufr_data_runhpss="YES" + # PRECIP STEP 1 + export precip1_obtype="ccpa" + export precip1_accum_length="24" + export precip1_model_bucket_list="06" + export precip1_model_varname_list="APCP" + export precip1_model_fileformat_list="pgbf{lead?fmt=%H}.gfs.{init?fmt=%Y%m%d%H}.grib2" + export precip1_grid="G211" + fi +fi + +echo "END: config.metp" diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy index 6bcc6e7f21..ea03e1785e 100755 --- a/parm/config/config.vrfy +++ b/parm/config/config.vrfy @@ -14,7 +14,6 @@ export CDFNL="gdas" # Scores verification against GDAS/GFS analysi export MKPGB4PRCP="YES" # Make 0.25-deg pgb files in ARCDIR for precip verification export VRFYFITS="YES" # Fit to observations -export VRFY_PCKG2RUN="VSDB" # "VSDB" = run VSDB verification package only, "METPLUS" = run METplus verification package only, "BOTH = run both verification packages export VSDB_STEP1="YES" # Populate VSDB database export VSDB_STEP2="NO" export VRFYG2OBS="YES" # Grid to observations, see note below if turning ON @@ -22,12 +21,9 @@ export VRFYPRCP="YES" # Precip threat scores export VRFYRAD="YES" # Radiance data assimilation monitoring export VRFYOZN="YES" # Ozone data assimilation monitoring export VRFYMINMON="YES" # GSI minimization monitoring -export VRFYTRAK="YES" # Hurricane track verification -export VRFYGENESIS="YES" # Cyclone genesis verification +export VRFYTRAK="YES" # Hurricane track verification +export VRFYGENESIS="YES" # Cyclone genesis verification export RUNMOS="NO" # whether to run entire MOS package -export RUN_METPLUS_GRID2GRID_STEP1="YES" # Run grid-to-grid verification using METplus -export RUN_METPLUS_GRID2OBS_STEP1="YES" # Run grid-to-obs verification using METplus -export RUN_METPLUS_PRECIP_STEP1="YES" # Run precip verification using METplus #------------------------------------------------- # Fit to Observations @@ -36,7 +32,7 @@ export RUN_METPLUS_PRECIP_STEP1="YES" # Run precip verification using METplus if [ $VRFYFITS = "YES" ]; then export PRVT=$HOMEgfs/fix/fix_gsi/prepobs_errtable.global - export HYBLEVS=$HOMEgfs/fix/fix_am/global_hyblev.l65.txt + export HYBLEVS=$HOMEgfs/fix/fix_am/global_hyblev.l${LEVS}.txt export CUE2RUN=$QUEUE if [ $machine = "WCOSS_C" ]; then @@ -62,84 +58,25 @@ if [ "$CDUMP" = "gfs" ] ; then ddd=`echo $CDATE |cut -c 1-8` #if [ $ddd -eq 5 -o $ddd -eq 10 ]; then export VSDB_STEP2 = "YES" ;fi - if [ $VRFY_PCKG2RUN = "VSDB" -o $VRFY_PCKG2RUN = "BOTH" ]; then - if [ $VSDB_STEP1 = "YES" -o $VSDB_STEP2 = "YES" -o $VRFYPRCP = "YES" -o $VRFYG2OBS = "YES" ]; then - export BACKDATEVSDB=24 # execute vsdbjob for the previous day - export VBACKUP_PRCP=24 # back up for QPF verification data - export vsdbsave="$NOSCRUB/archive/vsdb_data" # place to save vsdb database - export vsdbhome=$BASE_VERIF # location of global verification scripts - export VSDBSH="$vsdbhome/vsdbjob.sh" # VSDB job script - export vlength=$FHMAX_GFS # verification length - export vhr_rain=$FHMAX_GFS # verification length for precip - export ftyplist="pgbq" # verif. files used for computing QPF ETS scores - export ptyplist="PRATE" # precip types in GRIB: PRATE or APCP - export anltype="gfs" # default=gfs, analysis type (gfs or gdas) for verification - export rain_bucket=6 # prate in pgb files is 6-hr accumulated - - export VSDB_START_DATE="$SDATE" # starting date for vsdb maps - export webhost="emcrzdm.ncep.noaa.gov" # webhost(rzdm) computer - export webhostid="$USER" # webhost(rzdm) user name - export SEND2WEB="NO" # whether or not to send maps to webhost - export WEBDIR="/home/people/emc/www/htdocs/gmb/${webhostid}/vsdb/$PSLOT" - export mdlist="gfs $PSLOT " # exps (up to 10) to compare in maps - fi - fi -fi - - -#---------------------------------------------------------- -# METplus, Verify grid-to-grid, and/or grid-to-obs, and/or precipitation options -#---------------------------------------------------------- -# All these call $METPLUSSH - -if [ "$CDUMP" = "gfs" ] ; then - if [ $RUN_METPLUS_GRID2GRID_STEP1 = "YES" -o $RUN_METPLUS_GRID2OBS_STEP1 = "YES" -o $RUN_METPLUS_PRECIP_STEP1 = "YES" ]; then - if [ $VRFY_PCKG2RUN = "METPLUS" -o $VRFY_PCKG2RUN = "BOTH" ]; then - export HOMEverif_global=${HOMEgfs}/sorc/verif-global.fd - export VERIF_GLOBALSH=$HOMEverif_global/ush/run_verif_global_in_global_workflow.sh - export RUNBATCH="NO" - ## INPUT DATA SETTINGS - export model_list=$PSLOT - export model_data_dir_list=$ARCDIR/.. - export model_fileformat_list="pgbf{lead?fmt=%H}.${CDUMP}.{init?fmt=%Y%m%d%H}" - export model_hpssdir_list=$ATARDIR/.. - export get_data_from_hpss="NO" - export hpss_walltime="10" - ## OUTPUT SETTINGS - export OUTPUTROOT=$RUNDIR/$CDUMP/$CDATE/vrfy/metplus_exp - export model_arch_dir_list=$ARCDIR/.. - export make_met_data_by="VALID" - export gather_by="VSDB" - ## DATE SETTINGS - export VRFYBACK_HRS="24" - ## METPLUS SETTINGS - export METplus_verbosity="INFO" - export MET_verbosity="2" - export log_MET_output_to_METplus="yes" - ## FORECAST VERIFICATION SETTINGS - export fhr_min=$FHMIN_GFS - export fhr_max=$FHMAX_GFS - # GRID-TO-GRID STEP 1 - export g2g1_type_list="anom pres sfc" - export g2g1_anl_name="self_anl" - export g2g1_anl_fileformat_list="pgbanl.gfs.{valid?fmt=%Y%m%d%H}" - export g2g1_grid="G002" - # GRID-TO-OBS STEP 1 - export g2o1_type_list="upper_air conus_sfc" - export g2o1_obtype_upper_air="ADPUPA" - export g2o1_grid_upper_air="G003" - export g2o1_fhr_out_upper_air="6" - export g2o1_obtype_conus_sfc="ONLYSF" - export g2o1_grid_conus_sfc="G104" - export g2o1_fhr_out_conus_sfc="3" - # PRECIP STEP 1 - export precip1_obtype="ccpa" - export precip1_accum_length="24" - export precip1_model_bucket_list="06" - export precip1_model_varname_list="APCP" - export precip1_model_fileformat_list="pgbf{lead?fmt=%H}.gfs.{init?fmt=%Y%m%d%H}" - export precip1_grid="G211" - fi + if [ $VSDB_STEP1 = "YES" -o $VSDB_STEP2 = "YES" -o $VRFYPRCP = "YES" -o $VRFYG2OBS = "YES" ]; then + export BACKDATEVSDB=24 # execute vsdbjob for the previous day + export VBACKUP_PRCP=24 # back up for QPF verification data + export vsdbsave="$NOSCRUB/archive/vsdb_data" # place to save vsdb database + export vsdbhome=$BASE_VERIF # location of global verification scripts + export VSDBSH="$vsdbhome/vsdbjob.sh" # VSDB job script + export vlength=$FHMAX_GFS # verification length + export vhr_rain=$FHMAX_GFS # verification length for precip + export ftyplist="pgbq" # verif. files used for computing QPF ETS scores + export ptyplist="PRATE" # precip types in GRIB: PRATE or APCP + export anltype="gfs" # default=gfs, analysis type (gfs or gdas) for verification + export rain_bucket=6 # prate in pgb files is 6-hr accumulated + + export VSDB_START_DATE="$SDATE" # starting date for vsdb maps + export webhost="emcrzdm.ncep.noaa.gov" # webhost(rzdm) computer + export webhostid="$USER" # webhost(rzdm) user name + export SEND2WEB="NO" # whether or not to send maps to webhost + export WEBDIR="/home/people/emc/www/htdocs/gmb/${webhostid}/vsdb/$PSLOT" + export mdlist="gfs $PSLOT " # exps (up to 10) to compare in maps fi fi @@ -205,12 +142,11 @@ if [ "$VRFYTRAK" = "YES" ]; then export FHOUT_CYCLONE=6 export FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 )) fi - if [ $machine = "THEIA" ]; then - export COMROOTp1="/scratch4/NCEPDEV/rstprod/com" - export COMINsyn="$COMROOTp1/arch/prod/syndat" - elif [ $machine = "HERA" ]; then + if [ $machine = "HERA" ]; then export COMROOTp1="/scratch1/NCEPDEV/rstprod/com" export COMINsyn=${COMINsyn:-${COMROOTp1}/arch/prod/syndat} + else + export COMINsyn=${COMINsyn:-${COMROOT}/gfs/prod/syndat} fi fi diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py index 0d66a3691d..a4630f74d2 100755 --- a/ush/rocoto/setup_expt.py +++ b/ush/rocoto/setup_expt.py @@ -76,12 +76,12 @@ def edit_baseconfig(): top = os.path.abspath(os.path.join( os.path.abspath(here), '../..')) - # make a copy of the default before editing - shutil.copy(base_config, base_config + '.default') + if os.path.exists(base_config): + os.unlink(base_config) print '\nSDATE = %s\nEDATE = %s' % (idate, edate) - with open(base_config + '.default', 'rt') as fi: - with open(base_config + '.new', 'wt') as fo: + with open(base_config + '.emc.dyn', 'rt') as fi: + with open(base_config, 'wt') as fo: for line in fi: line = line.replace('@MACHINE@', machine.upper()) \ .replace('@PSLOT@', pslot) \ @@ -99,12 +99,10 @@ def edit_baseconfig(): if 'ICSDIR' in line: continue fo.write(line) - os.unlink(base_config) - os.rename(base_config + '.new', base_config) print '' print 'EDITED: %s/config.base as per user input.' % expdir - print 'DEFAULT: %s/config.base.default is for reference only.' % expdir + print 'DEFAULT: %s/config.base.emc.dyn is for reference only.' % expdir print 'Please verify and delete the default file before proceeding.' print '' diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py index e936b7efbf..67d6759cec 100755 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ b/ush/rocoto/setup_workflow_fcstonly.py @@ -28,7 +28,7 @@ import workflow_utils as wfu -taskplan = ['getic', 'fv3ic', 'fcst', 'post', 'vrfy', 'arch'] +taskplan = ['getic', 'fv3ic', 'fcst', 'post', 'vrfy', 'metp', 'arch'] def main(): parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) @@ -321,6 +321,22 @@ def get_workflow(dict_configs, cdump='gdas'): tasks.append(task) tasks.append('\n') + # metp + deps = [] + dep_dict = {'type':'metatask', 'name':'%spost' % cdump} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type':'task', 'name':'%sarch' % cdump, 'offset':'-&INTERVAL;'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + metpcase = rocoto.create_envar(name='METPCASE', value='#metpcase#') + metpenvars = envars + [metpcase] + varname1 = 'metpcase' + varval1 = 'g2g1 g2o1 pcp1' + task = wfu.create_wf_task('metp', cdump=cdump, envar=metpenvars, dependency=dependencies, + metatask='metp', varname=varname1, varval=varval1) + tasks.append(task) + tasks.append('\n') + # arch deps = [] dep_dict = {'type':'metatask', 'name':'%spost' % cdump}