diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 032f9e9161..317bf8857a 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -437,15 +437,34 @@ cat > itag <- + Sample config file for the work of importing RRFS_dev1 tasks + to SRW +user: + RUN_ENVIR: nco + MACHINE: hera + ACCOUNT: zrtrr + SERVICE_ACCOUNT: zrtrr + HPSS_ACCOUNT: zrtrr +platform: + RESERVATION: "" + SST_update_hour: 1 + GVF_update_hour: 4 + SNOWICE_update_hour: 0 + DOMAIN_PREGEN_BASEDIR: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/lam" + FIXgsi: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/gsi" + FIXcrtm: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/crtm/CRTM_v2.3.0" + FIXcrtmupp: "/scratch2/BMC/gsd-hpcs/Daniel.Abdi/rrfs_dev/ufs-srweather-app/regional_workflow/fix/crtm/CRTM_v2.4.0" +workflow: + TAG: RRFS_CONUS_3km_ + USE_CRON_TO_RELAUNCH: false + EXPT_SUBDIR: RRFS_CONUS_3km + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: RRFS_CONUS_3km + DATE_FIRST_CYCL: '2022072000' + DATE_LAST_CYCL: '2022072100' + FCST_LEN_HRS: 3 + FCST_LEN_HRS_SPINUP: 1 + POSTPROC_LEN_HRS: 3 + POSTPROC_LONG_LEN_HRS: 12 + PREEXISTING_DIR_METHOD: rename + DEBUG: true + VERBOSE: true + COMPILER: intel + CYCL_HRS: [0, 12] + CYCL_HRS_SPINSTART: [3, 15] + CYCL_HRS_PRODSTART: [9, 21] + FCST_LEN_HRS_CYCLES: [12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3, 12, 3, 3] + DA_CYCLE_INTERV: 1 + BOUNDARY_LEN_HRS: 18 + BOUNDARY_PROC_GROUP_NUM: 10 +rrfs: + DO_DACYCLE: false # turn on later + DO_SURFACE_CYCLE: false # turn on later + DO_SPINUP: false # turn on later + DO_NONVAR_CLDANAL: false # turn on later + DO_REFL2TTEN: false + DO_NLDN_LGHT: false + DO_SOIL_ADJUST: false # turn on later + DO_RADDA: false + DO_BUFRSND: false + DO_RRFS_DEV: true + DO_POST_SPINUP: false # turn on later + DO_POST_PROD: false # turn on later +cycledefs: + INITIAL_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0300 ", workflow.DATE_LAST_CYCL[0:8], "2300 12:00:00"]|join }}' + BOUNDARY_CYCLEDEF: '{{ [workflow.DATE_FIRST_CYCL[0:8], "0000 ", workflow.DATE_LAST_CYCL[0:8], "2300 06:00:00"]|join }}' + PROD_CYCLEDEF: "00 10,11,13,14,16,17,19,20,22,23 20-21 7 2022 *" + PRODLONG_CYCLEDEF: "00 9-23/3 20-21 7 2022 *" + SPINUP_CYCLEDEF: "00 03-08,15-20 20-21 7 2022 *" +#gsi: +# HYBENSMEM_NMIN: 66 +nco: + NET: rrfs + envir: para + model_ver: v1.0 + RUN: RRFS_conus_3km +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: FV3GFS + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{yy}{jjj}{hh}0000{fcst_hr:02d}' + EXTRN_MDL_ICS_OFFSET_HRS: 3 + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_ICS: "/scratch2/BMC/zrtrr/rli/data/gfs/0p25deg/grib2" +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: FV3GFS + FV3GFS_FILE_FMT_LBCS: grib2 + EXTRN_MDL_FILES_LBCS: + - '{yy}{jjj}{hh}0000{fcst_hr:02d}' + LBC_SPEC_INTVL_HRS: 1 + EXTRN_MDL_LBCS_OFFSET_HRS: 6 + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_LBCS: "/scratch2/BMC/zrtrr/rli/data/gfs/0p25deg/grib2" +task_run_fcst: + DT_ATMOS: 60 + RESTART_INTERVAL: 1 + QUILTING: true +# IO_LAYOUT_Y: 1 +# FH_DFI_RADAR: "0.0,0.25,0.5" +task_run_post: + USE_CUSTOM_POST_CONFIG_FILE: true + POST_OUTPUT_DOMAIN_NAME: RRFS_CONUS_3km + CUSTOM_POST_CONFIG_FP: '{{ [user.PARMdir, "upp/postxconfig-NT-fv3lam_rrfs.txt"]|path_join }}' +# CUSTOM_POST_PARAMS_FP: '{{ [user.PARMdir, "upp/params_grib2_tbl_new"]|path_join }}' +# TESTBED_FIELDS_FN: "testbed_fields_bgdawp.txt" +# TESTBED_FIELDS_FN2: "testbed_fields_bgrd3d.txt" +#task_process_radarref: +# RADARREFL_TIMELEVEL: [0] +#task_run_prdgen: +# ADDNL_OUTPUT_GRIDS: [] +global: + DO_ENSEMBLE: false + NUM_ENS_MEMBERS: 9 + HALO_BLEND: 20 + PRINT_DIFF_PGR: true +rocoto: + tasks: + taskgroups: '{{ ["parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 02:45:00 + task_make_ics_mem#mem#: + nnodes: 3 + ppn: 20 + task_make_lbcs_mem#mem#: + walltime: 02:00:00 + nnodes: 3 + ppn: 20 + metatask_run_ens_post: + metatask_run_post: + task_run_post_mem#mem#_f#fhr#: + nnodes: 1 + ppn: 40 + #task_run_ref2tten: + # ppn: 1 + #task_run_nonvarcldanl: + # ppn: '{{ task_run_fcst.IO_LAYOUT_Y }}' + #task_run_anal: + # cores: 240 #anal + # cores: 80 #observer diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 2ea8fb349e..5b95e72841 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -5,7 +5,7 @@ metadata: description: >- Default configuration for an experiment. The valid values for most of the parameters are specified in valid_param_vals.yaml - version: !!str '1.0' + version: !!str '2.0' #---------------------------- # USER config parameters #---------------------------- @@ -51,9 +51,19 @@ user: # ACCOUNT: # The account under which to submit jobs to the queue. # + # SERVICE_ACCOUNT: + # The account under which to submit non-reservation jobs to the queue. + # Defaults to ACCOUNT if not set. + # + # HPSS_ACCOUNT: + # The account under which to submit non-reservation jobs to the queue. + # Defaults to SERVICE_ACCOUNT if not set. + # #----------------------------------------------------------------------- MACHINE: "BIG_COMPUTER" ACCOUNT: "" + SERVICE_ACCOUNT: '{{ user.ACCOUNT }}' + HPSS_ACCOUNT: '{{ user.ACCOUNT }}' HOMEdir: '{{ user.HOMEdir }}' USHdir: '{{ user.USHdir }}' @@ -159,10 +169,52 @@ platform: # If this is not set or set to an empty string, it will be (re)set to a # machine-dependent value. # + # PARTITION_GRAPHICS + # Partition to which graphics tasks are submitted + # + # QUEUE_GRAPHICS + # Queue for graphics tasks + # + # PARTITION_ANALYSIS + # Partion to which analysis tasks are submitted + # + # QUEUE_ANALYSIS: + # The queue or QOS to which the task that runs a analysis is submitted. + # If this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. + # + # PARTITION_PRDGEN: + # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), + # the partition to which the task that remaps output grids is submitted. If + # this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. This is not used if SCHED is not set to + # "slurm". + # + # QUEUE_PRDGEN: + # The queue or QOS to which the task that prodgen is submitted. + # If this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. + # + # PARTITION_POST: + # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), + # the partition to which the task that upp is submitted. + # + # QUEUE_POST: + # The queue or QOS to which the task that upp is submitted. + # If this is not set or set to an empty string, it will be (re)set to a + # machine-dependent value. + # + # RESERVATION: + # The reservation for major tasks. + # + # RESERVATION_POST: + # The reservation for post tasks. + # # REMOVE_MEMORY: # Boolean flag determining whether to remove the memory flag for the # Rocoto XML. Some platforms are not configured to accept the memory # flag, so must not be included at all in the XML. + # #----------------------------------------------------------------------- # WORKFLOW_MANAGER: "" @@ -179,6 +231,16 @@ platform: QUEUE_HPSS: "" PARTITION_FCST: "" QUEUE_FCST: "" + PARTITION_GRAPHICS: "" + QUEUE_GRAPHICS: "" + PARTITION_ANALYSIS: "" + QUEUE_ANALYSIS: "" + PARTITION_PRDGEN: "" + QUEUE_PRDGEN: "" + PARTITION_POST: "" + QUEUE_POST: "" + RESERVATION: "" + RESERVATION_POST: "" REMOVE_MEMORY: False # #----------------------------------------------------------------------- @@ -223,9 +285,12 @@ platform: # #----------------------------------------------------------------------- # + # SCHED_NATIVE_CMD # Allows an extra parameter to be passed to SCHEDULER (SLURM/PBSPRO) via # XML Native command # + #----------------------------------------------------------------------- + # SCHED_NATIVE_CMD: "" # @@ -397,6 +462,21 @@ platform: # FIXsfc: # System directory where surface climatology data is located # + # FIXgsi: + # System directory where GSI fixed files are loacated + # + # FIXsmoke: + # System directory where smoke and dust fixed files are loacated + # + # FIXbufr: + # System directory where bufrsnd fixed files are loacated + # + # FIXcrtm: + # System directory where CRTM fixed files are loacated + # + # FIXcrtmupp: + # System directory where CRTM fixed files specifically for UPP are loacated + # #----------------------------------------------------------------------- # FIXgsm: "" @@ -405,6 +485,11 @@ platform: FIXorg: "" FIXsfc: "" FIXshp: "" + FIXgsi: "" + FIXsmoke: "" + FIXbufr: "" + FIXcrtm: "" + FIXcrtmupp: "" # #----------------------------------------------------------------------- # @@ -435,6 +520,60 @@ platform: COMINgfs: "" COMINgefs: "" COMINairnow: "/path/to/real/time/airnow/data" + # + #----------------------------------------------------------------------- + # + # Setup default observation locations for data assimilation: + # + # OBSPATH: observation BUFR file path + # OBSPATH_PM: + # OBSPATH_NSSLMOSIAC: location of NSSL radar reflectivity + # LIGHTNING_ROOT: location of lightning observations + # ENKF_FCSTL: location of global ensemble forecast + # FFG_DIR: location of flash flood guidance for QPF comparison + # + # Setup default locations for global SST and update time: + # SST_ROOT: locations of global SST + # SST_update_hour: cycle time for updating SST + # + # Setup default locations for GVF and update time: + # GVF_ROOT: locations of GVF observations + # GVF_update_hour: cycle time for updating GVF + # + # Setup default locations for IMS snow/ice and update time: + # IMSSNOW_ROOT: locations of IMS snow/ice observations + # SNOWICE_update_hour: cycle time for updating snow/ice + # + # Setup default resource data locations for soil surgery and time: + # RAPHRR_SOIL_ROOT: locations of RAP/HRRR forecast netcdf files + # SOIL_SURGERY_time: cycle time for soil surgery + # + # Setup default locations for FIRE_RRFS files and update time + # FIRE_RAVE_DIR + # FIRE_RRFS_ROOT + # FIRE_RRFS_update_hour + # + #----------------------------------------------------------------------- + # + OBSPATH: "" + OBSPATH_PM: "" + OBSPATH_NSSLMOSIAC: "" + LIGHTNING_ROOT: "" + ENKF_FCST: "" + FFG_DIR: "" + SST_ROOT: "" + GVF_ROOT: "" + IMSSNOW_ROOT: "" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + FIRE_RRFS_ROOT: "" + AIRCRAFT_REJECT: "" + SFCOBS_USELIST: "" + + SST_update_hour: 99 + GVF_update_hour: 99 + SNOWICE_update_hour: 99 + SOIL_SURGERY_time: 9999999999 #----------------------------- # WORKFLOW config parameters @@ -443,11 +582,16 @@ workflow: # #----------------------------------------------------------------------- # + # WORKFLOW_ID # Unique ID for workflow run that will be set in setup.py # + # TAG + # Prepend all task name with this name if specified + # #----------------------------------------------------------------------- # WORKFLOW_ID: !nowtimestamp '' + TAG: "" # #----------------------------------------------------------------------- # @@ -630,7 +774,7 @@ workflow: FV3_NML_BASE_SUITE_FN: "input.nml.FV3" FV3_NML_YAML_CONFIG_FN: "FV3.input.yml" FV3_NML_BASE_ENS_FN: "input.nml.base_ens" - FV3_NML_FN: "input.nml" + FV3_NML_FN: '{{ FV3_NML_BASE_SUITE_FN[:-4] }}' FV3_EXEC_FN: "ufs_model" DATA_TABLE_FN: "data_table" @@ -658,6 +802,10 @@ workflow: FIELD_TABLE_FP: '{{ [EXPTDIR, FIELD_TABLE_FN]|path_join }}' NEMS_CONFIG_FP: '{{ [EXPTDIR, NEMS_CONFIG_FN]|path_join }}' FV3_NML_FP: '{{ [EXPTDIR, FV3_NML_FN]|path_join }}' + FV3_NML_CYCSFC_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_cycsfc"]|join ]|path_join }}' + FV3_NML_RESTART_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_restart"]|join ]|path_join }}' + FV3_NML_STOCH_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_stoch"]|join ]|path_join }}' + FV3_NML_RESTART_STOCH_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_restart_stoch"]|join ]|path_join }}' FCST_MODEL: "ufs-weather-model" WFLOW_XML_FN: "FV3LAM_wflow.xml" @@ -701,7 +849,7 @@ workflow: # #----------------------------------------------------------------------- # - FIXdir: '{{ EXPTDIR if rocoto.tasks.get("task_make_grid") else [user.HOMEdir, "fix"]|path_join }}' + FIXdir: '{{ [EXPTDIR, "fix"]|path_join if rocoto.tasks.get("task_make_grid") else [user.HOMEdir, "fix"]|path_join }}' FIXam: '{{ [FIXdir, "fix_am"]|path_join }}' FIXclim: '{{ [FIXdir, "fix_clim"]|path_join }}' FIXlam: '{{ [FIXdir, "fix_lam"]|path_join }}' @@ -849,6 +997,60 @@ workflow: # run. By default, set it to a 1-item list containing the standard # fcst length. # + # CYCL_HRS: + # An array containing the hours of the day at which to launch forecasts. + # Forecasts are launched at these hours on each day from DATE_FIRST_CYCL + # to DATE_LAST_CYCL, inclusive. Each element of this array must be a + # two-digit string representing an integer that is less than or equal to + # 23, e.g. "00", "03", "12", "23". + # + # CYCL_HRS_SPINSTART: + # An array containing the hours of the day at which the spin up cycle starts. + # + # CYCL_HRS_PRODSTART: + # An array containing the hours of the day at which the product cycle starts, + # from cold start input or from spin-up cycle forcast + # + # CYCL_HRS_PRODSTART_ENS: + # An array containing the hours of the day at which the product cycle starts, + # from cold start input or from spin-up cycle forcast, for the ensemble. + # this is only needed for locating the RRFS ensemble files for the the deterministic hybrid analysis. + # + # CYCL_HRS_RECENTER: + # An array containing the hours of the day at which the ensemble recenter is on + # + # CYCL_HRS_STOCH + # An array containing the hours of the day at which the stochastics physcis is on + # this might include: SPPT, SHUM, SKEB, SPP, LSM_SPP + # + # BOUNDARY_LEN_HRS + # The length of boundary condition for normal forecast, in integer hours. + # + # BOUNDARY_LONG_LEN_HRS + # The length of boundary condition for long forecast, in integer hours. + # + # BOUNDARY_PROC_GROUP_NUM + # The number of groups used to run make_lbcs, in integer from 1 to forecast longest hours. + # + # FCST_LEN_HRS_SPINUP: + # The length of each forecast in spin up cycles, in integer hours. + # + # FCST_LEN_HRS_CYCLES: + # The length of forecast for each cycle, in integer hours. + # When it empty, all forecast will be FCST_LEN_HRS + # + # DA_CYCLE_INTERV: + # Data assimilation cycle interval, in integer hours for now. + # + # POSTPROC_LEN_HRS: + # The length of post process, in integer hours. + # + # POSTPROC_LONG_LEN_HRS: + # The length of long post process, in integer hours. + # + # CYCL_HRS_HYB_FV3LAM_ENS: + # An array containing the hours of the day at which the GSI hybrid using FV3LAM ensemeble. + # #----------------------------------------------------------------------- # DATE_FIRST_CYCL: "YYYYMMDDHH" @@ -857,8 +1059,22 @@ workflow: FCST_LEN_HRS: 24 FCST_LEN_CYCL: - '{{ FCST_LEN_HRS }}' - LONG_FCST_LEN: '{% if FCST_LEN_HRS < 0 %}{{ FCST_LEN_CYCL|max }}{% else %}{{ FCST_LEN_HRS }}{% endif %}' - + CYCL_HRS: [] + CYCL_HRS_SPINSTART: [] + CYCL_HRS_PRODSTART: [] + CYCL_HRS_PRODSTART_ENS: [] + CYCL_HRS_RECENTER: [] + CYCL_HRS_STOCH: [] + BOUNDARY_LEN_HRS: 0 + BOUNDARY_LONG_LEN_HRS: 0 + BOUNDARY_PROC_GROUP_NUM: 1 + POSTPROC_LEN_HRS: 1 + POSTPROC_LONG_LEN_HRS: 1 + FCST_LEN_HRS_SPINUP: 1 + FCST_LEN_HRS_CYCLES: [] + DA_CYCLE_INTERV: 1 + CYCL_HRS_HYB_FV3LAM_ENS: [ "99" ] + LONG_FCST_LEN: '{% if FCST_LEN_HRS < 0 %}{{ FCST_LEN_CYCL|min }}{% else %}{{ FCST_LEN_HRS }}{% endif %}' # #----------------------------------------------------------------------- # @@ -918,10 +1134,17 @@ workflow: # messages are printed to out. Note that if DEBUG is set to TRUE, then # VERBOSE will also get reset to TRUE if it isn't already. # + # SAVE_CYCLE_LOG: + # This is a flag that determines whether or not save + # the information related to data assimilation cycling, such as background + # used in each cycle + # #----------------------------------------------------------------------- # VERBOSE: true DEBUG: false + SAVE_CYCLE_LOG: true + # #----------------------------------------------------------------------- # @@ -960,6 +1183,7 @@ workflow: COLDSTART: true WARMSTART_CYCLE_DIR: "/path/to/warm/start/cycle/dir" + #---------------------------- # NCO specific variables #----------------------------- @@ -994,6 +1218,12 @@ nco: # LOGBASEDIR: # Directory in which the log files from the workflow tasks will be placed. # + # NWGES: + # The beginning portion of the directory that will contain the output + # files from the forecast for a given cycle. For a cycle + # that starts on the date specified by yyyymmdd and hour specified by hh + # ${NWGES}/yyyymmddhh + # # For more information on NCO standards # # https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf @@ -1004,6 +1234,7 @@ nco: NET: "rrfs" RUN: "rrfs" model_ver: "v1.0.0" + OPSROOT: '{{ workflow.EXPT_BASEDIR }}/../nco_dirs' COMROOT: '{{ OPSROOT }}/com' PACKAGEROOT: '{{ OPSROOT }}/packages' @@ -1014,6 +1245,35 @@ nco: COMIN_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}' COMOUT_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}' + # + # New additions from RRFS_dev1 + # + NWGES: '{{ OPSROOT }}/nwges' + NWGES_BASEDIR: '{{ NWGES }}' + RRFSE_NWGES: '{{ OPSROOT }}/nwges' + RRFSE_NWGES_BASEDIR: '{{ RRFSE_NWGES }}' + ENSCTRL_NWGES: '{{ OPSROOT }}/nwges' + ENSCTRL_NWGES_BASEDIR: '{{ ENSCTRL_NWGES}}' + + ENSCTRL_COMIN: '{{ COMIN_BASEDIR}}' + ENSCTRL_COMOUT: '{{ COMOUT_BASEDIR}}' + ENSCTRL_COMROOT: '{{ ENSCTRL_COMOUT }}' + ENSCTRL_COMIN_BASEDIR: '{{ ENSCTRL_COMIN }}' + ENSCTRL_COMOUT_BASEDIR: '{{ ENSCTRL_COMROOT}}/{{ envir}}' + + # + #----------------------------------------------------------------------- + # + # FG_ROOTDIR: + # First Guess Root Directory, APP will find corresponding first guess + # fields from this directory. RRFS will find FG under NWGES_BASEDIR, + # but we needs to explicitly specify where to find FG for RTMA. + # So this parameter only matters for RTMA + # + #----------------------------------------------------------------------- + # + FG_ROOTDIR: "" + # #----------------------------------------------------------------------- # @@ -1031,6 +1291,260 @@ nco: MAILTO: "" MAILCC: "" +#---------------------------- +# DO_ parameters. These look like workflow switches since some +# of them are used in FV3LAM_wflow.xml +#----------------------------- +rrfs: + # + #----------------------------------------------------------------------- + # + # Set parameters associated with running retrospective experiments. Definitions: + # + # DO_RRFS_DEV: + # Flag to turn on RRFS dev workflow. Should be removed + # in the future once SRW and RRFS dev workflows are completely integrated + # + # DO_SPINUP: + # Flag turn on the spin-up cycle. + # + # DO_POST_SPINUP: + # Flag turn on the UPP for spin-up cycle. + # + # DO_POST_PROD: + # Flag turn on the UPP for prod cycle. + # + #----------------------------------------------------------------------- + # + DO_RRFS_DEV: false # Turn this on to test RRFS dev workflow + DO_POST_PROD: false # Turn this off temporarily + DO_SPINUP: false + DO_POST_SPINUP: false + # + #----------------------------------------------------------------------- + # + # Set switches associated with ensembles. Definitions: + # + # DO_ENSCONTROL: + # In ensemble mode, whether or not to run member 1 as control member + # + # DO_GSIOBSERVER: + # Decide whether or not to run GSI observer + # + # DO_ENKFUPDATE: + # Decide whether or not to run EnKF update for the ensemble members + # + # DO_ENKF_RADAR_REF: + # Decide whether or not to run Radar Reflectivity EnKF update for the ensemble members + # + # DO_ENVAR_RADAR_REF: + # Decide whether or not to run Radar Reflectivity hybrid analysis + # + # DO_ENVAR_RADAR_REF_ONCE: + # Decide whether or not to run Radar Reflectivity hybrid analysis simultaneously with other observations + # + # DO_RECENTER: + # Decide whether or not to run recenter for the ensemble members + # + # DO_SAVE_DA_OUTPUT: + # Decide whether or not to run save_input after the DA analysis + # + # DO_ENS_GRAPHICS: + # Flag to turn on/off ensemble graphics. Turns OFF deterministic + # graphics. + # + # DO_ENSPOST: + # Flag to turn on/off python ensemble postprocessing for WPC testbeds. + # + # DO_ENSINIT: + # Decide whether or not to do ensemble initialization by running 1 timestep ensemble + # forecast and recentering on the deterministic analysis + # + # DO_ENSFCST: + # Flag that determines whether to run ensemble free forecasts (for + # each set of specified cycles). If this is set to "TRUE", NUM_ENS_MEMBERS_FCST + # forecasts are run for each specified cycle, each with a different set of stochastic + # seed values. + # + # DO_ENS_RADDA: + # It decides whether to include radiance DA in EnKF or not. Note that when one sets + # 'DO_ENS_RADDA="TRUE"', the radiance DA must be true, i.e., 'DO_RADDA="TRUE"'. This + # is because the radiance DA in EnKF relies the radiance procedures in the GSI-observer, + # which is mainly controled by DO_RADDA. + # + #----------------------------------------------------------------------- + # + DO_ENSFCST: false + DO_ENSCONTROL: false + DO_GSIOBSERVER: false + DO_ENKFUPDATE: false + DO_ENKF_RADAR_REF: false + DO_ENVAR_RADAR_REF: false + DO_ENVAR_RADAR_REF_ONCE: false + DO_RECENTER: false + DO_ENS_GRAPHICS: false + DO_ENSPOST: false + DO_ENSINIT: false + DO_SAVE_DA_OUTPUT: false + DO_ENS_RADDA: false + # + #----------------------------------------------------------------------- + # + # Set switches associated with running data assimilation. Definitions: + # + # DO_DACYCLE: + # Flag that determines whether to run a data assimilation cycle. + # + # DO_SDDACYCLE: + # Flag that determines whether to run a SMOKE and DUST data assimilation cycle. + # + # DO_SURFACE_CYCLE: + # Flag that determines whether to continue cycle surface fields. + # + # DO_SOIL_ADJUST: + # Flag that determines whether to adjust soil T and Q based on + # the lowest level T/Q analysis increments. + # + # DO_UPDATE_BC: + # Flag that determines whether to update boundary conditions based on the + # analysis results + # + # DO_RADDA: + # Flag that determines whether to assimilate satellite radiance data + # + # DO_BUFRSND: + # Decide whether or not to run EMC BUFR sounding + # + # DO_SMOKE_DUST + # Flag turn on smoke and dust for RRFS-SD + # + #----------------------------------------------------------------------- + # + DO_DACYCLE: false + DO_SDDACYCLE: false + DO_SURFACE_CYCLE: false + DO_SOIL_ADJUST: false + DO_UPDATE_BC: false + DO_RADDA: false + DO_BUFRSND: false + DO_SMOKE_DUST: false + # + #----------------------------------------------------------------------- + # + # Parameters for JEDI options + # DO_JEDI_ENVAR_IODA: + # Flag turn on the JEDI-IODA converters for EnVAR. It requires GSI + # to produce NetCDF diag files + #----------------------------------------------------------------------- + # + DO_JEDI_ENVAR_IODA: false + # + #----------------------------------------------------------------------- + # + # Parameters for analysis options + # DO_NONVAR_CLDANAL: + # Flag turn on the non-var cloud analysis. + # DO_REFL2TTEN: + # Flag turn on the radar reflectivity to temperature tendenecy. + # DO_NLDN_LGHT + # Flag turn on processing NLDN NetCDF lightning data + # + #----------------------------------------------------------------------- + # + DO_NONVAR_CLDANAL: false + DO_REFL2TTEN: false + DO_NLDN_LGHT: false + DO_SMOKE_DUST: false + # + #----------------------------------------------------------------------- + # + # IS_RTMA: + # If true, some ICs,LBCs,GSI rocoto tasks will be turned off + # + #----------------------------------------------------------------------- + # + IS_RTMA: false + +#---------------------------------- +# Start times for different setups +#---------------------------------- +start_time: + START_TIME_SPINUP: 01:10:00 + START_TIME_PROD: 02:20:00 + START_TIME_CONVENTIONAL_SPINUP: 00:40:00 + START_TIME_LATE_ANALYSIS: 01:40:00 + START_TIME_CONVENTIONAL: 00:40:00 + START_TIME_NSSLMOSIAC: 00:45:00 + START_TIME_LIGHTNINGNC: 00:45:00 + START_TIME_PROCSMOKE: 00:45:00 + +#---------------------------------- +# Cycle definiton for each group +#---------------------------------- +cycledefs: + #----------------------------------------------------------------------- + # + # Set cycle definition for each group. The cycle definition sets the cycle + # time that the group will run. It has two way to set up: + # 1) 00 HHs DDs MMs YYYYs * + # HHs can be "01-03/01" or "01,02,03" or "*" + # DDs,MMs can be "01-03" or "01,02,03" or "*" + # YYYYs can be "2020-2021" or "2020,2021" or "*" + # 2) start_time(YYYYMMDDHH00) end_time(YYYYMMDDHH00) interval(HH:MM:SS) + # for example: 202104010000 202104310000 12:00:00 + # The default cycle definition is: + # "00 01 01 01 2100 *" + # which will likely never get to run. + # + # Definitions: + # + # AT_START_CYCLEDEF: + # cycle definition for "at start" group + # This group runs: make_grid, make_orog, make_sfc_climo + # + # INITIAL_CYCLEDEF: + # cycle definition for "initial" group + # This group runs get_extrn_ics, make_ics + # + # BOUNDARY_CYCLEDEF: + # cycle definition for "boundary" group + # This group runs: get_extrn_lbcs,make_lbcs + # + # BOUNDARY_LONG_CYCLEDEF: + # cycle definition for "boundary_long" group + # This group runs: get_extrn_lbcs_long,make_lbcs + # + # SPINUP_CYCLEDEF: + # cycle definition for spin-up cycle group + # This group runs: anal_gsi_input_spinup and data process, run_fcst_spinup, run_post_spinup + # + # PROD_CYCLEDEF: + # cycle definition for product cycle group + # This group runs: anal_gsi_input and data process, run_fcst, python_skewt, run_post, run_clean + # + # SAVEINPUT_CYCLEDEF: + # cycle definition for saving INPUT files + # This group runs: saveinput + # + # RECENTER_CYCLEDEF: + # cycle definition for recenter cycle group + # This group runs: recenter + # + # PRODLONG_CYCLEDEF: + # same as PROD_CYCLEDEF, but for long forecast + # + #----------------------------------------------------------------------- + # + AT_START_CYCLEDEF: "00 01 01 01 2100 *" + INITIAL_CYCLEDEF: "00 01 01 01 2100 *" + BOUNDARY_CYCLEDEF: "00 01 01 01 2100 *" + BOUNDARY_LONG_CYCLEDEF: "00 01 01 01 2100 *" + SPINUP_CYCLEDEF: "00 01 01 01 2100 *" + PROD_CYCLEDEF: "00 01 01 01 2100 *" + PRODLONG_CYCLEDEF: "00 01 01 01 2100 *" + RECENTER_CYCLEDEF: "00 01 01 01 2100 *" + SAVEDA_CYCLEDEF: "00 01 01 01 2100 *" + #---------------------------- # MAKE GRID config parameters #----------------------------- @@ -1466,6 +1980,26 @@ task_get_extrn_lbcs: LBC_SPEC_INTVL_HRS: 6 EXTRN_MDL_LBCS_OFFSET_HRS: '{{ 3 if EXTRN_MDL_NAME_LBCS == "RAP" else 0 }}' FV3GFS_FILE_FMT_LBCS: "nemsio" + # + #----------------------------------------------------------------------- + # + # LBCS_SEARCH_HRS: + # When search boundary conditions tasks from previous cycles in prep_cyc step, + # For example: 0 means search start for the same cycle lbcs task. + # 1 means search start for 1-h previous cycle lbcs task. + # 2 means search start for 2-h previous cycle lbcs task. + # + # EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS: + # When search boundary conditions from previous cycles in prep_start step, + # the search will start at cycle before (this parameter) of current cycle. + # For example: 0 means search start at the same cycle lbcs directory. + # 1 means search start at 1-h previous cycle lbcs directory. + # 2 means search start at 2-h previous cycle lbcs directory. + # + #----------------------------------------------------------------------- + # + LBCS_SEARCH_HRS: 6 + EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS: 0 #----------------------------------------------------------------------- # # EXTRN_MDL_SYSBASEDIR_LBCS: @@ -1612,6 +2146,10 @@ task_run_fcst: # ii) RESTART_INTERVAL: 1 2 5 => restart files are produced only when # fh = 1, 2, and 5. # + # RESTART_INTERVAL_LONG: + # Set up frequenency or list of the forecast hours that FV3 should + # generate the restart files. + # # WRITE_DOPOST: # Flag that determines whether or not to use the inline post feature # [i.e. calling the Unified Post Processor (UPP) from within the @@ -1622,7 +2160,8 @@ task_run_fcst: # DT_ATMOS: "" FHROT: 0 - RESTART_INTERVAL: 0 + RESTART_INTERVAL: 0 # was 1,2 in rrfs_dev + RESTART_INTERVAL_LONG: 0 # was 1,2 in rrfs_dev WRITE_DOPOST: false # #----------------------------------------------------------------------- @@ -2132,6 +2671,15 @@ global: HALO_BLEND: 10 # #----------------------------------------------------------------------- + # + # PRINT_DIFF_PGR: + # Option to turn on/off pressure tendency diagnostic + # + #----------------------------------------------------------------------- + # + PRINT_DIFF_PGR: false + # + #----------------------------------------------------------------------- # #---------------------------- diff --git a/ush/fill_jinja_template.py b/ush/fill_jinja_template.py index f810136753..f31a3dfc3b 100755 --- a/ush/fill_jinja_template.py +++ b/ush/fill_jinja_template.py @@ -186,6 +186,12 @@ def parse_args(argv): action="store_true", help="Suppress all output", ) + parser.add_argument( + "-i", + "--ignore_undefined", + action="store_true", + help="Ignore undefined jinja template variables", + ) parser.add_argument( "-u", "--user_config", @@ -283,7 +289,7 @@ def fill_jinja_template(argv, config_dict=None): tvars = {} for var in template_vars: - if cfg.get(var, "NULL") == "NULL": + if (not cla.ignore_undefined) and (cfg.get(var, "NULL") == "NULL"): raise KeyError(f"{var} does not exist in user-supplied settings!") if not cla.quiet: diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 3e3d689dc7..80a1069ff7 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -176,7 +176,6 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # Copy or symlink fix files # if SYMLINK_FIX_FILES: - log_info( f""" Symlinking fixed files from system directory (FIXgsm) to a subdirectory (FIXam): @@ -360,6 +359,8 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de "target_lon": LON_CTR, "target_lat": LAT_CTR, "nrows_blend": HALO_BLEND, + "regional_bcs_from_gsi": False, + "write_restart_with_bcs": False, # # Question: # For a ESGgrid type grid, what should stretch_fac be set to? This depends @@ -412,15 +413,9 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de gfs_physics_nml_dict.update({ "kice": kice or None, "lsoil": lsoil or None, - "do_shum": DO_SHUM, - "do_sppt": DO_SPPT, - "do_skeb": DO_SKEB, - "do_spp": DO_SPP, - "n_var_spp": N_VAR_SPP, - "n_var_lndp": N_VAR_LNDP, - "lndp_type": LNDP_TYPE, - "fhcyc": FHCYC_LSM_SPP_OR_NOT, + "print_diff_pgr": PRINT_DIFF_PGR, }) + if CPL_AQM: gfs_physics_nml_dict.update({ "cplaqm": True, @@ -494,12 +489,211 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de # if PREDEF_GRID_NAME == "RRFS_NA_3km": settings["fms2_io_nml"] = {"netcdf_default_format": "netcdf4"} + + settings_str = cfg_to_yaml_str(settings) + + log_info( + f""" + The variable 'settings' specifying values of the weather model's + namelist variables has been set as follows:\n""", + verbose=verbose, + ) + log_info("\nsettings =\n\n" + settings_str, verbose=verbose) + # + # ----------------------------------------------------------------------- + # + # Call the set_namelist.py script to create a new FV3 namelist file (full + # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as + # the base (i.e. starting) namelist file, with physics-suite-dependent + # modifications to the base file specified in the yaml configuration file + # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), + # and with additional physics-suite-independent modifications specified + # in the variable "settings" set above. + # + # ----------------------------------------------------------------------- + # + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_BASE_SUITE_FP, + "-c", + FV3_NML_YAML_CONFIG_FP, + CCPP_PHYS_SUITE, + "-u", + settings_str, + "-o", + FV3_NML_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' + Full path to yaml configuration file for various physics suites: + FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' + Physics suite to extract from yaml configuration file: + CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) + # + # If not running the TN_MAKE_GRID task (which implies the workflow will + # use pregenerated grid files), set the namelist variables specifying + # the paths to surface climatology files. These files are located in + # (or have symlinks that point to them) in the FIXlam directory. + # + # Note that if running the TN_MAKE_GRID task, this action usually cannot + # be performed here but must be performed in that task because the names + # of the surface climatology files depend on the CRES parameter (which is + # the C-resolution of the grid), and this parameter is in most workflow + # configurations is not known until the grid is created. + # + if not expt_config['rocoto']['tasks'].get('task_make_grid'): + + set_FV3nml_sfc_climo_filenames() + + # + # ----------------------------------------------------------------------- + # + # Generate namelist for surface cycle + # Deleted in newer RRFS_dev version so here for testing only + # + # ----------------------------------------------------------------------- + # + if DO_SURFACE_CYCLE: + if SDF_USES_RUC_LSM: + lsoil=9 + settings = {} + settings["gfs_physics_nml"] = { + "lsoil": lsoil or None + #"fh_dfi_radar": FH_DFI_RADAR # commented out untile develop gets radar tten code + } + + settings_str = cfg_to_yaml_str(settings) + # + # populate the namelist file + # + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_CYCSFC_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for DA: + FV3_NML_RESTART_FP = '{FV3_NML_CYCSFC_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) + + # + # ----------------------------------------------------------------------- + # + # Generate namelist for DA cycle + # + # ----------------------------------------------------------------------- + # + if DO_DACYCLE or DO_ENKFUPDATE: + + if SDF_USES_RUC_LSM: + lsoil = 9 + + lupdatebc = False + if DO_UPDATE_BC: + lupdatebc = False # not ready for setting this to true yet + + settings = {} + settings["fv_core_nml"] = { + "external_ic": False, + "make_nh": False, + "na_init": 0, + "nggps_ic": False, + "mountain": True, + "regional_bcs_from_gsi": lupdatebc, + "warm_start": True, + } + settings["gfs_physics_nml"] = { + "lsoil": lsoil or None + #"fh_dfi_radar": FH_DFI_RADAR # commented out untile develop gets radar tten code + } + + settings_str = cfg_to_yaml_str(settings) + # + # populate the namelist file + # + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_RESTART_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for DA: + FV3_NML_RESTART_FP = '{FV3_NML_RESTART_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) + # + # ----------------------------------------------------------------------- # # Add the relevant tendency-based stochastic physics namelist variables to # "settings" when running with SPPT, SHUM, or SKEB turned on. If running # with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only # include an empty "nam_stochy" stanza. # + # ----------------------------------------------------------------------- + # + settings = {} + settings["gfs_physics_nml"] = { + "do_shum": DO_SHUM, + "do_sppt": DO_SPPT, + "do_skeb": DO_SKEB, + "do_spp": DO_SPP, + "n_var_spp": N_VAR_SPP, + "n_var_lndp": N_VAR_LNDP, + "lndp_type": LNDP_TYPE, + "fhcyc": FHCYC_LSM_SPP_OR_NOT, + } nam_stochy_dict = {} if DO_SPPT: nam_stochy_dict.update( @@ -583,76 +777,71 @@ def generate_FV3LAM_wflow(ushdir, logfile: str = "log.generate_FV3LAM_wflow", de settings["nam_sfcperts"] = nam_sfcperts_dict settings_str = cfg_to_yaml_str(settings) - - log_info( - f""" - The variable 'settings' specifying values of the weather model's - namelist variables has been set as follows:\n""", - verbose=verbose, - ) - log_info("\nsettings =\n\n" + settings_str, verbose=verbose) # - # ----------------------------------------------------------------------- + #----------------------------------------------------------------------- # - # Call the set_namelist.py script to create a new FV3 namelist file (full - # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as - # the base (i.e. starting) namelist file, with physics-suite-dependent - # modifications to the base file specified in the yaml configuration file - # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), - # and with additional physics-suite-independent modifications specified - # in the variable "settings" set above. + # Generate namelist files with stochastic physics if needed # - # ----------------------------------------------------------------------- + #----------------------------------------------------------------------- # - try: - set_namelist( - [ - "-q", - "-n", - FV3_NML_BASE_SUITE_FP, - "-c", - FV3_NML_YAML_CONFIG_FP, - CCPP_PHYS_SUITE, - "-u", - settings_str, - "-o", - FV3_NML_FP, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' - Full path to yaml configuration file for various physics suites: - FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' - Physics suite to extract from yaml configuration file: - CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Namelist settings specified on command line:\n - settings =\n\n""" + if DO_ENSEMBLE and ( DO_SPP or DO_SPPT or DO_SHUM or DO_SKEB or DO_LSM_SPP): + + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_FP, + "-u", + settings_str, + "-o", + FV3_NML_STOCH_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for stochastics: + FV3_NML_STOCH_FP = '{FV3_NML_STOCH_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str ) - + settings_str - ) - # - # If not running the TN_MAKE_GRID task (which implies the workflow will - # use pregenerated grid files), set the namelist variables specifying - # the paths to surface climatology files. These files are located in - # (or have symlinks that point to them) in the FIXlam directory. - # - # Note that if running the TN_MAKE_GRID task, this action usually cannot - # be performed here but must be performed in that task because the names - # of the surface climatology files depend on the CRES parameter (which is - # the C-resolution of the grid), and this parameter is in most workflow - # configurations is not known until the grid is created. - # - if not expt_config['rocoto']['tasks'].get('task_make_grid'): - set_FV3nml_sfc_climo_filenames() + if DO_DACYCLE or DO_ENKFUPDATE: + try: + set_namelist( + [ + "-q", + "-n", + FV3_NML_RESTART_FP, + "-u", + settings_str, + "-o", + FV3_NML_RESTART_STOCH_FP, + ] + ) + except: + logging.exception( + dedent( + f""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file for stochastics: + FV3_NML_RESTART_STOCH_FP = '{FV3_NML_RESTART_STOCH_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" + ) + + settings_str + ) # # ----------------------------------------------------------------------- diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index c738a772fc..8b0fd790a8 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -12,6 +12,10 @@ platform: QUEUE_DEFAULT: regular QUEUE_FCST: regular QUEUE_HPSS: regular + QUEUE_GRAPHICS: regular + QUEUE_ANALYSIS: regular + QUEUE_PRDGEN: regular + QUEUE_POST: regular RUN_CMD_FCST: mpirun -np ${PE_MEMBER01} RUN_CMD_POST: mpirun -np $nprocs RUN_CMD_PRDGEN: mpirun -np $nprocs diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 9653d74c45..79016f1f21 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -12,6 +12,10 @@ platform: QUEUE_DEFAULT: normal QUEUE_FCST: normal QUEUE_HPSS: normal + QUEUE_GRAPHICS: normal + QUEUE_ANALYSIS: normal + QUEUE_PRDGEN: normal + QUEUE_POST: normal REMOVE_MEMORY: True RUN_CMD_FCST: srun --export=ALL --mpi=pmi2 -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL --mpi=pmi2 -n $nprocs diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index b3caacf97b..33b3f8c4c6 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -15,6 +15,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: hera + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: hera + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: hera + QUEUE_PRDGEN: batch + PARTITION_POST: hera + QUEUE_POST: batch RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL @@ -37,8 +45,36 @@ platform: FIXsfc: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/NaturalEarth EXTRN_MDL_DATA_STORES: hpss aws nomads - - +data: + obs: + RAP_obs: /scratch2/BMC/public/data/grids/rap/obs + rrfs: + retro: + OBSPATH: "/scratch2/BMC/zrtrr/rli/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/scratch2/BMC/zrtrr/rli/data/reflectivity" + LIGHTNING_ROOT: "/scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/rrfs_retro_data/lightning" + ENKF_FCST: "/scratch2/BMC/zrtrr/rli/data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/scratch2/BMC/zrtrr/rli/data/highres_sst" + GVF_ROOT: "/scratch2/BMC/zrtrr/rli/data/gvf/grib2" + IMSSNOW_ROOT: "/scratch2/BMC/zrtrr/rli/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/scratch2/BMC/zrtrr/rli/data/amdar_reject_lists" + SFCOBS_USELIST: "/scratch2/BMC/zrtrr/rli/data/mesonet_uselists" + real: + OBSPATH: "/scratch2/BMC/public/data/grids/rap/obs" + OBSPATH_NSSLMOSIAC: "/scratch2/BMC/public/data/radar/nssl/mrms/conus" + LIGHTNING_ROOT: "/scratch2/BMC/public/data/lightning" + ENKF_FCST: "/scratch1/NCEPDEV/rstprod/com/gfs/prod" + FFG_DIR: "" + SST_ROOT: "/scratch2/BMC/zrtrr/rli/data/highres_sst" + GVF_ROOT: "/scratch2/BMC/zrtrr/rli/data/gvf/grib2" + IMSSNOW_ROOT: "/scratch2/BMC/zrtrr/rli/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/scratch2/BMC/zrtrr/rli/data/amdar_reject_lists" + SFCOBS_USELIST: "/scratch2/BMC/zrtrr/rli/data/mesonet_uselists" rocoto: tasks: metatask_run_ensemble: @@ -49,10 +85,6 @@ rocoto: nnodes: nodesize: ppn: - -data: - obs: - RAP_obs: /scratch2/BMC/public/data/grids/rap/obs cpl_aqm_parm: AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/epa/data AQM_BIO_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio @@ -64,5 +96,3 @@ cpl_aqm_parm: NEXUS_INPUT_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/emissions/nexus NEXUS_FIX_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/nexus/fix PT_SRC_BASEDIR: /scratch1/RDARCH/rda-arl-gpu/YouHua.Tang/nei2016v1-pt - - diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index b8f8a3b921..02e6b18849 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -15,6 +15,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: sjet,vjet,kjet,xjet + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: sjet,vjet,kjet,xjet + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: sjet,vjet,kjet,xjet + QUEUE_PRDGEN: batch + PARTITION_POST: sjet,vjet,kjet,xjet + QUEUE_POST: batch RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL @@ -47,7 +55,33 @@ data: GFS_obs: prepbufr: /public/data/grids/gfs/prepbufr tcvitals: /public/data/grids/gfs/bufr - + rrfs: + retro: + OBSPATH: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/reflectivity" + LIGHTNING_ROOT: "/mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/rrfs_retro_data/lightning" + ENKF_FCST: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/highres_sst" + GVF_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/gvf/grib2" + IMSSNOW_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/rap_hrrr_soil" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/amdar_reject_lists" + SFCOBS_USELIST: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/mesonet_uselists" + real: + OBSPATH: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/obs_rap" + OBSPATH_NSSLMOSIAC: "/public/data/radar/nssl/mrms/conus" + LIGHTNING_ROOT: "/public/data/lightning" + ENKF_FCST: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/enkf/atm" + FFG_DIR: "/public/data/grids/ncep/ffg/grib2" + SST_ROOT: "/lfs4/BMC/public/data/grids/ncep/sst/0p083deg/grib2" + GVF_ROOT: "/public/data/sat/ncep/viirs/gvf/grib2" + IMSSNOW_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "/mnt/lfs4/BMC/wrfruc/Ruifang.Li/data/rap_hrrr_soil" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/home/amb-verif/acars_RR/amdar_reject_lists" + SFCOBS_USELIST: "/lfs4/BMC/amb-verif/rap_ops_mesonet_uselists" rocoto: tasks: metatask_run_ensemble: diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml index 8cd1627da1..b498c11c45 100644 --- a/ush/machine/odin.yaml +++ b/ush/machine/odin.yaml @@ -9,6 +9,14 @@ platform: QUEUE_FCST: workq PARTITION_HPSS: workq QUEUE_HPSS: workq + PARTITION_GRAPHICS: workq + QUEUE_GRAPHICS: workq + PARTITION_ANALYSIS: workq + QUEUE_ANALYSIS: workq + PARTITION_PRDGEN: workq + QUEUE_PRDGEN: workq + PARTITION_POST: workq + QUEUE_POST: workq RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL -n 1 RUN_CMD_PRDGEN: srun --export=ALL -n 1 diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index a9c8baf031..5238e51d3b 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -15,6 +15,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: orion + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: orion + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: orion + QUEUE_PRDGEN: batch + PARTITION_POST: orion + QUEUE_POST: batch RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL diff --git a/ush/machine/singularity.yaml b/ush/machine/singularity.yaml index 72db7bdb45..a35a0c7fc1 100644 --- a/ush/machine/singularity.yaml +++ b/ush/machine/singularity.yaml @@ -9,6 +9,14 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch + PARTITION_GRAPHICS: null + QUEUE_GRAPHICS: batch + PARTITION_ANALYSIS: null + QUEUE_ANALYSIS: batch + PARTITION_PRDGEN: null + QUEUE_PRDGEN: batch + PARTITION_POST: null + QUEUE_POST: batch RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} --oversubscribe RUN_CMD_POST: mpirun -np $nprocs --oversubscribe RUN_CMD_PRDGEN: mpirun -np $nprocs --oversubscribe diff --git a/ush/machine/stampede.yaml b/ush/machine/stampede.yaml index 270cd3d593..4f11092ca5 100644 --- a/ush/machine/stampede.yaml +++ b/ush/machine/stampede.yaml @@ -9,6 +9,14 @@ platform: QUEUE_FCST: normal PARTITION_HPSS: normal QUEUE_HPSS: normal + PARTITION_GRAPHICS: normal + QUEUE_GRAPHICS: normal + PARTITION_ANALYSIS: normal + QUEUE_ANALYSIS: normal + PARTITION_PRDGEN: normal + QUEUE_PRDGEN: normal + PARTITION_POST: normal + QUEUE_POST: normal RUN_CMD_FCST: ibrun -np $nprocs RUN_CMD_POST: ibrun -np $nprocs RUN_CMD_PRDGEN: ibrun -np $nprocs diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index b0997381c0..ee57b7fb0f 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -12,6 +12,10 @@ platform: QUEUE_DEFAULT: dev QUEUE_FCST: dev QUEUE_HPSS: dev_transfer + QUEUE_GRAPHICS: dev + QUEUE_ANALYSIS: dev + QUEUE_PRDGEN: dev + QUEUE_POST: dev RUN_CMD_FCST: mpiexec -n ${PE_MEMBER01} -ppn ${PPN_RUN_FCST} --cpu-bind core -depth ${OMP_NUM_THREADS_RUN_FCST} RUN_CMD_POST: mpiexec -n ${nprocs} RUN_CMD_PRDGEN: mpiexec -n ${nprocs} --cpu-bind core cfp @@ -40,6 +44,33 @@ data: RAP: compath.py ${envir}/rap/${rap_ver}/rap.${PDYext} NAM: compath.py ${envir}/nam/${nam_ver}/nam.${PDYext} HRRR: compath.py ${envir}/hrrr/${hrrr_ver}/hrrr.${PDYext}/conus + rrfs: + retro: + OBSPATH: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/obs_rap" + OBSPATH_NSSLMOSIAC: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/reflectivity" + LIGHTNING_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/lightning" + ENKF_FCST: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/enkf/atm" + FFG_DIR: "" + SST_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/highres_sst" + GVF_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/gvf/grib2" + IMSSNOW_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/snow/ims96/grib2" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "" + AIRCRAFT_REJECT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/amdar_reject_list" + SFCOBS_USELIST: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/mesonet_uselists" + real: + OBSPATH: "/lfs/h1/ops/prod/com/obsproc/v1.1" + OBSPATH_NSSLMOSIAC: "/lfs/h1/ops/prod/dcom/ldmdata/obs/upperair/mrms/conus/MergedReflectivityQC" + LIGHTNING_ROOT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/lightning" + ENKF_FCST: "/lfs/h1/ops/prod/com/gfs/v16.3" + FFG_DIR: "" + SST_ROOT: "/lfs/h1/ops/prod/com/gfs/v16.3" + GVF_ROOT: "/lfs/h1/ops/prod/dcom/viirs" + IMSSNOW_ROOT: "/lfs/h1/ops/prod/com/obsproc/v1.1" + RAPHRR_SOIL_ROOT: "" + FIRE_RAVE_DIR: "/lfs/h2/emc/physics/noscrub/partha.bhattacharjee/RAVE_rawdata/RAVE_NA" + AIRCRAFT_REJECT: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/amdar_reject_list" + SFCOBS_USELIST: "/lfs/h2/emc/lam/noscrub/emc.lam/rrfs_retro_data/mesonet_uselists" cpl_aqm_parm: AQM_CONFIG_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/epa/data AQM_BIO_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/bio diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 08ed944f46..04aa628d43 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -58,7 +58,7 @@ def set_FV3nml_ens_stoch_seeds(cdate): # # ----------------------------------------------------------------------- # - fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}" + fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}_base" ensmem_num = ENSMEM_INDX @@ -174,7 +174,7 @@ def setUp(self): mkdir_vrfy("-p", EXPTDIR) cp_vrfy( os.path.join(PARMdir, "input.nml.FV3"), - os.path.join(EXPTDIR, "input.nml"), + os.path.join(EXPTDIR, "input.nml_base"), ) for i in range(2): mkdir_vrfy( @@ -192,7 +192,7 @@ def setUp(self): set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") - set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) + set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml_base")) set_env_var("DO_SHUM", True) set_env_var("DO_SKEB", True) set_env_var("DO_SPPT", True) diff --git a/ush/setup.py b/ush/setup.py index d4ad08d4ab..406c6ea9b3 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -577,10 +577,6 @@ def get_location(xcs, fmt, expt_cfg): ) get_extrn_lbcs["EXTRN_MDL_SYSBASEDIR_LBCS"] = extrn_mdl_sysbasedir_lbcs - # remove the data key -- it's not needed beyond this point - if "data" in expt_config: - expt_config.pop("data") - # Check for the user-specified directories for external model files if # USE_USER_STAGED_EXTRN_FILES is set to TRUE task_keys = zip( @@ -606,6 +602,26 @@ def get_location(xcs, fmt, expt_cfg): {data_key} = \"{basedir}\"''' ) + # + # ----------------------------------------------------------------------- + # + # Set RRFS data paths for retrospective/realtime runs + # + # ----------------------------------------------------------------------- + # + if expt_config["rrfs"]["DO_RRFS_DEV"]: + mode = "real" if expt_config["workflow"]["DO_REAL_TIME"] else "retro" + rrfs = expt_config.get("data", {}).get("rrfs") + if rrfs is not None: + for k,v in rrfs[mode].items(): + expt_config["platform"][k] = v + + # ----------------------------------------------------------------------- + # remove the data key -- it's not needed beyond this point + # ----------------------------------------------------------------------- + if "data" in expt_config: + expt_config.pop("data") + # # ----------------------------------------------------------------------- #