From a6f4633dd7f184642752cff0fd382302aa2dcd7b Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 31 Jan 2023 12:35:04 -0600 Subject: [PATCH] Make initial config changes for COM refactor THIS COMMIT BREAKS THE ENTIRE WORKFLOW. Functionality will be returned in subsequent commits for this project. Removes the old `COMIN` variables from `config.base` and adds a new `config.com`. This new configuration is currently sourced by `config.base` and contains a series of path templates. These templates are substituted at runtime to produce the appropriate paths. Templates must use SINGLE quotation marks to prevent substituting variables when sourced. The substitutions can be made one of two ways. First, the `config.com` script also defines a function `generate_com`, which takes in a list of variable names and automatically parses the associated template into the each variable. The function also takes a list of options that mirrors those of the `declare` (aka `typeset`) built-in, which is what the function uses under the hood. For this method, the template must be defined to the variable `${VARIABLE_NAME}_TMPL`. Assign COM_OBS from COM_OBS_TMPL template and mark it for export and read-only ``` generate_com -rx COM_OBS ``` The second method is to do the substitution directly using `envsubst`. If you need to override variables in the template without changing the actual variables (for instance, to generate the path for a previous cycle), or assign to a different variable name than the template, you'll need to use this method. To use this method, use a subshell to echo the template and pipe it through `envsubst` to substitute in variables from the environment: This is the equivilent of the generate_com example above ``` COM_OBS=$(echo "${COM_OBS_TMPL}" | envsubst) declare -rx COM_OBS ``` As mentioned, you can also locally override variables if needed: ``` COM_ATMOS_RESTART_PREV=$({ RUN="${rCDUMP}" PDY="${PDY_PREV}" cyc="${cyc_PREV}" echo "${COM_ATMOS_RESTART_TMPL}" | envsubst }) declare -rx COM_ATMOS_RESTART_PREV ``` ``` Refs: #761 --- parm/config/config.base.emc.dyn | 15 +---- parm/config/config.com | 104 ++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 parm/config/config.com diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 27e8594e47..43f0cf1023 100644 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -121,19 +121,8 @@ export RUN=${RUN:-${CDUMP:-"gfs"}} # RUN is defined in the job-card (ecf); CDUM # TODO: determine where is RUN actually used in the workflow other than here # TODO: is it possible to replace all instances of ${CDUMP} to ${RUN} to be # consistent w/ EE2? -export COMIN_OBS=${COMIN_OBS:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/obs} -export COMIN_GES_OBS=${COMIN_GES_OBS:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/obs} - -export COMINatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos -export COMOUTatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos -export COMINwave=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/wave -export COMOUTwave=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/wave -export COMINocean=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean -export COMOUTocean=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean -export COMINice=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ice -export COMOUTice=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ice -export COMINaero=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/chem -export COMOUTaero=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/chem + +source "${EXPDIR}/config.com" export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'} export LOGSCRIPT=${LOGSCRIPT:-""} diff --git a/parm/config/config.com b/parm/config/config.com new file mode 100644 index 0000000000..e9e2fb8de8 --- /dev/null +++ b/parm/config/config.com @@ -0,0 +1,104 @@ +# shellcheck shell=bash +echo "BEGIN: config.com" + +# These are just templates. Substitute variables in at runtime like this: +# ``` +# COM_OBS=$(echo "${COM_OBS_TMPL}" | envsubst) +# ``` +# +# Or use the `generate_com` function included below, which uses the same +# options as `declare`/`typeset` +# +# # Assign COM_OBS from COM_OBS_TMPL template and mark it for export and read-only +# generate_com -rx COM_OBS +# +# If you need to override variables in the template without changing the +# actual variable in your program, you can make assignments in the subshell: +# +# COM_ATMOS_RESTART_PREV=$({ +# # Override env variables for this subshell to get correct template substitution +# RUN=${rCDUMP} +# PDY="${PDY_PREV}" +# cyc="${cyc_PREV}" +# echo "${COM_ATMOS_RESTART_TMPL}" | envsubst +# }) +# + +# shellcheck disable=SC2016 +export COM_OBS_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/obs' +export COM_OBSDMP_TMPL='${DMPDIR}/${RUN}.${PDY}/${cyc}/atmos' + +export COM_ATMOS_INPUT_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/model_data/input' +export COM_ATMOS_RESTART_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/model_data/restart' +export COM_ATMOS_ANALYSIS_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/model_data/analysis' +export COM_ATMOS_HISTORY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/model_data/history' +export COM_ATMOS_MASTER_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/model_data/master' +export COM_ATMOS_GRIB_0p25_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/0p25' +export COM_ATMOS_GRIB_0p50_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/0p50' +export COM_ATMOS_GRIB_1p00_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/1p00' +export COM_ATMOS_BUFR_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/bufr' +export COM_ATMOS_GEMPAK_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/gempak' +export COM_ATMOS_GENESIS_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/cyclone/genesis_vital' +export COM_ATMOS_TRACK_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/cyclone/tracks' +export COM_ATMOS_GOES_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/goes_sim' +export COM_ATMOS_IMAGERY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/imagery' +export COM_ATMOS_MINMON_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/minmon' +export COM_ATMOS_WAFS_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/wafs' +export COM_ATMOS_WMO_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/atmos/products/wmo' + +export COM_WAVE_RESTART_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/wave/model_data/restart' +export COM_WAVE_PREP_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/wave/model_data/prep' +export COM_WAVE_HISTORY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/wave/model_data/history' +export COM_WAVE_GRID_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/wave/products/gridded' +export COM_WAVE_STATION_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/wave/products/station' + +export COM_OCEAN_HISTORY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ocean/model_data/history' +export COM_OCEAN_INPUT_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ocean/model_data/input' +export COM_OCEAN_ANALYSIS_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ocean/model_data/analysis' +export COM_OCEAN_DAILY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ocean/products/daily' +export COM_OCEAN_GRIB_0p50_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ocean/products/0p50' +export COM_OCEAN_GRIB_0p25_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ocean/products/0p25' + +export COM_ICE_HISTORY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/ice/model_data/history' + +export COM_CHEM_HISTORY_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/chem/model_data/history' + +export COM_MED_RESTART_TMPL='${ROTDIR}/${RUN}.${PDY}/${cyc}/${MEMDIR}/med/model_data/restart' + +# shellcheck disable= + +function generate_com() { + # + # Generate a list COM variables from a template by substituting in env variables. + # + # Each argument must have a corresponding template with the name ${ARG}_TMPL. + # + # Accepts as options all the same options the bash built-in `declare` allows except + # -g, which is assumed, and -p. These options are passed to `declare`. + # + # Syntax: + # generate_com [-aAfFilrtux] $var1 [$var2 [$var3 ...]] + # + # var1, var2, etc: Variable names whose values will be generated from a template + # and declared + # options: Same function as the bash `declare` built-in + # + local opts="-g" + local OPTIND=1 + while getopts "aAfFilrtux" option; do + opts="${opts}${option}" + done + shift + + for com_var in "$@"; do + local template="${com_var}_TMPL" + local value + value=$(echo "${!template}" | envsubst) + # shellcheck disable=SC2086 + declare ${opts} "${com_var}"="${value}" + done +} +# shellcheck disable= +export -f generate_com + +echo "END: config.com"