Parse jediyaml only once (#2387)

`JEDIYAML` was being parsed 3 times; once in `get_obs_dict`, second in
`get_bias_dict` and a third time in `initialize` for the specific
component analysis task.

This PR:
- eliminates the duplications and constructs the `jedi_config`
dictionary just once. The dictionary is written out before calling the
executable.
- updates hash to gdasapp
- updates configs for snow, aerosol, atmvar and atmens JEDI-DA to
include `JEDI_FIX_YAML` and `CRTM_FIX_YAML` . This allows greater
flexibility and control over the contents of these fix data sets to be
copied into the run directory.
- Combines snowDA and aerosolDA into a single test

Co-authored-by: Cory Martin <cory.r.martin@noaa.gov>
Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>

ci/Jenkinsfile

@@ -119,7 +119,7 @@ pipeline {
                     axis {
                         name 'Case'
                         // TODO add dynamic list of cases from env vars (needs addtional plugins)
-                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA'
+                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
                     }
                 }
                 stages {
@@ -166,7 +166,7 @@ pipeline {
                                                 for (line in lines) {
                                                     echo "archiving: ${line}"
                                                     archiveArtifacts artifacts: "${line}", fingerprint: true
-                                                }   
+                                                }
                                             }
                                         }
                                         error("Failed to run experiments ${Case} on ${Machine}")

ci/cases/pr/C96_atmsnowDA.yaml → ci/cases/pr/C96_atmaerosnowDA.yaml

@@ -4,7 +4,7 @@ experiment:
 
 arguments:
   pslot: {{ 'pslot' | getenv }}
-  app: ATM
+  app: ATMA
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
@@ -14,7 +14,7 @@ arguments:
   nens: 0
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/cases/yamls/atmsnowDA_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
   - orion

parm/config/gfs/config.aeroanl

@@ -6,24 +6,26 @@
 echo "BEGIN: config.aeroanl"
 
 export CASE_ANL=${CASE}
-export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/config/
-export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/lists/gdas_aero_prototype.yaml
+export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2"
 export STATICB_TYPE='identity'
-export BERROR_YAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/berror/staticb_${STATICB_TYPE}.yaml
-export BERROR_DATA_DIR=${FIXgfs}/gdas/bump/aero/${CASE_ANL}/
+export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2"
+export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/"
 export BERROR_DATE="20160630.000000"
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2"
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/fv3jedi_var.x
+export JEDIEXE="${EXECgfs}/fv3jedi_var.x"
 
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"
-  export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_fgat_gfs_aero.yaml
+  export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2"
 else
   export aero_bkg_times="6"
-  export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_gfs_aero.yaml
+  export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_gfs_aero.yaml.j2"
 fi
 
 echo "END: config.aeroanl"

parm/config/gfs/config.aeroanlfinal

@@ -6,5 +6,5 @@
 echo "BEGIN: config.aeroanlfinal"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlfinal
+source "${EXPDIR}/config.resources" aeroanlfinal
 echo "END: config.aeroanlfinal"

parm/config/gfs/config.aeroanlinit

@@ -6,5 +6,5 @@
 echo "BEGIN: config.aeroanlinit"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlinit
+source "${EXPDIR}/config.resources" aeroanlinit
 echo "END: config.aeroanlinit"

parm/config/gfs/config.aeroanlrun

@@ -6,6 +6,6 @@
 echo "BEGIN: config.aeroanlrun"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlrun
+source "${EXPDIR}/config.resources" aeroanlrun
 
 echo "END: config.aeroanlrun"

parm/config/gfs/config.atmanl

@@ -19,6 +19,9 @@ else
     export BERROR_YAML="${PARMgfs}/gdas/atm/berror/staticb_${STATICB_TYPE}.yaml.j2"
 fi
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
 export layout_x_atmanl=@LAYOUT_X_ATMANL@
 export layout_y_atmanl=@LAYOUT_Y_ATMANL@
 

parm/config/gfs/config.atmensanl

@@ -9,6 +9,9 @@ export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/lgetkf_prototype.yaml.j2"
 export JEDIYAML="${PARMgfs}/gdas/atm/lgetkf/lgetkf.yaml.j2"
 export INTERP_METHOD='barycentric'
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
 export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
 

parm/config/gfs/config.snowanl

@@ -22,6 +22,8 @@ export BESTDDEV="30."  # Background Error Std. Dev. for LETKFOI
 export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
 export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/apply_incr_nml.j2"
 
+export JEDI_FIX_YAML="${PARMgfs}/gdas/snow_jedi_fix.yaml.j2"
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 

parm/gdas/aero_crtm_coeff.yaml.j2

@@ -0,0 +1,13 @@
+mkdir:
+- '{{ DATA }}/crtm/'
+copy:
+- ['{{ CRTM_FIX }}/AerosolCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/CloudCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/v.viirs-m_npp.SpcCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/v.viirs-m_npp.TauCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/v.viirs-m_j1.SpcCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/v.viirs-m_j1.TauCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/NPOESS.VISice.EmisCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/NPOESS.VISland.EmisCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/NPOESS.VISsnow.EmisCoeff.bin', '{{ DATA }}/crtm/']
+- ['{{ CRTM_FIX }}/NPOESS.VISwater.EmisCoeff.bin', '{{ DATA }}/crtm/']

parm/gdas/atm_jedi_fix.yaml.j2

@@ -0,0 +1,7 @@
+mkdir:
+- '{{ DATA }}/fv3jedi'
+copy:
+- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4']
+- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml']
+- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table']
+- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml']

parm/gdas/snow_jedi_fix.yaml.j2

@@ -1,7 +1,7 @@
 mkdir:
 - '{{ DATA }}/fv3jedi'
 copy:
-- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4']
-- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml']
-- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table']
-- ['{{ HOMEgfs }}/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml']
+- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4']
+- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml']
+- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table']
+- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml']

ush/forecast_postdet.sh

@@ -931,7 +931,7 @@ GOCART_rc() {
 GOCART_postdet() {
   echo "SUB ${FUNCNAME[0]}: Linking output data for GOCART"
 
-  for fhr in ${FV3_OUTPUT_FH}; do
+  for fhr in ${GOCART_OUTPUT_FH}; do
     local vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H)
 
     # Temporarily delete existing files due to noclobber in GOCART
@@ -952,7 +952,7 @@ GOCART_out() {
   # TO DO: this should be linked but there were issues where gocart was crashing if it was linked
   local fhr
   local vdate
-  for fhr in ${FV3_OUTPUT_FH}; do
+  for fhr in ${GOCART_OUTPUT_FH}; do
     if (( fhr == 0 )); then continue; fi
     vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H)
     ${NCP} "${DATA}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4" \

ush/forecast_predet.sh

@@ -10,52 +10,51 @@
 
 to_seconds() {
   # Function to convert HHMMSS to seconds since 00Z
-  local hhmmss=${1:?}
-  local hh=${hhmmss:0:2}
-  local mm=${hhmmss:2:2}
-  local ss=${hhmmss:4:2}
-  local seconds=$((10#${hh}*3600+10#${mm}*60+10#${ss}))
-  local padded_seconds=$(printf "%05d" "${seconds}")
+  local hhmmss hh mm ss seconds padded_seconds
+  hhmmss=${1:?}
+  hh=${hhmmss:0:2}
+  mm=${hhmmss:2:2}
+  ss=${hhmmss:4:2}
+  seconds=$((10#${hh}*3600+10#${mm}*60+10#${ss}))
+  padded_seconds=$(printf "%05d" "${seconds}")
   echo "${padded_seconds}"
 }
 
 middle_date(){
   # Function to calculate mid-point date in YYYYMMDDHH between two dates also in YYYYMMDDHH
-  local date1=${1:?}
-  local date2=${2:?}
-  local date1s=$(date --utc -d "${date1:0:8} ${date1:8:2}:00:00" +%s)
-  local date2s=$(date --utc -d "${date2:0:8} ${date2:8:2}:00:00" +%s)
-  local dtsecsby2=$(( $((date2s - date1s)) / 2 ))
-  local mid_date=$(date --utc -d "${date1:0:8} ${date1:8:2} + ${dtsecsby2} seconds" +%Y%m%d%H%M%S)
+  local date1 date2 date1s date2s dtsecsby2 mid_date
+  date1=${1:?}
+  date2=${2:?}
+  date1s=$(date --utc -d "${date1:0:8} ${date1:8:2}:00:00" +%s)
+  date2s=$(date --utc -d "${date2:0:8} ${date2:8:2}:00:00" +%s)
+  dtsecsby2=$(( $((date2s - date1s)) / 2 ))
+  mid_date=$(date --utc -d "${date1:0:8} ${date1:8:2} + ${dtsecsby2} seconds" +%Y%m%d%H%M%S)
   echo "${mid_date:0:10}"
 }
 
 nhour(){
   # Function to calculate hours between two dates (This replicates prod-util NHOUR)
-  local date1=${1:?}
-  local date2=${2:?}
+  local date1 date2 seconds1 seconds2 hours
+  date1=${1:?}
+  date2=${2:?}
   # Convert dates to UNIX timestamps
   seconds1=$(date --utc -d "${date1:0:8} ${date1:8:2}:00:00" +%s)
   seconds2=$(date --utc -d "${date2:0:8} ${date2:8:2}:00:00" +%s)
   hours=$(( $((seconds1 - seconds2)) / 3600 ))  # Calculate the difference in seconds and convert to hours
   echo "${hours}"
 }
 
+# shellcheck disable=SC2034
 common_predet(){
   echo "SUB ${FUNCNAME[0]}: Defining variables for shared through model components"
-  # Ignore "not used" warning
-  # shellcheck disable=SC2034
   pwd=$(pwd)
   CDUMP=${CDUMP:-gdas}
-  CASE=${CASE:-C96}
   CDATE=${CDATE:-"${PDY}${cyc}"}
   ENSMEM=${ENSMEM:-000}
 
   # Define significant cycles
   current_cycle="${PDY}${cyc}"
   previous_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${assim_freq} hours" +%Y%m%d%H)
-  # ignore errors that variable isn't used
-  # shellcheck disable=SC2034
   next_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${assim_freq} hours" +%Y%m%d%H)
   forecast_end_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHMAX} hours" +%Y%m%d%H)
 
@@ -89,6 +88,7 @@ common_predet(){
   cd "${DATA}" || ( echo "FATAL ERROR: Unable to 'cd ${DATA}', ABORT!"; exit 8 )
 }
 
+# shellcheck disable=SC2034
 FV3_predet(){
   echo "SUB ${FUNCNAME[0]}: Defining variables for FV3"
 
@@ -105,8 +105,6 @@ FV3_predet(){
   fi
 
   # Convert output settings into an explicit list for FV3
-  # Ignore "not used" warning
-  # shellcheck disable=SC2034
   FV3_OUTPUT_FH=""
   local fhr=${FHMIN}
   if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then
@@ -116,8 +114,6 @@ FV3_predet(){
   FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")"
 
   # Other options
-  # ignore errors that variable isn't used
-  # shellcheck disable=SC2034
   MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER
   PREFIX_ATMINC=${PREFIX_ATMINC:-""} # allow ensemble to use recentered increment
 
@@ -169,7 +165,6 @@ FV3_predet(){
   nstf_name=${nstf_name:-"${NST_MODEL},${NST_SPINUP},${NST_RESV},${ZSEA1},${ZSEA2}"}
   nst_anl=${nst_anl:-".false."}
 
-
   # blocking factor used for threading and general physics performance
   #nyblocks=$(expr \( $npy - 1 \) \/ $layout_y )
   #nxblocks=$(expr \( $npx - 1 \) \/ $layout_x \/ 32)
@@ -215,6 +210,7 @@ WW3_predet(){
   ${NLN} "${COM_WAVE_RESTART}" "restart_wave"
 }
 
+# shellcheck disable=SC2034
 CICE_predet(){
   echo "SUB ${FUNCNAME[0]}: CICE before run type determination"
 
@@ -227,12 +223,11 @@ CICE_predet(){
 
   # CICE does not have a concept of high frequency output like FV3
   # Convert output settings into an explicit list for CICE
-  # Ignore "not used" warning
-  # shellcheck disable=SC2034
   CICE_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCNICE}" "${FHMAX}")
 
 }
 
+# shellcheck disable=SC2034
 MOM6_predet(){
   echo "SUB ${FUNCNAME[0]}: MOM6 before run type determination"
 
@@ -245,8 +240,6 @@ MOM6_predet(){
 
   # MOM6 does not have a concept of high frequency output like FV3
   # Convert output settings into an explicit list for MOM6
-  # Ignore "not used" warning
-  # shellcheck disable=SC2034
   MOM6_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCNICE}" "${FHMAX}")
 
 }
@@ -260,9 +253,12 @@ CMEPS_predet(){
 
 }
 
+# shellcheck disable=SC2034
 GOCART_predet(){
   echo "SUB ${FUNCNAME[0]}: GOCART before run type determination"
 
   if [[ ! -d "${COM_CHEM_HISTORY}" ]]; then mkdir -p "${COM_CHEM_HISTORY}"; fi
 
+  GOCART_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "6" "${FHMAX}")
+  # TODO: AERO_HISTORY.rc has hardwired output frequency to 6 hours
 }