Merge remote-tracking branch 'origin/develop' into feature/issue_2475

* origin/develop: Add observation preparation job for aerosols DA to workflow (NOAA-EMC#2624) Remove ocean daily files (NOAA-EMC#2689) Update Jenkinsfile Add Hercules-EMC to the Jenkins configurable parameter list (NOAA-EMC#2685) Update gdas.cd and gsi_utils hashes (NOAA-EMC#2641) Add ability to use GEFS replay ICs (NOAA-EMC#2559) Replace `sleep` with `wait_for_file` (NOAA-EMC#2586) Add COM template for JEDI obs (NOAA-EMC#2678) Link both global-nest fix files and non-nest ones at the same time (NOAA-EMC#2632) Update ufs-weather-model (NOAA-EMC#2663) Add ability to process ocean/ice products specific to GEFS (NOAA-EMC#2561) Update cleanup job to use COMIN/COMOUT (NOAA-EMC#2649) Add overwrite to creat experiment in BASH CI (NOAA-EMC#2676) Add handling to select CRTM cloud optical table based on cloud scheme and update calcanal_gfs.py (NOAA-EMC#2645) Refs NOAA-EMC#2475
KateFriedman-NOAA · Jun 14, 2024 · 87309fc · 87309fc
2 parents 937a245 + 6c93b45
commit 87309fc
Show file tree

Hide file tree

Showing 80 changed files with 1,112 additions and 483 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,8 +35,10 @@ fix/gsi
 fix/lut
 fix/mom6
 fix/orog
+fix/orog_nest
 fix/sfc_climo
 fix/ugwd
+fix/ugwd_nest
 fix/verif
 fix/wave
 

diff --git a/.gitmodules b/.gitmodules
@@ -26,10 +26,6 @@
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
   url = https://github.com/NOAA-EMC/GSI-Monitor.git
-[submodule "sorc/upp.fd"]
-  path = sorc/upp.fd
-  url = https://github.com/NOAA-EMC/UPP.git
-  ignore = dirty
 [submodule "sorc/jcb"]
   path = sorc/jcb
   url = https://github.com/noaa-emc/jcb

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -76,7 +76,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
                         sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -228,7 +228,7 @@ for pr in ${pr_list}; do
         } >> "${output_ci}"
         continue
       fi
-      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml"  > "${LOGFILE_PATH}" 2>&1
+      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml" --overwrite  > "${LOGFILE_PATH}" 2>&1
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then

diff --git a/env/HERA.env b/env/HERA.env
@@ -102,6 +102,13 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))
@@ -316,7 +323,6 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
 
-
 elif [[ "${step}" = "fit2obs" ]]; then
 
     nth_max=$((npe_node_max / npe_node_fit2obs))

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -99,6 +99,12 @@ case ${step} in
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
  ;;
+ "prepobsaero")
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+;;
  "snowanl")
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/JET.env b/env/JET.env
@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/ORION.env b/env/ORION.env
@@ -90,6 +90,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/S4.env b/env/S4.env
@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -76,6 +76,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --ppn ${npe_node_prepobsaero}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/jobs/JGLOBAL_PREP_OBS_AERO b/jobs/JGLOBAL_PREP_OBS_AERO
@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prepobsaero" -c "base prepobsaero"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export COMIN_OBS="${DATA}"
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_OBS:COM_OBS_TMPL
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_obs_aero.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0
diff --git a/jobs/rocoto/prepobsaero.sh b/jobs/rocoto/prepobsaero.sh
@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="prepobsaero"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO"
+status=$?
+exit "${status}"
diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2
@@ -47,6 +47,11 @@ Deterministic: &deterministic
             - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"]
             {% endif %}
 
+            {% if DO_PREP_OBS_AERO %}
+            - ["{{ COM_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"]
+            - ["{{ COM_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"]
+            {% endif %}
+
     {% endif %}
 
 {% if RUN == "gfs" %}

diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2
@@ -16,7 +16,7 @@ enkf:
         {% endfor %}
 
         {% if DO_JEDIATMENS %}
-        {% set steps = ["atmensanlinit", "atmensanlrun", "atmensanlfinal"] %}
+        {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %}
         {% else %}
         {% set steps = ["eobs", "eupd"] %}
         {% if lobsdiag_forenkf %}

diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2
@@ -69,6 +69,10 @@ gdas:
         {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %}
         - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat"
         {% endif %}
+        {% if DO_PREP_OBS_AERO %}
+        - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs"
+        - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs"
+        {% endif %}
         {% if DO_JEDISNOWDA %}
         - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz"
         {% endif %}

diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2
@@ -4,7 +4,7 @@ gdasice:
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar"
     required:
         - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc"
-        {% for fhr in range(FHOUT_OCNICE, FHMAX+1, FHOUT_OCNICE) %}
+        {% for fhr in range(FHOUT_ICE, FHMAX+1, FHOUT_ICE) %}
         - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc"
         {% endfor %}
         - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in'
diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2
@@ -3,7 +3,7 @@ gdasocean:
     name: "GDASOCEAN"
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar"
     required:
-        {% for fhr in range(FHMIN, FHMAX + 1, FHOUT_OCNICE) %}
+        {% for fhr in range(FHMIN, FHMAX + 1, FHOUT_OCN) %}
         - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc"
         {% endfor %}
         - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.MOM_input'
diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2
@@ -40,6 +40,10 @@ gfsa:
         {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %}
         - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat"
         {% endif %}
+        {% if DO_PREP_OBS_AERO %}
+        - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs"
+        - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs"
+        {% endif %}
 
         # BUFR inputs
         - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr"

diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2
@@ -4,7 +4,7 @@ ice_grib2:
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar"
     required:
         # Ice forecast GRIB2 products
-        {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %}
+        {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %}
         {% set fhr3 = '%03d' % fhr %}
         {% if ICERES == 500 %}
         - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2"

diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2
@@ -56,7 +56,6 @@ datasets:
 # Ocean forecasts
 {% filter indent(width=4) %}
 {% include "ocean_6hravg.yaml.j2" %}
-{% include "ocean_daily.yaml.j2" %}
 {% include "ocean_grib2.yaml.j2" %}
 {% include "gfs_flux_1p00.yaml.j2" %}
 {% endfilter %}

diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2
diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2
@@ -3,7 +3,7 @@ ocean_grib2:
     name: "OCEAN_GRIB2"
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar"
     required:
-        {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %}
+        {% for fhr in range(FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %}
         {% set fhr3 = '%03d' % fhr %}
         {% if OCNRES == 500 %}
         - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2"

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -30,6 +30,8 @@ export FIXgfs=${HOMEgfs}/fix
 export PARMgfs=${HOMEgfs}/parm
 export SCRgfs=${HOMEgfs}/scripts
 export USHgfs=${HOMEgfs}/ush
+export FIXorog=${FIXgfs}/orog
+export FIXugwd=${FIXgfs}/ugwd
 
 ########################################################################
 
@@ -221,6 +223,13 @@ case "${APP}" in
     ;;
 esac
 
+# Output frequency of the forecast model (for cycling)
+export FHMIN=0
+export FHMAX=9
+export FHOUT=3           # Will be changed to 1 in config.base if (DOHYBVAR set to NO and l4densvar set to false)
+export FHOUT_OCN=3
+export FHOUT_ICE=3
+
 # GFS cycle info
 export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles.
 
@@ -231,7 +240,8 @@ export FHMAX_GFS=@FHMAX_GFS@
 export FHOUT_GFS=6
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
-export FHOUT_OCNICE_GFS=6
+export FHOUT_OCN_GFS=6
+export FHOUT_ICE_GFS=6
 export FHMIN_WAV=0
 export FHOUT_WAV=3
 export FHMAX_HF_WAV=120
@@ -285,12 +295,17 @@ export MEMDIR="mem${ENSMEM}"
 
 # initialize ocean ensemble members with perturbations
 #   if true, only occurs for members greater than zero
-export USE_OCN_PERTURB_FILES=@USE_OCN_PERTURB_FILES@
+export REPLAY_ICS=@REPLAY_ICS@
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+    export OFFSET_START_HOUR=$(( assim_freq / 2 ))
+else
+    export OFFSET_START_HOUR=0
+fi
 
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then
-  export IAU_FHROT=0
+  export IAU_FHROT=${OFFSET_START_HOUR}
 else
   if [[ "${DOIAU}" = "YES" ]]; then
     export IAU_FHROT=3

diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs
@@ -67,7 +67,7 @@ export EPBL="0.8,0.4,0.2,0.08,0.04"
 export EPBL_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
 export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
 
-if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
     export ODA_INCUPD="True"
     export ODA_TEMPINC_VAR='t_pert'
     export ODA_SALTINC_VAR='s_pert'

diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst
@@ -36,7 +36,8 @@ export FHMAX=${FHMAX_GFS}
 export FHOUT=${FHOUT_GFS}
 export FHMAX_HF=${FHMAX_HF_GFS}
 export FHOUT_HF=${FHOUT_HF_GFS}
-export FHOUT_OCNICE=${FHOUT_OCNICE_GFS}
+export FHOUT_OCN=${FHOUT_OCN_GFS}
+export FHOUT_ICE=${FHOUT_ICE_GFS} 
 
 # Get task specific resources
 source "${EXPDIR}/config.resources" fcst
@@ -255,6 +256,9 @@ else
   export io_layout="1,1"
 fi
 
+if (( OFFSET_START_HOUR != 0 )); then
+    export reforecast="YES"
+fi
 # Remember config.efcs will over-ride these values for ensemble forecasts
 # if these variables are re-defined there.
 # Otherwise, the ensemble forecast will inherit from config.fcst

diff --git a/parm/config/gefs/config.oceanice_products b/parm/config/gefs/config.oceanice_products
diff --git a/parm/config/gefs/config.oceanice_products b/parm/config/gefs/config.oceanice_products
@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+########## config.oceanice_products ##########
+
+echo "BEGIN: config.oceanice_products"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" oceanice_products
+
+export OCEANICEPRODUCTS_CONFIG="${PARMgfs}/post/oceanice_products_gefs.yaml"
+
+# No. of forecast hours to process in a single job
+export NFHRS_PER_GROUP=3
+
+echo "END: config.oceanice_products"