Merge branch 'NOAA-EMC:develop' into feature/gwdev_issue_2129

ci/scripts/check_ci.sh

@@ -8,7 +8,7 @@ set -eux
 #                     to run from within a cron job in the CI Managers account
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -20,21 +20,22 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -x
@@ -57,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
@@ -89,13 +90,13 @@ for pr in ${pr_list}; do
     sed -i "1 i\`\`\`" "${output_ci}"
     sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
     if [[ -n "${weekly_labels}" ]]; then
-      num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l)
+      num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l)
       passed=0
-      for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do
+      for platforms in "${HOMEgfs}"/ci/platforms/config.*; do
         machine=$(basename "${platforms}" | cut -d. -f2)
         if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then
           ((passed=passed+1))
@@ -139,7 +140,7 @@ for pr in ${pr_list}; do
       } >> "${output_ci}"
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
-      "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
          cancel_slurm_jobs "${pslot}"

ci/scripts/driver.sh

@@ -47,12 +47,15 @@ esac
 ######################################################
 # setup runtime env for correct python install and git
 ######################################################
+HOMEgfs=${ROOT_DIR}
+export HOMEgfs
 set +x
 source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 set -x
+unset HOMEgfs
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI

ci/scripts/run_ci.sh

@@ -9,7 +9,7 @@ set -eux
 # Abstract TODO
 #####################################################################################
 
-ROOT_DIR="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
 scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
@@ -18,20 +18,21 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${ROOT_DIR}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
-   source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
+   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
  *)
    echo "Unsupported platform. Exiting with error."
    exit 1
    ;;
 esac
 set +x
-source "${ROOT_DIR}/ush/module-setup.sh"
-module use "${ROOT_DIR}/modulefiles"
+export HOMEgfs
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
 set -eux
@@ -47,7 +48,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
 fi
 if [[ -z "${pr_list}" ]]; then
   echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting"

parm/config/gfs/config.base.emc.dyn

@@ -39,6 +39,7 @@ export FIXcice="${FIXgfs}/cice"
 export FIXmom="${FIXgfs}/mom6"
 export FIXreg2grb2="${FIXgfs}/reg2grb2"
 export FIXugwd="${FIXgfs}/ugwd"
+export FIXgdas="${FIXgfs}/gdas"
 
 ########################################################################
 

sorc/link_workflow.sh

@@ -190,7 +190,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
   cd "${HOMEgfs}/fix" || exit 1
   [[ ! -d gdas ]] && mkdir -p gdas
   cd gdas || exit 1
-  for gdas_sub in fv3jedi gsibec; do
+  for gdas_sub in fv3jedi gsibec obs; do
     if [[ -d "${gdas_sub}" ]]; then
        rm -rf "${gdas_sub}"
     fi

ush/python/pygfs/task/land_analysis.py

@@ -153,7 +153,7 @@ def prepare_IMS(self) -> None:
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
         keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
-                'OPREFIX', 'CASE', 'OCNRES', 'ntiles']
+                'OPREFIX', 'CASE', 'OCNRES', 'ntiles', 'FIXgdas']
         for key in keys:
             localconf[key] = self.task_config[key]
 

versions/fix.ver

@@ -10,6 +10,7 @@ export datm_ver=20220805
 export gdas_crtm_ver=20220805
 export gdas_fv3jedi_ver=20220805
 export gdas_gsibec_ver=20221031
+export gdas_obs_ver=20240213
 export glwu_ver=20220805
 export gsi_ver=20230911
 export lut_ver=20220805