Merge branch 'NOAA-EMC:develop' into develop_badgetest

NOAA-EMC · Jun 1, 2023 · a6b5569 · a6b5569
2 parents 8693cc9 + dce693b
commit a6b5569
Show file tree

Hide file tree

Showing 14 changed files with 224 additions and 56 deletions.
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
@@ -57,7 +57,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display "${pr_list_dbfile}" | grep -v Failed | grep Running | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --sbfile "${pr_list_dbfile}" | grep -v Failed | grep Running | awk '{print $1}') || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
@@ -86,7 +86,7 @@ for pr in ${pr_list}; do
     "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed"
     sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
     "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --sbfile "${pr_list_dbfile}"
     # Completely remove the PR and its cloned repo on sucess of all cases
     rm -Rf "${pr_dir}" 
     continue 
@@ -122,7 +122,7 @@ for pr in ${pr_list}; do
       } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}" 
       sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" "${pr_list_dbfile}"
+      "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --sbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
          sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" |  awk '{print $1}' | xargs scancel || true

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -59,18 +59,38 @@ set -x
 ############################################################
 pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 if [[ ! -f "${pr_list_dbfile}" ]]; then
-  "${HOMEgfs}/ci/scripts/pr_list_database.py" --create "${pr_list_dbfile}"
+  "${HOMEgfs}/ci/scripts/pr_list_database.py" --create --sbfile "${pr_list_dbfile}"
 fi
 
 pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" --state "open" | awk '{print $1}') || true
 
 for pr in ${pr_list}; do
-  "${HOMEgfs}/ci/scripts/pr_list_database.py" --add_pr "${pr}" "${pr_list_dbfile}"
+  db_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --add_pr "${pr}" --sbfile "${pr_list_dbfile}")
+  pr_id=0
+  #############################################################
+  # Check if a Ready labeled PR has changed back from once set   
+  # and in that case remove all previous jobs in scheduler and
+  # and remove PR from filesystem to start clean
+  #############################################################
+  if [[ "${db_list}" == *"already is in list"* ]]; then
+    pr_id=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --sbfile "${pr_list_dbfile}" | awk '{print $4}') || true
+    pr_id=$((pr_id+1))
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --sbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Ready "${pr_id}"
+    pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+    for cases in "${pr_dir}/RUNTESTS/"*; do
+      if [[ -z "${cases+x}" ]]; then
+         break
+      fi   
+      pslot=$(basename "${cases}")
+      sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" |  awk '{print $1}' | xargs scancel || true
+    done
+    rm -Rf "${pr_dir}"
+  fi
 done
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Ready | awk '{print $1}') || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --sbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Ready | awk '{print $1}') || true
 fi
 if [[ -z "${pr_list+x}" ]]; then
   echo "no PRs open and ready for checkout/build .. exiting"
@@ -85,23 +105,35 @@ fi
 
 for pr in ${pr_list}; do
   # Skip pr's that are currently Building for when driver is called in cron
-  pr_building=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display "${pr_list_dbfile}" | awk -v pr="${pr}" '{ if ($1 == pr) print $0 }' | grep Building) || true
+  pr_building=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --sbfile "${pr_list_dbfile}" | awk -v pr="${pr}" '{ if ($1 == pr) print $0 }' | grep Building) || true
   if [[ -z "${pr_building+x}" ]]; then
       continue
   fi
   "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building"
-  "${HOMEgfs}/ci/scripts/pr_list_database.py" --update_pr "${pr}" Open Building "${pr_list_dbfile}"
+  "${HOMEgfs}/ci/scripts/pr_list_database.py" --sbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Building
   echo "Processing Pull Request #${pr}"
   pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  rm -Rf "${pr_dir}"
   mkdir -p "${pr_dir}"
   # call clone-build_ci to clone and build PR
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
   set +e
   "${HOMEgfs}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${pr_dir}/output_${id}"
   ci_status=$?
+  ##################################################################
+  # Checking for special case when Ready label was updated
+  # that cause a running driver exit fail because was currently
+  # building so we force and exit 0 instead to does not get relabled
+  #################################################################
+  if [[ ${ci_status} -ne 0 ]]; then
+     pr_id_check=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --sbfile "${pr_list_dbfile}" | awk '{print $4}') || true
+     if [[ "${pr_id}" -ne "${pr_id_check}" ]]; then
+        exit 0
+     fi   
+  fi
   set -e
   if [[ ${ci_status} -eq 0 ]]; then
-    "${HOMEgfs}/ci/scripts/pr_list_database.py" --update_pr "${pr}" Open Built "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --sbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Built
     #setup space to put an experiment
     # export RUNTESTS for yaml case files to pickup
     export RUNTESTS="${pr_dir}/RUNTESTS"
@@ -126,14 +158,14 @@ for pr in ${pr_list}; do
           echo "Case setup: Completed at $(date) for experiment ${pslot}" || true
         } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
         "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running"
-        "${HOMEgfs}/ci/scripts/pr_list_database.py" --update_pr "${pr}" Open Running "${pr_list_dbfile}"
+        "${HOMEgfs}/ci/scripts/pr_list_database.py" --sbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running
       else 
         {
           echo "Failed to create experiment:  *FAIL* ${pslot}"
           echo "Experiment setup: failed at $(date) for experiment ${pslot}" || true
         } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
         "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
-        "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" "${pr_list_dbfile}"
+        "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" -sbfile "${pr_list_dbfile}"
       fi
     done
 
@@ -144,7 +176,7 @@ for pr in ${pr_list}; do
       echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true
     } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
     "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
-    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" "${pr_list_dbfile}"
+    "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --sbfile "${pr_list_dbfile}"
   fi
   sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
   "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"

diff --git a/ci/scripts/pr_list_database.py b/ci/scripts/pr_list_database.py
@@ -2,7 +2,7 @@
 
 import sys
 from pathlib import Path
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER
 import sqlite3
 
 
@@ -39,7 +39,7 @@ def sql_table(obj: sqlite3.Cursor) -> None:
 
     """
 
-    obj.execute("CREATE TABLE processing(pr integer PRIMARY KEY, state text, status text)")
+    obj.execute("CREATE TABLE processing(pr integer PRIMARY KEY, state text, status text, reset integer)")
 
 
 def sql_insert(obj: sqlite3.Cursor, entities: list) -> None:
@@ -51,14 +51,18 @@ def sql_insert(obj: sqlite3.Cursor, entities: list) -> None:
     obj : sqlite3.Cursor
         Cursor object for Sqlite3
     entities : list
-        The list three string values that go into sqlite table (pr, state, status)
+        A list of four string values that go into sqlite table (pr, state, status, reset)
+            pr: pull request number
+            state: The new value for the state (Open, Closed)
+            status: The new value for the status (Ready, Running, Failed)
+            reset: The value for number of times reset to Ready
 
     """
 
-    obj.execute('INSERT INTO processing(pr, state, status) VALUES(?, ?, ?)', entities)
+    obj.execute('INSERT INTO processing(pr, state, status, reset) VALUES(?, ?, ?, ?)', entities)
 
 
-def sql_update(obj: sqlite3.Cursor, pr: str, state: str, status: str) -> None:
+def sql_update(obj: sqlite3.Cursor, pr: str, updates: dict) -> None:
     """Updates table for a given pr with new values for state and status
 
     Parameters
@@ -67,14 +71,18 @@ def sql_update(obj: sqlite3.Cursor, pr: str, state: str, status: str) -> None:
         sqlite3 Cursor Object
     pr : str
         The given pr number to update in the table
-    state : str
-        The new value for the state (Open, Closed)
-    status: str
-        The new value for the status (Ready, Running, Failed)
+    updates : dict
+        Dictionary of values to update for a given PR to include by postion
+        state, The new value for the state (Open, Closed)
+        status, The new value for the status (Ready, Running, Failed)
+        reset, The value for number of times reset to Ready
 
     """
 
-    obj.execute(f'UPDATE processing SET state = "{state}", status = "{status}" WHERE pr = {pr}')
+    update_list = ['state', 'status', 'reset']
+    for value in updates:
+        update = update_list.pop(0)
+        obj.execute(f'UPDATE processing SET "{update}" = "{value}" WHERE pr = {pr}')
 
 
 def sql_fetch(obj: sqlite3.Cursor) -> list:
@@ -114,11 +122,12 @@ def input_args():
     parser = ArgumentParser(description=description,
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('sbfile', help='SQLite3 database file with PR list', type=str)
+    parser.add_argument('--sbfile', help='SQLite3 database file with PR list', type=str)
     parser.add_argument('--create', help='create sqlite file for pr list status', action='store_true', required=False)
     parser.add_argument('--add_pr', nargs=1, metavar='PR', help='add new pr to list (defults to: Open,Ready)', required=False)
     parser.add_argument('--remove_pr', nargs=1, metavar='PR', help='removes pr from list', required=False)
-    parser.add_argument('--update_pr', nargs=3, metavar=('pr', 'state', 'status'), help='updates state and status of a given pr', required=False)
+    parser.add_argument('--update_pr', nargs=REMAINDER, metavar=('pr', 'state', 'status', 'reset'),
+                        help='updates state and status of a given pr', required=False)
     parser.add_argument('--display', help='output pr table', action='store_true', required=False)
 
     args = parser.parse_args()
@@ -142,14 +151,16 @@ def input_args():
                 print(f"pr {row[0]} already is in list: nothing added")
                 sys.exit(0)
 
-        entities = (args.add_pr[0], 'Open', 'Ready')
+        entities = (args.add_pr[0], 'Open', 'Ready', 0)
         sql_insert(obj, entities)
 
     if args.update_pr:
+        if len(args.update_pr) < 2:
+            print(f"update_pr must have at least one vaule to update")
+            sys.exit(0)
         pr = args.update_pr[0]
-        state = args.update_pr[1]
-        status = args.update_pr[2]
-        sql_update(obj, pr, state, status)
+
+        sql_update(obj, pr, args.update_pr[1:])
 
     if args.remove_pr:
         sql_remove(obj, args.remove_pr[0])

diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
@@ -47,7 +47,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db"
 
 pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
-  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
+  pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --sbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true
 fi
 if [[ -z "${pr_list}" ]]; then
   echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting"

diff --git a/docs/source/wave.rst b/docs/source/wave.rst
@@ -98,7 +98,7 @@ You will need to add the following files:
 
 For config.ufs: 
 
-If this is a computational you will need to update this file. If this is a new output file you can skip this update.
+If this is a computational grid, you will need to update this file. If this is a new output file you can skip this update.
 There are two sections to update: 
 1. Add the new grid as a possible ww3 resolution at the top of the file 
 2. In the WW3 specific settings section, add a case for the new grid and define ntasks_ww3 (number of tasks for WW3) and 
@@ -122,11 +122,17 @@ computational grid or just an output grid you will need a new parm/wave/ww3_grib
 WW3/model/inp/ww3_grib.inp for more infomration about this input file.  The other existing templated input files can be used 
 as guides to create these new files. 
 
-
-
-
 *****************************
 Updates for New Product Grids 
 *****************************
 
-If this is a new file for AWIPS Products, additional updates are likely required.  
+If this is a new file for AWIPS Products, additional updates are likely required.
+
+**************
+New Staged ICs 
+**************
+
+Depending on the desired use of the new grid, or if the default grid is changed, staged initial conditions for use in 
+workflow testing will also be needed. 
+
+For example, C384 S2SW need an IC for 20130101 00z and low resolution tests need an IC for 20210323 12z. 
diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn
@@ -161,15 +161,32 @@ export CASE_ENS="@CASEENS@"
 # TODO: This should not depend on $CASE or $CASE_ENS
 # These are the currently available grid-combinations
 case "${CASE}" in
-    "C48") export OCNRES=500;;
-    "C96") export OCNRES=100;;
-    "C192") export OCNRES=050;;
-    "C384") export OCNRES=025;;
-    "C768") export OCNRES=025;;
-    *) export OCNRES=025;;
+    "C48")
+        export OCNRES=500
+        export waveGRD='glo_500'
+        ;;
+    "C96")
+        export OCNRES=100
+        export waveGRD='glo_200'
+        ;;
+    "C192")
+        export OCNRES=050
+        export waveGRD='glo_200'
+        ;;
+    "C384")
+        export OCNRES=025
+        export waveGRD='glo_025'
+        ;;
+    "C768")
+        export OCNRES=025
+        export waveGRD='mx025'
+        ;;
+    *) 
+        export OCNRES=025
+        export waveGRD='glo_025'
+        ;;
 esac
 export ICERES=${OCNRES}
-export waveGRD='glo_025'
 
 case "${APP}" in
   ATM)

diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs
@@ -15,7 +15,7 @@ if [ $# -le 1 ]; then
     echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072"
     echo "--mom6 500|100|025"
     echo "--cice6 500|100|025"
-    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|mx025"
+    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025"
 
     exit 1
 
@@ -157,7 +157,7 @@ case "${fv3_res}" in
         export WRITE_GROUP=2
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=8
         export WRITE_GROUP_GFS=2
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=4
+        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10
         ;;
     "C768")
         export DELTIM=150
@@ -353,15 +353,18 @@ if [[ "${skip_ww3}" == "false" ]]; then
     "gwes_30m")
       ntasks_ww3=100
       ;;
-    "mx050")
-      ntasks_ww3=240
+    "glo_025")
+      ntasks_ww3=262
+      ;;
+    "glo_200")
+      ntasks_ww3=40
+      ;;
+    "glo_500")
+      ntasks_ww3=10
       ;;
     "mx025")
       ntasks_ww3=80
       ;;
-    "glo_025")
-      ntasks_ww3=262
-      ;;
     *)
       echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!"
       exit 1

diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
@@ -161,15 +161,32 @@ export CASE_ENS="@CASEENS@"
 # TODO: This should not depend on $CASE or $CASE_ENS
 # These are the currently available grid-combinations
 case "${CASE}" in
-    "C48") export OCNRES=500;;
-    "C96") export OCNRES=100;;
-    "C192") export OCNRES=050;;
-    "C384") export OCNRES=025;;
-    "C768") export OCNRES=025;;
-    *) export OCNRES=025;;
+    "C48")
+        export OCNRES=500
+        export waveGRD='glo_500'
+        ;;
+    "C96")
+        export OCNRES=100
+        export waveGRD='glo_200'
+        ;;
+    "C192")
+        export OCNRES=050
+        export waveGRD='glo_200'
+        ;;
+    "C384")
+        export OCNRES=025
+        export waveGRD='glo_025'
+        ;;
+    "C768")
+        export OCNRES=025
+        export waveGRD='mx025'
+        ;;
+    *) 
+        export OCNRES=025
+        export waveGRD='mx025'
+        ;;
 esac
 export ICERES=${OCNRES}
-export waveGRD='mx025'
 
 case "${APP}" in
   ATM)