Tier-2 tests

README.md

@@ -5,17 +5,19 @@ The one app to rule them all
 
 ## Notes to developers
 
-1 - As a developer, you will have to build the GDASApp againt the `dev/gdasapp` branch of the `global-workflow`
+1 - As a developer, you will have to build the GDASApp againt the up to date `develop` branch of the `global-workflow`
 
 ```bash
-git clone --recursive --jobs 8 --branch dev/gdasapp https://github.com/NOAA-EMC/global-workflow.git
+git clone --recursive --jobs 8 https://github.com/NOAA-EMC/global-workflow.git
 ```
 
-2 - For the most part (boundaries of that terminology is stil TBD), changes to the `global-workflow` related to the `GDASApp` developement will be issued in the `dev/gdasapp` branch.
+2 - To trigger the "fast" GDASApp tests: add the label **`{machine}-GW-RT`**
 
-3 - If your feature development involves changes to the global-workflow and the GDASApp, you will have to issue a PR in each repository. The GDASApp PR will have to have the same name in order for the GDASApp CI to build and run the correct branches.
+3 - To trigger all the GDASApp tests, including the tier-2 testing (subset of the global-workflow ci): add the labels **`{machine}-GW-RT`** and **`{machine}-GW-RT-tier2`**
 
-4 - If your work is contained within the `global-workflow` only, you will have to submit a dummy PR in the `GDASApp` with the same branch name.
+4 - If your feature development involves changes to the `global-workflow` and the `GDASApp`, you will have to issue a PR in each repository. The `GDASApp` PR will have to have the same name in order for the `GDASApp` CI to build and run the correct branches.
+
+5 - If your work is contained within the `global-workflow` only, you will have to submit a dummy PR in the `GDASApp` with the same branch name.
 To submit a dummy PR, just create a branch with an empty commit:
 ```
 git commit --allow-empty -m "Dummy commit to trigger PR"

ci/gw_driver.sh

@@ -50,13 +50,15 @@ esac
 # pull on the repo and get list of open PRs
 cd $GDAS_CI_ROOT/repo
 CI_LABEL="${GDAS_CI_HOST}-GW-RT"
+tier2_label="${GDAS_CI_HOST}-GW-RT-tier2"
 gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list_gw
 open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list_gw)
 
 # ==============================================================================
 # clone, checkout, build, test, etc.
 repo_url="https://github.com/NOAA-EMC/GDASApp.git"
 workflow_url="https://github.com/NOAA-EMC/global-workflow.git"
+workflow_branch="develop"
 # loop through all open PRs
 for pr in $open_pr_list; do
   gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running
@@ -65,6 +67,15 @@ for pr in $open_pr_list; do
   # get the branch name used for the PR
   gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName")
 
+  # check if we should run the tier-2 tests
+  pr_number=$(gh pr list --head "$gdasapp_branch" --json number --jq '.[0].number')
+  all_labels=$(gh pr view "$pr_number" --json labels --jq '.labels[].name')
+  do_tier2="false"
+  if echo "$all_labels" | grep -q "^$tier2_label$"; then
+      echo "Tier-2 label found"
+      do_tier2="true"
+  fi
+
   # check for a companion PR in the global-workflow
   companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open)
   if [ -n "$companion_pr_exists" ]; then
@@ -77,7 +88,7 @@ for pr in $open_pr_list; do
 
     # Construct the fork URL
     workflow_url="https://github.com/$fork_owner/$fork_name.git"
-
+    workflow_branch=$gdasapp_branch
     echo "Fork URL: $workflow_url"
     echo "Branch Name: $gdasapp_branch"
   fi
@@ -88,9 +99,9 @@ for pr in $open_pr_list; do
   fi
   mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr
   cd $GDAS_CI_ROOT/workflow/PR/$pr
-  
+
   # clone global workflow develop branch
-  git clone --recursive --jobs 8 --branch dev/gdasapp $workflow_url
+  git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url
 
   # checkout pull request
   cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd
@@ -103,7 +114,9 @@ for pr in $open_pr_list; do
   commit=$(git log --pretty=format:'%h' -n 1)
   echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit
 
-  $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit}
+  $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow \
+                       -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} \
+                       -t $do_tier2
   ci_status=$?
   gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit}
   if [ $ci_status -eq 0 ]; then
@@ -116,4 +129,3 @@ done
 # ==============================================================================
 # scrub working directory for older files
 find $GDAS_CI_ROOT/workflow/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \;
-

ci/run_gw_ci.sh

@@ -9,25 +9,31 @@ usage() {
   echo
   echo "  -d  Run build and ctest for clone in <directory>"
   echo "  -o  Path to output message detailing results of CI tests"
+  echo "  -t  run the tier-2 testing, default is false"
   echo "  -h  display this message and quit"
   echo
   exit 1
 }
 
 # ==============================================================================
-while getopts "d:o:h" opt; do
+do_tier2="false"
+while getopts "d:o:h:t:" opt; do
   case $opt in
     d)
       repodir=$OPTARG
       ;;
     o)
       outfile=$OPTARG
       ;;
+    t)
+      do_tier2=$OPTARG
+      ;;
     h|\?|:)
       usage
       ;;
   esac
 done
+echo "do_tier2: $do_tier2"
 
 # ==============================================================================
 # start output file
@@ -61,6 +67,15 @@ cd $repodir/sorc/gdas.cd/build
 module use $repodir/sorc/gdas.cd/modulefiles
 module load GDAS/$TARGET
 echo "---------------------------------------------------" >> $outfile
+# Reconfigure if the tier-2 testing is required
+# TODO: Not the most efficient, but even when exported, the variable is out of scope
+#       when running build.sh
+if [ "$do_tier2" = "true" ]; then
+  echo "Tier-2 Testing: Activated" >> $outfile
+  cmake -DGDAS_TIER2_TESTING=ON . >> log.cmake_tier2
+fi
+(cd gdas && ctest -N) >> log.gdasapp_tests
+
 rm -rf log.ctest
 ctest -R gdasapp --output-on-failure &>> log.ctest
 ctest_status=$?

test/CMakeLists.txt

@@ -139,8 +139,8 @@ if (${BUILD_GDASBUNDLE})
   add_subdirectory(fv3jedi)   # fv3jedi tests
   add_subdirectory(soca)      # soca tests
   add_subdirectory(snow)      # snow da tests
-  option(RUN_GW_CI "Enable the global-workflow CI tests" OFF)
-  if (RUN_GW_CI)
+  option(GDAS_TIER2_TESTING "Enable the global-workflow CI tests" OFF)
+  if (GDAS_TIER2_TESTING)
     add_subdirectory(gw-ci)   # subset of the global-workflow ci tests
   endif()
 

test/gw-ci/CMakeLists.txt

@@ -1,10 +1,10 @@
 # Function that generates the 1/2 cycle forecast and DA tasks
-function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR TASK_LIST)
+function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR TASK_LIST NMEM)
   # Prepare the COMROOT and EXPDIR for the cycling ctests
   add_test(NAME ${pslot}
     COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/create_exp.sh ${YAML_PATH} ${pslot} ${HOMEgfs} ${RUNTESTS}"
     WORKING_DIRECTORY ${RUNTESTS})
-  set_tests_properties(${pslot} PROPERTIES LABELS "manual")
+  set_tests_properties(${pslot} PROPERTIES LABELS "gw-ci")
 
   # Get the 1/2 cycle and full cycle's dates
   execute_process(
@@ -22,7 +22,17 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T
   add_test(NAME ${pslot}_gdasfcst_${HALF_CYCLE}
            COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} gdasfcst ${HALF_CYCLE}"
            WORKING_DIRECTORY ${RUNTESTS})
-  set_tests_properties(${pslot}_gdasfcst_${HALF_CYCLE} PROPERTIES LABELS "manual")
+  set_tests_properties(${pslot}_gdasfcst_${HALF_CYCLE} PROPERTIES LABELS "gw-ci")
+
+  # 1/2 cycle enkfgdasfcst
+  message(STATUS "preparing 1/2 cycle ensemble forecast for ${pslot} ctest")
+  foreach(mem RANGE 1 ${NMEM})
+    set(test_name "${pslot}_enkfgdasfcst_00${mem}_${HALF_CYCLE}")
+    add_test(NAME ${test_name}
+             COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} enkfgdasfcst_00${mem} ${HALF_CYCLE}"
+             WORKING_DIRECTORY ${RUNTESTS})
+    set_tests_properties(${test_name} PROPERTIES LABELS "gw-ci")
+  endforeach()
 
   # Select the list of tasks to run for the full cycle
   message(STATUS "Tasks ${TASK_LIST}")
@@ -32,7 +42,7 @@ function(add_cycling_tests pslot YAML_PATH HOMEgfs RUNTESTS PROJECT_SOURCE_DIR T
     add_test(NAME ${pslot}_${task}_${FULL_CYCLE}
              COMMAND /bin/bash -c "${PROJECT_SOURCE_DIR}/test/gw-ci/run_exp.sh ${pslot} ${task} ${FULL_CYCLE}"
              WORKING_DIRECTORY ${RUNTESTS})
-    set_tests_properties(${pslot}_${task}_${FULL_CYCLE} PROPERTIES LABELS "manual")
+    set_tests_properties(${pslot}_${task}_${FULL_CYCLE} PROPERTIES LABELS "gw-ci")
   endforeach()
 endfunction()
 
@@ -52,21 +62,32 @@ set(TASK_LIST
   "gdasocnanalchkpt"
   "gdasocnanalpost"
   )
-add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}")
+add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}" 0)
 
 # Aero-Land DA, C96
 # -----------------
 set(pslot "Aero-Snow-3DVAR-C96")
 set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C96_atmaerosnowDA.yaml)
 set(TASK_LIST)  # empty list for now
-add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}")
+add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}" 0)
 
 # Atm DA, C96/C48
 # ---------------
 set(pslot "Atm-hyb-C96C48")
 set(YAML_PATH ${HOMEgfs}/ci/cases/pr/C96C48_ufs_hybatmDA.yaml)
-set(TASK_LIST)  # empty list for now
-add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}")
+set(TASK_LIST
+  "gdasprep"
+  "gdasprepatmiodaobs"
+  "gdasatmanlinit"
+  "gdasatmanlvar"
+  "gdasatmanlfv3inc"
+  "gdasatmanlfinal"
+  "enkfgdasatmensanlinit"
+  "enkfgdasatmensanlletkf"
+  "enkfgdasatmensanlfv3inc"
+  "enkfgdasatmensanlfinal"
+  )  # empty list for now
+add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}" 2)
 
 # GFSv17, 3DVAR prototype
 # -----------------------
@@ -83,4 +104,4 @@ set(TASK_LIST
   "gdasprep"
   "gdasanal"
   )
-add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}")
+add_cycling_tests(${pslot} ${YAML_PATH} ${HOMEgfs} ${RUNTESTS} ${PROJECT_SOURCE_DIR} "${TASK_LIST}" 0)