Merge branch 'NOAA-EMC:develop' into feature/debug_switch

Jenkinsfile

@@ -1,8 +1,8 @@
 def Machine = 'none'
 def machine = 'none'
 def HOME = 'none'
-def localworkspace = 'none'
-def commonworkspace = 'none'
+def caseList = ''
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']
 
 pipeline {
     agent { label 'built-in' }
@@ -20,7 +20,6 @@ pipeline {
             agent { label 'built-in' }
             steps {
                 script {
-                    localworkspace = env.WORKSPACE
                     machine = 'none'
                     for (label in pullRequest.labels) {
                         echo "Label: ${label}"
@@ -42,14 +41,16 @@ pipeline {
             agent { label "${machine}-emc" }
             steps {
                 script {
-                    properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
-                    HOME = "${WORKSPACE}/TESTDIR"
-                    commonworkspace = "${WORKSPACE}"
-                    sh(script: "mkdir -p ${HOME}/RUNTESTS")
-                    pullRequest.addLabel("CI-${Machine}-Building")
-                    if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
-                        pullRequest.removeLabel("CI-${Machine}-Ready")
+                    ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        HOME = "${WORKSPACE}"
+                        sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
+                        pullRequest.addLabel("CI-${Machine}-Building")
+                        if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
+                            pullRequest.removeLabel("CI-${Machine}-Ready")
+                        }
                     }
+                    pullRequest.comment("Building and running on ${Machine} in directory ${HOME}")
                 }
             }
         }
@@ -73,17 +74,15 @@ pipeline {
                                 def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
                                 sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
-                                    env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
                                         sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
                                         pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}<br>Still doing a checkout to get the latest changes")
-                                        sh(script: 'source workflow/gw_setup.sh; git pull --recurse-submodules')
+                                        checkout scm
                                         dir('sorc') {
                                             sh(script: './link_workflow.sh')
                                         }
                                     } else {
                                         checkout scm
-                                        sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive')
                                         def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
                                         def build_args_list = builds_file['builds']
                                         def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
@@ -99,6 +98,9 @@ pipeline {
                                        }
                                        pullRequest.addLabel("CI-${Machine}-Running")
                                     }
+                                    if (system == 'gfs') {
+                                        caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                    }
                                }
                            }
                         }
@@ -114,11 +116,15 @@ pipeline {
                     axis {
                         name 'Case'
                         // TODO add dynamic list of cases from env vars (needs addtional plugins)
-                        values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA', 'C96_atmsnowDA'  // 'C48mx500_3DVarAOWCDA'
+                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA'
                     }
                 }
                 stages {
-                    stage('Create Experiment') {
+
+                    stage('Create Experiments') {
+                        when {
+                            expression { return caseList.contains(Case) }
+                        }
                         steps {
                                 script {
                                     sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
@@ -130,15 +136,20 @@ pipeline {
                                 }
                         }
                     }
+
                     stage('Run Experiments') {
+                        when {
+                            expression { return caseList.contains(Case) }
+                        }
                         steps {
                             script {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                     pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
-                                    // pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
-                                    err =  sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
-                                    if (err != 0) {
+                                    pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
+                                    try {
+                                       sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
+                                    } catch (Exception e) {
                                         pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
                                         sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
                                         ws(HOME) {
@@ -153,12 +164,12 @@ pipeline {
                                         }
                                         error("Failed to run experiments ${Case} on ${Machine}")
                                     }
-                                   // pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
                                 }
+                                pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
                             }
-
                         }
                     }
+
                 }
             }
         }

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -16,8 +16,3 @@ arguments:
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
-
-skip_ci_on_hosts:
-  - hera
-  - orion
-  - hercules

ci/cases/yamls/atmsnowDA_defaults_ci.yaml

@@ -2,5 +2,5 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
   DOIAU: "NO"
-  DO_JEDILANDDA: "YES"
+  DO_JEDISNOWDA: "YES"
   ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -5,10 +5,16 @@ base:
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
   ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+atmanl:
+  LAYOUT_X_ATMANL: 1
+  LAYOUT_Y_ATMANL: 1
+atmensanl:
+  LAYOUT_X_ATMENSANL: 1
+  LAYOUT_Y_ATMENSANL: 1
 esfc:
   DONST: "NO"
 nsst:
   NST_MODEL: "1"
 sfcanl:
   DONST: "NO"
-
+

ci/scripts/utils/get_host_case_list.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import os
+from os.path import basename, splitext
+import sys
+import glob
+from wxflow import parse_j2yaml
+from wxflow import AttrDict
+
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))
+
+if __name__ == '__main__':
+
+    if len(sys.argv) < 2:
+        print('Usage: get_host_case_list.py <host_name>')
+        sys.exit(1)
+
+    host = sys.argv[1]
+
+    case_list = []
+    HOMEgfs = _top
+    data = AttrDict(HOMEgfs=_top)
+    data.update(os.environ)
+
+    case_files = glob.glob(f'{HOMEgfs}/ci/cases/pr/*.yaml')
+    for case_yaml in case_files:
+        case_conf = parse_j2yaml(path=case_yaml, data=data)
+        if 'skip_ci_on_hosts' in case_conf:
+            if host.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]:
+                continue
+        case_list.append(splitext(basename(case_yaml))[0])
+    print(' '.join(case_list))

ci/scripts/utils/wxflow

@@ -0,0 +1 @@
+../../../sorc/wxflow/src/wxflow

env/AWSPW.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun landanlrun"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"

env/CONTAINER.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun landanl"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"

env/HERA.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun landanl"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -42,7 +42,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export sys_tp="HERA"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "preplandobs" ]]; then
+elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
@@ -79,13 +79,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
-elif [[ "${step}" = "landanl" ]]; then
+elif [[ "${step}" = "snowanl" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_landanl))
+    nth_max=$((npe_node_max / npe_node_snowanl))
 
-    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
-    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
-    export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
+    export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
+    [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
+    export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

env/HERCULES.env

@@ -41,7 +41,7 @@ case ${step} in
     export sys_tp="HERCULES"
     export launcher_PREP="srun"
  ;;
- "preplandobs")
+ "prepsnowobs")
 
     export APRUN_CALCFIMS="${launcher} -n 1"
  ;;
@@ -79,13 +79,13 @@ case ${step} in
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
  ;;
- "landanl")
+ "snowanl")
 
-    nth_max=$((npe_node_max / npe_node_landanl))
+    nth_max=$((npe_node_max / npe_node_snowanl))
 
-    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
-    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
-    export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
+    export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
+    [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
+    export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;

env/JET.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun landanl"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -33,7 +33,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export sys_tp="JET"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "preplandobs" ]]; then
+elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
@@ -70,13 +70,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
-elif [[ "${step}" = "landanl" ]]; then
+elif [[ "${step}" = "snowanl" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_landanl))
+    nth_max=$((npe_node_max / npe_node_snowanl))
 
-    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
-    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
-    export APRUN_LANDANL="${launcher} -n ${npe_landanl}"
+    export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
+    [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
+    export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

env/ORION.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun landanl"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -40,7 +40,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export sys_tp="ORION"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "preplandobs" ]]; then
+elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
@@ -78,13 +78,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
-elif [[ "${step}" = "landanl" ]]; then
+elif [[ "${step}" = "snowanl" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_landanl))
+    nth_max=$((npe_node_max / npe_node_snowanl))
 
-    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
-    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
-    export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
+    export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
+    [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
+    export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

env/S4.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun landanl"
+    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -33,7 +33,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export sys_tp="S4"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "preplandobs" ]]; then
+elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
@@ -70,13 +70,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
-elif [[ "${step}" = "landanl" ]]; then
+elif [[ "${step}" = "snowanl" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_landanl))
+    nth_max=$((npe_node_max / npe_node_snowanl))
 
-    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
-    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
-    export APRUN_LANDANL="${launcher} -n ${npe_landanl}"
+    export NTHREADS_SNOWANL=${nth_snowanl:-${nth_max}}
+    [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
+    export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"