Updates for cloud autort and blstores

tests/auto-jenkins/jobs/bl.py

@@ -30,12 +30,12 @@ def set_directories(job_obj):
                  f'emc.nemspara/FV3_RT/REGRESSION_TEST'
     elif job_obj.machine == 'gaea':
         workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr'
-        blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT/NEMSfv3gfs'
+        blstore = '/lustre/f2/pdata/ncep/role.epic/RT/NEMSfv3gfs'
         rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\
                  f'REGRESSION_TEST'
     elif job_obj.machine == 'orion':
         workdir = '/work/noaa/epic-ps/role-epic-ps/autort/tests/auto/pr'
-        blstore = '/work2/noaa/epic-ps/RT/NEMSfv3gfs'
+        blstore = '/work/noaa/epic/UFS-WM_RT/NEMSfv3gfs'
         rtbldir = '/work/noaa/stmp/role-epic-ps/stmp/role-epic-ps/FV3_RT/'\
                  f'REGRESSION_TEST'
     elif job_obj.machine == 'cheyenne':

tests/ci/Jenkinsfile

@@ -6,8 +6,18 @@ pipeline {
         overrideIndexTriggers(false)
         skipDefaultCheckout(true)
     }
-
     stages {
+
+        stage('Launch SonarQube') {
+            steps {
+                script {
+                    build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [
+                        string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop')
+                    ], wait: false
+                }
+            }
+        }
+
         stage('ORT') {
             matrix {
                 axes {
@@ -81,17 +91,17 @@ pipeline {
                         }
                     }
                 }
-
-                post {
+            
+                post {     
                     always {
                         sh '''
                             docker rmi -f "$(docker image ls | grep -E -m1 "${IMG_NAME}" | awk '{ print $3 }')"
                             docker rmi -f "$(docker image ls | grep -E -m1 'noaaepic' | awk '{ print $3 }')"
                         '''
                     }
-                }
+
+                }   
             }
         }
     }
 }
-

tests/ci/Jenkinsfile.noaacloud

@@ -0,0 +1,144 @@
+pipeline {
+  agent none
+  stages {
+    stage('Start Parallelworks Cluster') {
+       agent {
+        label 'built-in'   
+       }
+      steps {
+        script {
+          for (label in pullRequest.labels) {
+            if ((label.matches("(.*)-aws-(.*)"))) {
+                 env.CHOICE_NODE='pclusternoaav2use1'
+            }  
+            else if ((label.matches("(.*)-gcp-(.*)"))) { 
+                env.CHOICE_NODE='gclusternoaav2usc1'
+            }  
+            else if ((label.matches("(.*)-azure-(.*)"))) {
+                env.CHOICE_NODE='azclusternoaav2use1'
+            }  
+            else { 
+                env.CHOICE_NODE='none'
+            }
+         }
+// Why do I need another if..block, because it just works this way.
+
+            if (CHOICE_NODE == 'azclusternoaav2use1') {
+                echo "Starting up Parallelworks cluster ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
+                build job: 'parallel-works-jenkins-client/start-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: 'azclusternoaav2use1'), string(name: 'PW_CLUSTER_SSH_KEY', value: '~/.ssh/pw_id_rsa'), string(name: 'JAVA_VERSION', value: '11')]
+
+            } 
+             else if (CHOICE_NODE == 'gclusternoaav2usc1') {
+                echo "Starting up Parallelworks cluster ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
+                build job: 'parallel-works-jenkins-client/start-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: 'gclusternoaav2usc1'), string(name: 'PW_CLUSTER_SSH_KEY', value: '~/.ssh/pw_id_rsa'), string(name: 'JAVA_VERSION', value: '11')]
+            }
+             else if (CHOICE_NODE == 'pclusternoaav2use1') {
+                 echo "Starting up Parallelworks cluster ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
+                build job: 'parallel-works-jenkins-client/start-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: 'pclusternoaav2use1'), string(name: 'PW_CLUSTER_SSH_KEY', value: '~/.ssh/pw_id_rsa'), string(name: 'JAVA_VERSION', value: '11')]
+            }
+             else {
+                echo "${CHOICE_NODE} is NOT a Parallelworks cluster, moving on..."
+            }
+       }    
+    }
+  } 
+  stage('Run Regression Tests script') {
+        agent {
+          label "${CHOICE_NODE}"
+        }
+        environment {
+        GITHUB_SSH_KEY = credentials('ParallelworksGithubPrivateKey')
+        ACCNR = 'epic'
+        GITHUB_TOKEN = credentials('zachgithubtoken')
+        NODE_PATH = '/lustre/'
+      }
+      steps {
+          script {
+            for (label in pullRequest.labels) {
+            if ((label.matches("(.*)-BL"))) {
+               env.CREATE_BASELINES='true'
+            } 
+            else {
+               env.CREATE_BASELINES='false'
+            }
+          }
+        }
+        cleanWs()
+        checkout scm
+        sh '''
+        pwd
+        git submodule update --init --recursive
+        cd tests/
+        if [[ "${CREATE_BASELINES}" = true ]] 
+        then
+        echo "Creating baselines for regressions tests"
+        ./rt.sh -c -l rt.conf
+        echo "Sending regression test logs to $GIT_URL"
+        git config user.email "ecc.platform@noaa.gov"
+        git config user.name "epic-cicd-jenkins"
+        set +x
+        cd ${WORKSPACE}
+        export httpsorigin=`echo ${GIT_URL} | sed -e "s/github/$GITHUB_TOKEN@github/"` > /dev/null 2>&1 
+        git remote -v | grep -w httpsorigin > /dev/null 2>&1 && git remote remove httpsorigin > /dev/null 2>&1
+        git remote add httpsorigin $httpsorigin > /dev/null 2>&1
+        git add tests/logs/RegressionTests_noaacloud.log
+        git commit -m "[AutoRT] noaacloud Job Completed.\n\n\n on-behalf-of @ufs-community <ecc.platform@noaa.gov>" 
+        git push httpsorigin HEAD:$CHANGE_BRANCH > /dev/null 2>&1
+        set -x
+        echo "Pushed logs to $GIT_URL $CHANGE_BRANCH"
+        echo "Removing labels from $GIT_URL"
+        GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
+        GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)
+        curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":[".*BL"]}'
+        else
+        echo "Running regression tests and NOT creating baselines."
+        ./rt.sh -l rt.conf
+        echo "Sending regression test logs to $GIT_URL"
+        git config user.email "ecc.platform@noaa.gov"
+        git config user.name "epic-cicd-jenkins"
+        set +x
+        cd ${WORKSPACE}
+        export httpsorigin=`echo ${GIT_URL} | sed -e "s/github/$GITHUB_TOKEN@github/"` > /dev/null 2>&1 
+        git remote -v | grep -w httpsorigin > /dev/null 2>&1 && git remote remove httpsorigin > /dev/null 2>&1
+        git remote add httpsorigin $httpsorigin > /dev/null 2>&1
+        git add tests/logs/RegressionTests_noaacloud.log
+        git commit -m "[AutoRT] noaacloud Job Completed.\n\n\n on-behalf-of @ufs-community <ecc.platform@noaa.gov>" 
+        git push httpsorigin HEAD:$CHANGE_BRANCH > /dev/null 2>&1
+        set -x
+        echo "Pushed logs to $GIT_URL $CHANGE_BRANCH"
+        echo "Removing labels from $GIT_URL"
+        GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
+        GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)
+        curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":[".*RT"]}'
+        fi
+        '''
+      }
+ }
+  stage('Stop Parallelworks Cluster') {
+       agent {
+        label 'built-in'   
+       }
+      // Pedantic comment.
+      steps {
+        script {
+          if (CHOICE_NODE == 'azclusternoaav2use1') {
+                echo "Shutting Down Parallelworks cluster ${CHOICE_NODE}."
+                build job: 'parallel-works-jenkins-client/stop-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: 'azclusternoaav2use1')]
+
+            } 
+          else if (CHOICE_NODE == 'gclusternoaav2usc1') {
+                echo "Shutting Down Parallelworks cluster ${CHOICE_NODE}."
+                build job: 'parallel-works-jenkins-client/stop-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: 'gclusternoaav2usc1')]
+            }
+          else if (CHOICE_NODE == 'pclusternoaav2use1') {
+                echo "Shutting Down Parallelworks cluster ${CHOICE_NODE}."
+                build job: 'parallel-works-jenkins-client/stop-cluster', parameters: [string(name: 'PW_CLUSTER_NAME', value: 'pclusternoaav2use1')]
+            }
+          else {
+                echo "${CHOICE_NODE} is NOT a Parallelworks cluster, moving on..."
+            }
+        }    
+      }
+    }
+  }
+}