Merge pull request #439 from GeminiDRSoftware/jenkins_setup_race

Avoid conda race conditions within a Jenkins test run
GeminiDRSoftware · Jul 4, 2023 · 465c7a0 · 465c7a0
2 parents 9e29c00 + 3a13f15
commit 465c7a0
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 32 deletions.
diff --git a/.jenkins/scripts/download_and_install_anaconda.sh b/.jenkins/scripts/download_and_install_anaconda.sh
@@ -32,6 +32,7 @@ fi
 
 echo ${PATH}
 which conda
+conda clean -i -t -y  # in case of corrupt package cache from previous run
 conda update --quiet conda
 # These 2 channels need removing if testing old branches has reinstated them:
 conda config --remove channels http://ssb.stsci.edu/astroconda || :

diff --git a/.jenkins/scripts/setup_agent.sh b/.jenkins/scripts/setup_agent.sh
@@ -3,17 +3,8 @@
 
 set -eux
 
-# Cleanup
+pwd
 git clean -fxd
-mkdir plots reports
-if [[ -n "${DRAGONS_TEST_OUTPUTS-}" ]]; then
-    if [[ -d "${DRAGONS_TEST_OUTPUTS}" ]]; then
-        echo "Cleaning previous test results in ${DRAGONS_TEST_OUTPUTS}"
-        rm -r ${DRAGONS_TEST_OUTPUTS}
-    else
-        echo "Skip delete unexisting ${DRAGONS_TEST_OUTPUTS}"
-    fi
-fi
 
 source .jenkins/scripts/download_and_install_anaconda.sh
 

diff --git a/.jenkins/scripts/setup_dirs.sh b/.jenkins/scripts/setup_dirs.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# -*- coding: utf-8 -*-
+
+set -eux
+
+# Cleanup
+pwd
+git clean -fxd
+mkdir plots reports
+if [[ -n "${DRAGONS_TEST_OUT-}" ]]; then
+    if [[ -d "${DRAGONS_TEST_OUT}" ]]; then
+        echo "Cleaning previous test results in ${DRAGONS_TEST_OUT}"
+        rm -r ${DRAGONS_TEST_OUT}
+    else
+        echo "Skip deletion of inexistent ${DRAGONS_TEST_OUT}"
+    fi
+else
+    echo "DRAGONS_TEST_OUT is not set, so not deleting it"
+fi
+
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -34,11 +34,12 @@ pipeline {
         skipDefaultCheckout(true)
         buildDiscarder(logRotator(numToKeepStr: '5'))
         timestamps()
-        timeout(time: 4, unit: 'HOURS')
+        timeout(time: 6, unit: 'HOURS')
     }
 
     environment {
         MPLBACKEND = "agg"
+        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
     }
 
     stages {
@@ -49,6 +50,29 @@ pipeline {
             }
         }
 
+        stage('Pre-install') {
+            agent { label "conda" }
+            environment {
+                TMPDIR = "${env.WORKSPACE}/.tmp/conda/"
+            }
+            steps {
+                echo "Update the Conda base install for all on-line nodes"
+                checkout scm
+                sh '.jenkins/scripts/setup_agent.sh'
+                echo "Create a trial Python 3.10 env, to cache new packages"
+                sh 'tox -e py310-noop -v -r -- --basetemp=${DRAGONS_TEST_OUT} ${TOX_ARGS}'
+            }
+            post {
+                always {
+                    echo "Deleting conda temp workspace ${env.WORKSPACE}"
+                    cleanWs()
+                    dir("${env.WORKSPACE}@tmp") {
+                      deleteDir()
+                    }
+                }
+            }
+        }
+
         stage('Quicker tests') {
             parallel {
 
@@ -59,15 +83,14 @@ pipeline {
                     }
                     environment {
                         MPLBACKEND = "agg"
-                        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                         DRAGONS_TEST_OUT = "unit_tests_outputs/"
                         TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                         TMPDIR = "${env.WORKSPACE}/.tmp/unit/"
                     }
                     steps {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
-                        sh '.jenkins/scripts/setup_agent.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Running tests with Python 3.10"
                         sh 'tox -e py310-unit -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/unittests_results.xml ${TOX_ARGS}'
                         echo "Reportint coverage to CodeCov"
@@ -97,7 +120,6 @@ pipeline {
                     agent { label "centos7" }
                     environment {
                         MPLBACKEND = "agg"
-                        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                         DRAGONS_TEST_OUT = "./integ_tests_outputs/"
                         TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                         TMPDIR = "${env.WORKSPACE}/.tmp/integ/"
@@ -106,7 +128,7 @@ pipeline {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         echo "${env.PATH}"
-                        sh '.jenkins/scripts/setup_agent.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Integration tests"
                         sh 'tox -e py310-integ -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/integration_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -131,7 +153,6 @@ pipeline {
                     agent { label "master" }
                     environment {
                         MPLBACKEND = "agg"
-                        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                         DRAGONS_TEST_OUT = "regression_tests_outputs"
                         TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                         TMPDIR = "${env.WORKSPACE}/.tmp/regr/"
@@ -140,7 +161,7 @@ pipeline {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         echo "${env.PATH}"
-                        sh '.jenkins/scripts/setup_agent.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Regression tests"
                         sh 'tox -e py310-reg -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/regression_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -171,15 +192,14 @@ pipeline {
             agent { label "master" }
             environment {
                 MPLBACKEND = "agg"
-                PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                 DRAGONS_TEST_OUT = "f2_tests_outputs"
                 TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                 TMPDIR = "${env.WORKSPACE}/.tmp/f2/"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
-                sh '.jenkins/scripts/setup_agent.sh'
+                sh '.jenkins/scripts/setup_dirs.sh'
                 echo "Running tests"
                 sh 'tox -e py310-f2 -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/f2_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
@@ -209,15 +229,14 @@ pipeline {
             agent { label "master" }
             environment {
                 MPLBACKEND = "agg"
-                PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                 DRAGONS_TEST_OUT = "gsaoi_tests_outputs"
                 TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                 TMPDIR = "${env.WORKSPACE}/.tmp/gsaoi/"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
-                sh '.jenkins/scripts/setup_agent.sh'
+                sh '.jenkins/scripts/setup_dirs.sh'
                 echo "Running tests"
                 sh 'tox -e py310-gsaoi -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gsaoi_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
@@ -247,15 +266,14 @@ pipeline {
             agent { label "master" }
             environment {
                 MPLBACKEND = "agg"
-                PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                 DRAGONS_TEST_OUT = "niri_tests_outputs"
                 TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                 TMPDIR = "${env.WORKSPACE}/.tmp/niri/"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
-                sh '.jenkins/scripts/setup_agent.sh'
+                sh '.jenkins/scripts/setup_dirs.sh'
                 echo "Running tests"
                 sh 'tox -e py310-niri -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/niri_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
@@ -285,15 +303,14 @@ pipeline {
             agent { label "master" }
             environment {
                 MPLBACKEND = "agg"
-                PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                 DRAGONS_TEST_OUT = "gnirs_tests_outputs"
                 TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                 TMPDIR = "${env.WORKSPACE}/.tmp/gnirs/"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
-                sh '.jenkins/scripts/setup_agent.sh'
+                sh '.jenkins/scripts/setup_dirs.sh'
                 echo "Running tests"
                 sh 'tox -e py310-gnirs -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gnirs_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
@@ -323,15 +340,14 @@ pipeline {
             agent { label "master" }
             environment {
                 MPLBACKEND = "agg"
-                PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                 DRAGONS_TEST_OUT = "wavecal_tests_outputs"
                 TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                 TMPDIR = "${env.WORKSPACE}/.tmp/wavecal/"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
-                sh '.jenkins/scripts/setup_agent.sh'
+                sh '.jenkins/scripts/setup_dirs.sh'
                 echo "Running tests"
                 sh 'tox -e py310-wavecal -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/wavecal_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
@@ -364,15 +380,14 @@ pipeline {
                     agent { label "master" }
                     environment {
                         MPLBACKEND = "agg"
-                        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                         DRAGONS_TEST_OUT = "gmosls_tests_outputs"
                         TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                         TMPDIR = "${env.WORKSPACE}/.tmp/gmosls/"
                     }
                     steps {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
-                        sh '.jenkins/scripts/setup_agent.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Running tests"
                         sh 'tox -e py310-gmosls -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gmosls_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -402,7 +417,6 @@ pipeline {
                     agent { label "master" }
                     environment {
                         MPLBACKEND = "agg"
-                        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
                         DRAGONS_TEST_OUT = "regression_tests_outputs"
                         TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system"
                         TMPDIR = "${env.WORKSPACE}/.tmp/slow/"
@@ -411,7 +425,7 @@ pipeline {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         echo "${env.PATH}"
-                        sh '.jenkins/scripts/setup_agent.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Slow tests"
                         sh 'tox -e py310-slow -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/slow_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"

diff --git a/tox.ini b/tox.ini
@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py{37,38,39,310,311}-{unit,f2,gsaoi,niri,gnirs,gmosls,integ,reg,slow}
+    py{37,38,39,310,311}-{noop,unit,f2,gsaoi,niri,gnirs,gmosls,integ,reg,slow}
     codecov
     check
     docs-{astrodata}
@@ -53,8 +53,10 @@ conda_channels =
     conda-forge
 conda_create_args =
     --override-channels
+    --experimental=lock
 conda_install_args =
     --override-channels
+    --experimental=lock
 extras =
     test
     docs: docs
@@ -71,6 +73,7 @@ commands =
     which pytest
     pip install git+https://github.com/GeminiDRSoftware/AstroFaker#egg=AstroFaker
     conda list
+    noop: python -c "pass"  # just install deps & ensure python runs
     unit: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "not integration_test and not gmosls and not f2 and not f2ls and not f2image and not gsaoi and not gsaoiimage and not niri and not nirils and not niriimage and not gnirs and not gnirsls and not gnirsimage and not wavecal and not regression and not slow" {posargs:astrodata geminidr gemini_instruments gempy recipe_system}
     integ: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "integration_test and not slow" {posargs:astrodata geminidr gemini_instruments gempy recipe_system}
     gmosls: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "gmosls and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system}