Retry Jenkins CI tests up to 3 times to improve reliability (dmlc#3769)

criteo-forks · Dec 4, 2018 · bfc0c16 · bfc0c16
1 parent 7d78f6d
commit bfc0c16
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 49 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -74,29 +74,31 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
     def test_suite = conf["withGpu"] ? (conf["multiGpu"] ? "mgpu" : "gpu") : "cpu"
     // Build node - this is returned result
     node(nodeReq) {
-        unstash name: 'srcs'
-        echo """
-        |===== XGBoost CMake build =====
-        |  dockerTarget: ${dockerTarget}
-        |  cmakeOpts   : ${opts}
-        |=========================
-        """.stripMargin('|')
-        // Invoke command inside docker
-        sh """
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${test_suite}.sh
-        """
-        if (!conf["multiGpu"]) {
+        retry(3) {
+            unstash name: 'srcs'
+            echo """
+            |===== XGBoost CMake build =====
+            |  dockerTarget: ${dockerTarget}
+            |  cmakeOpts   : ${opts}
+            |=========================
+            """.stripMargin('|')
+            // Invoke command inside docker
             sh """
-            ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
-            rm -rf "${distDir}"; mkdir -p "${distDir}/py"
-            cp xgboost "${distDir}"
-            cp -r python-package/dist "${distDir}/py"
-            # Test the wheel for compatibility on a barebones CPU container
-            ${dockerRun} release ${dockerArgs} bash -c " \
-                pip install --user python-package/dist/xgboost-*-none-any.whl && \
-                python -m nose tests/python"
+            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
+            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${test_suite}.sh
             """
+            if (!conf["multiGpu"]) {
+                sh """
+                ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
+                rm -rf "${distDir}"; mkdir -p "${distDir}/py"
+                cp xgboost "${distDir}"
+                cp -r python-package/dist "${distDir}/py"
+                # Test the wheel for compatibility on a barebones CPU container
+                ${dockerRun} release ${dockerArgs} bash -c " \
+                    pip install --user python-package/dist/xgboost-*-none-any.whl && \
+                    python -m nose tests/python"
+                """
+            }
         }
     }
 }
diff --git a/Jenkinsfile-restricted b/Jenkinsfile-restricted
@@ -52,17 +52,19 @@ pipeline {
                 label 'linux && cpu && restricted'
             }
             steps {
-                unstash name: 'srcs'
-                script {
-                    def commit_id = "${GIT_COMMIT}"
-                    def branch_name = "${GIT_LOCAL_BRANCH}"
-                    echo 'Building doc...'
-                    dir ('jvm-packages') {
-                        sh "bash ./build_doc.sh ${commit_id}"
-                        archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true
-                        echo 'Deploying doc...'
-                        withAWS(credentials:'xgboost-doc-bucket') {
-                            s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2"
+                retry(3) {
+                    unstash name: 'srcs'
+                    script {
+                        def commit_id = "${GIT_COMMIT}"
+                        def branch_name = "${GIT_LOCAL_BRANCH}"
+                        echo 'Building doc...'
+                        dir ('jvm-packages') {
+                            sh "bash ./build_doc.sh ${commit_id}"
+                            archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true
+                            echo 'Deploying doc...'
+                            withAWS(credentials:'xgboost-doc-bucket') {
+                                s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2"
+                            }
                         }
                     }
                 }
@@ -95,22 +97,24 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
     }
     // Build node - this is returned result
     node(nodeReq) {
-        unstash name: 'srcs'
-        echo """
-        |===== XGBoost CMake build =====
-        |  dockerTarget: ${dockerTarget}
-        |  cmakeOpts   : ${opts}
-        |=========================
-        """.stripMargin('|')
-        // Invoke command inside docker
-        sh """
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
-        ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
-        rm -rf "${distDir}"; mkdir -p "${distDir}/py"
-        cp xgboost "${distDir}"
-        cp -r lib "${distDir}"
-        cp -r python-package/dist "${distDir}/py"
-        """
-        archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
+        retry(3) {
+            unstash name: 'srcs'
+            echo """
+            |===== XGBoost CMake build =====
+            |  dockerTarget: ${dockerTarget}
+            |  cmakeOpts   : ${opts}
+            |=========================
+            """.stripMargin('|')
+            // Invoke command inside docker
+            sh """
+            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
+            ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
+            rm -rf "${distDir}"; mkdir -p "${distDir}/py"
+            cp xgboost "${distDir}"
+            cp -r lib "${distDir}"
+            cp -r python-package/dist "${distDir}/py"
+            """
+            archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
+        }
     }
 }