Retry Jenkins CI tests up to 3 times to improve reliability

dmlc · Oct 8, 2018 · 7950ac2 · 7950ac2
1 parent 813d243
commit 7950ac2
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 38 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -74,29 +74,31 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
     def test_suite = conf["withGpu"] ? (conf["multiGpu"] ? "mgpu" : "gpu") : "cpu"
     // Build node - this is returned result
     node(nodeReq) {
-        unstash name: 'srcs'
-        echo """
-        |===== XGBoost CMake build =====
-        |  dockerTarget: ${dockerTarget}
-        |  cmakeOpts   : ${opts}
-        |=========================
-        """.stripMargin('|')
-        // Invoke command inside docker
-        sh """
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${test_suite}.sh
-        """
-        if (!conf["multiGpu"]) {
+        retry(3) {
+            unstash name: 'srcs'
+            echo """
+            |===== XGBoost CMake build =====
+            |  dockerTarget: ${dockerTarget}
+            |  cmakeOpts   : ${opts}
+            |=========================
+            """.stripMargin('|')
+            // Invoke command inside docker
             sh """
-            ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
-            rm -rf "${distDir}"; mkdir -p "${distDir}/py"
-            cp xgboost "${distDir}"
-            cp -r python-package/dist "${distDir}/py"
-            # Test the wheel for compatibility on a barebones CPU container
-            ${dockerRun} release ${dockerArgs} bash -c " \
-                pip install --user python-package/dist/xgboost-*-none-any.whl && \
-                python -m nose tests/python"
+            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
+            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${test_suite}.sh
             """
+            if (!conf["multiGpu"]) {
+                sh """
+                ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
+                rm -rf "${distDir}"; mkdir -p "${distDir}/py"
+                cp xgboost "${distDir}"
+                cp -r python-package/dist "${distDir}/py"
+                # Test the wheel for compatibility on a barebones CPU container
+                ${dockerRun} release ${dockerArgs} bash -c " \
+                    pip install --user python-package/dist/xgboost-*-none-any.whl && \
+                    python -m nose tests/python"
+                """
+            }
         }
     }
 }
diff --git a/Jenkinsfile-restricted b/Jenkinsfile-restricted
@@ -95,22 +95,24 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
     }
     // Build node - this is returned result
     node(nodeReq) {
-        unstash name: 'srcs'
-        echo """
-        |===== XGBoost CMake build =====
-        |  dockerTarget: ${dockerTarget}
-        |  cmakeOpts   : ${opts}
-        |=========================
-        """.stripMargin('|')
-        // Invoke command inside docker
-        sh """
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
-        ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
-        rm -rf "${distDir}"; mkdir -p "${distDir}/py"
-        cp xgboost "${distDir}"
-        cp -r lib "${distDir}"
-        cp -r python-package/dist "${distDir}/py"
-        """
-        archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
+        retry(3) {
+            unstash name: 'srcs'
+            echo """
+            |===== XGBoost CMake build =====
+            |  dockerTarget: ${dockerTarget}
+            |  cmakeOpts   : ${opts}
+            |=========================
+            """.stripMargin('|')
+            // Invoke command inside docker
+            sh """
+            ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
+            ${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
+            rm -rf "${distDir}"; mkdir -p "${distDir}/py"
+            cp xgboost "${distDir}"
+            cp -r lib "${distDir}"
+            cp -r python-package/dist "${distDir}/py"
+            """
+            archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
+        }
     }
 }