Prevent concurrent integration tests

zanata · Apr 10, 2017 · 62746a9 · 62746a9
1 parent 215d0bf
commit 62746a9
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 26 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -4,24 +4,26 @@
  * Jenkinsfile for zanata-platform
  */
 
-import groovy.transform.Field
-
 // Import pipeline library for utility methods & classes:
 // ansicolor(), Notifier, PullRequests, Strings
 @Library('zanata-pipeline-library@master')
 import org.zanata.jenkins.Notifier
 import org.zanata.jenkins.PullRequests
 
+import groovy.transform.Field
+
 PullRequests.ensureJobDescription(env, manager, steps)
 
 @Field
 def notify
 // initialiser must be run separately (bindings not available during compilation phase)
 notify = new Notifier(env, steps)
 
-// we can't set this value yet, because we need a node to look at the environment
+// we can't set these values yet, because we need a node to look at the environment
 @Field
 def defaultNodeLabel
+@Field
+def jobName
 
 // Define project properties: general properties for the Pipeline-defined jobs.
 // 1. discard old artifacts and build logs
@@ -32,6 +34,8 @@ def defaultNodeLabel
 node {
   echo "running on node ${env.NODE_NAME}"
   defaultNodeLabel = env.DEFAULT_NODE ?: 'master || !master'
+  // eg github-zanata-org/zanata-platform/update-Jenkinsfile
+  jobName = env.JOB_NAME
   def projectProperties = [
     [
       $class: 'BuildDiscarderProperty',
@@ -91,6 +95,9 @@ timestamps {
       try {
 
         stage('Checkout') {
+          // The first milestone step starts tracking concurrent build order
+          milestone()
+
           // notify methods send instant messages about the build progress
           notify.started()
 
@@ -178,31 +185,44 @@ timestamps {
 
   // if the build is still green:
   if (currentBuild.result == null) {
-    // NB all the parallel tasks must be in one stage, because you can't use `stage` inside `parallel`.
-    // See https://issues.jenkins-ci.org/browse/JENKINS-34696 and https://issues.jenkins-ci.org/browse/JENKINS-33185
-    stage('Integration tests') {
-      // define tasks which will run in parallel
-      def tasks = [
-              "WILDFLY" : { integrationTests('wildfly8') },
-              "JBOSSEAP": { integrationTests('jbosseap6') },
-              // abort other tasks (for faster feedback) as soon as one fails
-              // disabled; not currently handled by pipeline-unit
+    // Only one build (per Jenkins job, ie per branch) is allowed to run
+    // integration tests at a time.
+    // Newer builds are pulled off the queue first. When a build reaches the
+    // milestone at the beginning of the lock, all jobs started prior to the
+    // current build that are still waiting for the lock will be aborted.
+    // Note that the milestone is at the beginning of the lock, because we
+    // want to abort concurrent tests before they are executed.
+    // Ref: https://jenkins.io/blog/2016/10/16/stage-lock-milestone/
+    // You can probably allow increased concurrency by defining more locks
+    // in Jenkins system config. q.v. JENKINS-42339
+    lock(resource: jobName, inversePrecedence: true, quantity: 1) {
+      milestone()
+      // NB all the parallel tasks must be in one stage, because you can't use `stage` inside `parallel`.
+      // See https://issues.jenkins-ci.org/browse/JENKINS-34696 and https://issues.jenkins-ci.org/browse/JENKINS-33185
+      stage('Integration tests') {
+        // define tasks which will run in parallel
+        def tasks = [
+                "WILDFLY" : { integrationTests('wildfly8') },
+                "JBOSSEAP": { integrationTests('jbosseap6') },
+                // abort other tasks (for faster feedback) as soon as one fails
+                // disabled; not currently handled by pipeline-unit
 //              failFast: true
-      ]
-      // run integration test tasks in parallel
-      parallel tasks
+        ]
+        // run integration test tasks in parallel
+        parallel tasks
 
-      // if the build is *still* green after running integration tests:
-      if (currentBuild.result == null) {
-        echo 'marking build as successful'
-        currentBuild.result = 'SUCCESS'
-      }
+        // if the build is *still* green after running integration tests:
+        if (currentBuild.result == null) {
+          echo 'marking build as successful'
+          currentBuild.result = 'SUCCESS'
+        }
 
-      // TODO notify finish
-      // TODO in case of failure, notify culprits via IRC, email and/or Rocket.Chat
-      // https://wiki.jenkins-ci.org/display/JENKINS/Email-ext+plugin#Email-extplugin-PipelineExamples
-      // http://stackoverflow.com/a/39535424/14379
-      // IRC: https://issues.jenkins-ci.org/browse/JENKINS-33922
+        // TODO notify finish
+        // TODO in case of failure, notify culprits via IRC, email and/or Rocket.Chat
+        // https://wiki.jenkins-ci.org/display/JENKINS/Email-ext+plugin#Email-extplugin-PipelineExamples
+        // http://stackoverflow.com/a/39535424/14379
+        // IRC: https://issues.jenkins-ci.org/browse/JENKINS-33922
+      }
     }
   }
 }

diff --git a/build-tools/src/test/java/TestJenkinsfile.java b/build-tools/src/test/java/TestJenkinsfile.java
@@ -45,8 +45,11 @@ public void setUp() throws Exception {
         getHelper().registerAllowedMethod("archive", ImmutableList.of(Object.class), null);
         getHelper().registerAllowedMethod("hipchatSend", ImmutableList.of(Map.class), null);
         getHelper().registerAllowedMethod("junit", ImmutableList.of(Map.class), null);
-        getHelper().registerAllowedMethod("timestamps", ImmutableList.of(Closure.class), null);
+        getHelper().registerAllowedMethod("lock", ImmutableList.of(Map.class, Closure.class), null);
+        getHelper().registerAllowedMethod("lock", ImmutableList.of(String.class, Closure.class), null);
+        getHelper().registerAllowedMethod("milestone", ImmutableList.of(), null);
         getHelper().registerAllowedMethod("stash", ImmutableList.of(Map.class), null);
+        getHelper().registerAllowedMethod("timestamps", ImmutableList.of(Closure.class), null);
         getHelper().registerAllowedMethod("unstash", ImmutableList.of(Map.class), null);
         getHelper().registerAllowedMethod("unstash", ImmutableList.of(String.class), null);
         getHelper().registerAllowedMethod("withEnv", ImmutableList.of(List.class, Closure.class), null);