diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index fe6d7b59eb3b2..5a508fa106537 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -37,10 +37,7 @@ apply plugin: 'application'
 archivesBaseName = 'elasticsearch-benchmarks'
 mainClassName = 'org.openjdk.jmh.Main'
 
-// never try to invoke tests on the benchmark project - there aren't any
-check.dependsOn.remove(test)
-// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
-task test(type: Test, overwrite: true)
+test.enabled = false
 
 dependencies {
     compile("org.elasticsearch:elasticsearch:${version}") {
@@ -59,7 +56,6 @@ compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-u
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
-compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 forbiddenApis {
     // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 3fc622ef5aa4f..ea3ed1b6fe958 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -51,16 +51,12 @@ class ClusterFormationTasks {
      *
      * Returns a list of NodeInfo objects for each node in the cluster.
      */
-    static List<NodeInfo> setup(Project project, Task task, ClusterConfiguration config) {
-        if (task.getEnabled() == false) {
-            // no need to add cluster formation tasks if the task won't run!
-            return
-        }
+    static List<NodeInfo> setup(Project project, String prefix, Task runner, ClusterConfiguration config) {
         File sharedDir = new File(project.buildDir, "cluster/shared")
         // first we remove everything in the shared cluster directory to ensure there are no leftovers in repos or anything
         // in theory this should not be necessary but repositories are only deleted in the cluster-state and not on-disk
         // such that snapshots survive failures / test runs and there is no simple way today to fix that.
-        Task cleanup = project.tasks.create(name: "${task.name}#prepareCluster.cleanShared", type: Delete, dependsOn: task.dependsOn.collect()) {
+        Task cleanup = project.tasks.create(name: "${prefix}#prepareCluster.cleanShared", type: Delete, dependsOn: runner.dependsOn.collect()) {
             delete sharedDir
             doLast {
                 sharedDir.mkdirs()
@@ -75,7 +71,7 @@ class ClusterFormationTasks {
             throw new GradleException("bwcVersion must not be null if numBwcNodes is > 0")
         }
         // this is our current version distribution configuration we use for all kinds of REST tests etc.
-        String distroConfigName = "${task.name}_elasticsearchDistro"
+        String distroConfigName = "${prefix}_elasticsearchDistro"
         Configuration currentDistro = project.configurations.create(distroConfigName)
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
         if (config.bwcVersion != null && config.numBwcNodes > 0) {
@@ -89,7 +85,7 @@ class ClusterFormationTasks {
             }
             configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
             for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
-                configureBwcPluginDependency("${task.name}_elasticsearchBwcPlugins", project, entry.getValue(),
+                configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(),
                         project.configurations.elasticsearchBwcPlugins, config.bwcVersion)
             }
             project.configurations.elasticsearchBwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
@@ -104,13 +100,13 @@ class ClusterFormationTasks {
                 elasticsearchVersion = config.bwcVersion
                 distro = project.configurations.elasticsearchBwcDistro
             }
-            NodeInfo node = new NodeInfo(config, i, project, task, elasticsearchVersion, sharedDir)
+            NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
             nodes.add(node)
-            startTasks.add(configureNode(project, task, cleanup, node, distro, nodes.get(0)))
+            startTasks.add(configureNode(project, prefix, runner, cleanup, node, distro, nodes.get(0)))
         }
 
-        Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
-        task.dependsOn(wait)
+        Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks)
+        runner.dependsOn(wait)
 
         return nodes
     }
@@ -150,58 +146,58 @@ class ClusterFormationTasks {
      *
      * @return a task which starts the node.
      */
-    static Task configureNode(Project project, Task task, Object dependsOn, NodeInfo node, Configuration configuration, NodeInfo seedNode) {
+    static Task configureNode(Project project, String prefix, Task runner, Object dependsOn, NodeInfo node, Configuration configuration, NodeInfo seedNode) {
 
         // tasks are chained so their execution order is maintained
-        Task setup = project.tasks.create(name: taskName(task, node, 'clean'), type: Delete, dependsOn: dependsOn) {
+        Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
             doLast {
                 node.cwd.mkdirs()
             }
         }
-        setup = configureCheckPreviousTask(taskName(task, node, 'checkPrevious'), project, setup, node)
-        setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node)
-        setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration)
-        setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode)
+        setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
+        setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
+        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, configuration)
+        setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node, seedNode)
         if (node.config.plugins.isEmpty() == false) {
             if (node.nodeVersion == VersionProperties.elasticsearch) {
-                setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node)
+                setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node)
             } else {
-                setup = configureCopyBwcPluginsTask(taskName(task, node, 'copyBwcPlugins'), project, setup, node)
+                setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node)
             }
         }
 
         // install modules
         for (Project module : node.config.modules) {
             String actionName = pluginTaskName('install', module.name, 'Module')
-            setup = configureInstallModuleTask(taskName(task, node, actionName), project, setup, node, module)
+            setup = configureInstallModuleTask(taskName(prefix, node, actionName), project, setup, node, module)
         }
 
         // install plugins
         for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
             String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
-            setup = configureInstallPluginTask(taskName(task, node, actionName), project, setup, node, plugin.getValue())
+            setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue())
         }
 
         // sets up any extra config files that need to be copied over to the ES instance;
         // its run after plugins have been installed, as the extra config files may belong to plugins
-        setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node)
+        setup = configureExtraConfigFilesTask(taskName(prefix, node, 'extraConfig'), project, setup, node)
 
         // extra setup commands
         for (Map.Entry<String, Object[]> command : node.config.setupCommands.entrySet()) {
             // the first argument is the actual script name, relative to home
             Object[] args = command.getValue().clone()
             args[0] = new File(node.homeDir, args[0].toString())
-            setup = configureExecTask(taskName(task, node, command.getKey()), project, setup, node, args)
+            setup = configureExecTask(taskName(prefix, node, command.getKey()), project, setup, node, args)
         }
 
-        Task start = configureStartTask(taskName(task, node, 'start'), project, setup, node)
+        Task start = configureStartTask(taskName(prefix, node, 'start'), project, setup, node)
 
         if (node.config.daemonize) {
-            Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node)
+            Task stop = configureStopTask(taskName(prefix, node, 'stop'), project, [], node)
             // if we are running in the background, make sure to stop the server when the task completes
-            task.finalizedBy(stop)
+            runner.finalizedBy(stop)
             start.finalizedBy(stop)
         }
         return start
@@ -648,11 +644,11 @@ class ClusterFormationTasks {
     }
 
     /** Returns a unique task name for this task and node configuration */
-    static String taskName(Task parentTask, NodeInfo node, String action) {
+    static String taskName(String prefix, NodeInfo node, String action) {
         if (node.config.numNodes > 1) {
-            return "${parentTask.name}#node${node.nodeNum}.${action}"
+            return "${prefix}#node${node.nodeNum}.${action}"
         } else {
-            return "${parentTask.name}#${action}"
+            return "${prefix}#${action}"
         }
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 73f32961fb33e..59c65c684acee 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -21,7 +21,6 @@ package org.elasticsearch.gradle.test
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
-import org.gradle.api.Task
 
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
@@ -96,17 +95,17 @@ class NodeInfo {
     /** the version of elasticsearch that this node runs */
     String nodeVersion
 
-    /** Creates a node to run as part of a cluster for the given task */
-    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, Task task, String nodeVersion, File sharedDir) {
+    /** Holds node configuration for part of a test cluster. */
+    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, String nodeVersion, File sharedDir) {
         this.config = config
         this.nodeNum = nodeNum
         this.sharedDir = sharedDir
         if (config.clusterName != null) {
             clusterName = config.clusterName
         } else {
-            clusterName = "${task.path.replace(':', '_').substring(1)}"
+            clusterName = project.path.replace(':', '_').substring(1) + '_' + prefix
         }
-        baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}")
+        baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         this.nodeVersion = nodeVersion
         homeDir = homeDir(baseDir, config.distribution, nodeVersion)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
index 51bccb4fe7580..075e8129e6fa8 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
+import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.plugins.JavaBasePlugin
@@ -27,12 +28,15 @@ import org.gradle.api.tasks.Input
 import org.gradle.util.ConfigureUtil
 
 /**
- * Runs integration tests, but first starts an ES cluster,
- * and passes the ES cluster info as parameters to the tests.
+ * A wrapper task around setting up a cluster and running rest tests.
  */
-public class RestIntegTestTask extends RandomizedTestingTask {
+public class RestIntegTestTask extends DefaultTask {
 
-    ClusterConfiguration clusterConfig
+    protected ClusterConfiguration clusterConfig
+
+    protected RandomizedTestingTask runner
+
+    protected Task clusterInit
 
     /** Info about nodes in the integ test cluster. Note this is *not* available until runtime. */
     List<NodeInfo> nodes
@@ -44,35 +48,44 @@ public class RestIntegTestTask extends RandomizedTestingTask {
     public RestIntegTestTask() {
         description = 'Runs rest tests against an elasticsearch cluster.'
         group = JavaBasePlugin.VERIFICATION_GROUP
-        dependsOn(project.testClasses)
-        classpath = project.sourceSets.test.runtimeClasspath
-        testClassesDir = project.sourceSets.test.output.classesDir
-        clusterConfig = new ClusterConfiguration(project)
+        runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
+        super.dependsOn(runner)
+        clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
+        runner.dependsOn(clusterInit)
+        runner.classpath = project.sourceSets.test.runtimeClasspath
+        runner.testClassesDir = project.sourceSets.test.output.classesDir
+        clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
 
         // start with the common test configuration
-        configure(BuildPlugin.commonTestConfig(project))
+        runner.configure(BuildPlugin.commonTestConfig(project))
         // override/add more for rest tests
-        parallelism = '1'
-        include('**/*IT.class')
-        systemProperty('tests.rest.load_packaged', 'false')
+        runner.parallelism = '1'
+        runner.include('**/*IT.class')
+        runner.systemProperty('tests.rest.load_packaged', 'false')
         // we pass all nodes to the rest cluster to allow the clients to round-robin between them
         // this is more realistic than just talking to a single node
-        systemProperty('tests.rest.cluster', "${-> nodes.collect{it.httpUri()}.join(",")}")
-        systemProperty('tests.config.dir', "${-> nodes[0].confDir}")
+        runner.systemProperty('tests.rest.cluster', "${-> nodes.collect{it.httpUri()}.join(",")}")
+        runner.systemProperty('tests.config.dir', "${-> nodes[0].confDir}")
         // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
         // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
         // both as separate sysprops
-        systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
+        runner.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
 
         // copy the rest spec/tests into the test resources
         RestSpecHack.configureDependencies(project)
         project.afterEvaluate {
-            dependsOn(RestSpecHack.configureTask(project, includePackaged))
+            runner.dependsOn(RestSpecHack.configureTask(project, includePackaged))
         }
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
-            nodes = ClusterFormationTasks.setup(project, this, clusterConfig)
+            if (enabled == false) {
+                runner.enabled = false
+                clusterInit.enabled = false
+                return // no need to add cluster formation tasks if the task won't run!
+            }
+            nodes = ClusterFormationTasks.setup(project, "${name}Cluster", runner, clusterConfig)
+            super.dependsOn(runner.finalizedBy)
         }
     }
 
@@ -84,25 +97,16 @@ public class RestIntegTestTask extends RandomizedTestingTask {
         clusterConfig.debug = enabled;
     }
 
-    @Input
-    public void cluster(Closure closure) {
-        ConfigureUtil.configure(closure, clusterConfig)
-    }
-
-    public ClusterConfiguration getCluster() {
-        return clusterConfig
-    }
-
     public List<NodeInfo> getNodes() {
         return nodes
     }
 
     @Override
     public Task dependsOn(Object... dependencies) {
-        super.dependsOn(dependencies)
+        runner.dependsOn(dependencies)
         for (Object dependency : dependencies) {
             if (dependency instanceof Fixture) {
-                finalizedBy(((Fixture)dependency).stopTask)
+                runner.finalizedBy(((Fixture)dependency).stopTask)
             }
         }
         return this
@@ -110,11 +114,16 @@ public class RestIntegTestTask extends RandomizedTestingTask {
 
     @Override
     public void setDependsOn(Iterable<?> dependencies) {
-        super.setDependsOn(dependencies)
+        runner.setDependsOn(dependencies)
         for (Object dependency : dependencies) {
             if (dependency instanceof Fixture) {
-                finalizedBy(((Fixture)dependency).stopTask)
+                runner.finalizedBy(((Fixture)dependency).stopTask)
             }
         }
     }
+
+    @Override
+    public Task mustRunAfter(Object... tasks) {
+        clusterInit.mustRunAfter(tasks)
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
index 47a559efccbd3..f00be89f6aed3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
@@ -43,7 +43,7 @@ public class RestTestPlugin implements Plugin<Project> {
         }
 
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
-        integTest.cluster.distribution = 'zip' // rest tests should run with the real zip
+        integTest.clusterConfig.distribution = 'zip' // rest tests should run with the real zip
         integTest.mustRunAfter(project.precommit)
         project.check.dependsOn(integTest)
     }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
index a71dc59dbf914..a88152d7865ff 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy
@@ -18,7 +18,7 @@ public class RunTask extends DefaultTask {
         clusterConfig.daemonize = false
         clusterConfig.distribution = 'zip'
         project.afterEvaluate {
-            ClusterFormationTasks.setup(project, this, clusterConfig)
+            ClusterFormationTasks.setup(project, name, this, clusterConfig)
         }
     }
 
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 3463487caea2e..e95ccf0932000 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -81,20 +81,16 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each
     }
   }
   // We would like to make sure integ tests for the distribution run after
-  // integ tests for the modules included in the distribution. However, gradle
-  // has a bug where depending on a task with a finalizer can sometimes not make
-  // the finalizer task follow the original task immediately. To work around this,
-  // we make the mustRunAfter the finalizer task itself.
-  // See https://discuss.gradle.org/t/cross-project-task-dependencies-ordering-screws-up-finalizers/13190
+  // integ tests for the modules included in the distribution.
   project.configure(distributions.findAll { it.name != 'integ-test-zip' }) { Project distribution ->
     distribution.afterEvaluate({
       // some integTest tasks will have multiple finalizers
-      distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }.getFinalizedBy()
+      distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }
     })
   }
   // also want to make sure the module's integration tests run after the integ-test-zip (ie rest tests)
   module.afterEvaluate({
-    module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest#stop')
+    module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest')
   })
   restTestExpansions['expected.modules.count'] += 1
 }
@@ -129,14 +125,13 @@ configure(distributions) {
   project.integTest {
     dependsOn project.assemble
     includePackaged project.name == 'integ-test-zip'
-    cluster {
-      distribution = project.name
-    }
     if (project.name != 'integ-test-zip') {
-      // see note above with module mustRunAfter about why integTest#stop is used here
-      mustRunAfter ':distribution:integ-test-zip:integTest#stop'
+      mustRunAfter ':distribution:integ-test-zip:integTest'
     }
   }
+  project.integTestCluster {
+    distribution = project.name
+  }
 
   processTestResources {
     inputs.properties(project(':distribution').restTestExpansions)
diff --git a/docs/build.gradle b/docs/build.gradle
index 529cb18899fd3..36727b12e5097 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -129,27 +129,25 @@ buildRestTests.expectedUnconvertedCandidates = [
   'reference/search/request/inner-hits.asciidoc',
 ]
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-    /* Enable regexes in painless so our tests don't complain about example
-     * snippets that use them. */
-    setting 'script.painless.regex.enabled', 'true'
-    Closure configFile = {
-      extraConfigFile it, "src/test/cluster/config/$it"
-    }
-    configFile 'scripts/my_script.painless'
-    configFile 'scripts/my_init_script.painless'
-    configFile 'scripts/my_map_script.painless'
-    configFile 'scripts/my_combine_script.painless'
-    configFile 'scripts/my_reduce_script.painless'
-    configFile 'userdict_ja.txt'
-    configFile 'KeywordTokenizer.rbbi'
-    // Whitelist reindexing from the local node so we can test it.
-    setting 'reindex.remote.whitelist', '127.0.0.1:*'
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
+  /* Enable regexes in painless so our tests don't complain about example
+   * snippets that use them. */
+  setting 'script.painless.regex.enabled', 'true'
+  Closure configFile = {
+    extraConfigFile it, "src/test/cluster/config/$it"
   }
+  configFile 'scripts/my_script.painless'
+  configFile 'scripts/my_init_script.painless'
+  configFile 'scripts/my_map_script.painless'
+  configFile 'scripts/my_combine_script.painless'
+  configFile 'scripts/my_reduce_script.painless'
+  configFile 'userdict_ja.txt'
+  configFile 'KeywordTokenizer.rbbi'
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
 // Build the cluster with all plugins
@@ -161,10 +159,8 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each {
     return
   }
   subproj.afterEvaluate { // need to wait until the project has been configured
-    integTest {
-      cluster {
-        plugin subproj.path
-      }
+    integTestCluster {
+      plugin subproj.path
     }
   }
 }
diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle
index ee8c330193461..23286edd28aea 100644
--- a/modules/lang-expression/build.gradle
+++ b/modules/lang-expression/build.gradle
@@ -35,8 +35,6 @@ dependencyLicenses {
   mapping from: /asm-.*/, to: 'asm'
 }
 
-integTest {
-  cluster {
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.max_compilations_per_minute', '1000'
 }
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 8fed78aca3249..8b695fbf574bc 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -27,11 +27,9 @@ dependencies {
   compile "com.github.spullara.mustache.java:compiler:0.9.3"
 }
 
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-    setting 'script.stored', 'true'
-    setting 'script.max_compilations_per_minute', '1000'
-    setting 'path.scripts', "${project.buildDir}/resources/test/templates"
-  }
+integTestCluster {
+  setting 'script.inline', 'true'
+  setting 'script.stored', 'true'
+  setting 'script.max_compilations_per_minute', '1000'
+  setting 'path.scripts', "${project.buildDir}/resources/test/templates"
 }
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index dc56039afeedd..c9e013d8c6d7f 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -47,10 +47,8 @@ dependencies {
 ant.references['regenerate.classpath'] = new Path(ant.project, configurations.regenerate.asPath)
 ant.importBuild 'ant.xml'
 
-integTest {
-  cluster {
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.max_compilations_per_minute', '1000'
 }
 
 /* Build Javadoc for the Java classes in Painless's public API that are in the
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index 0b222c1a3cd71..eba8b9646177d 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -25,11 +25,9 @@ esplugin {
   hasClientJar = true
 }
 
-integTest {
-  cluster {
-    // Whitelist reindexing from the local node so we can test it.
-    setting 'reindex.remote.whitelist', '127.0.0.1:*'
-  }
+integTestCluster {
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
 run {
diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle
index dc769e3c1f300..7008111ca9c54 100644
--- a/modules/repository-url/build.gradle
+++ b/modules/repository-url/build.gradle
@@ -22,8 +22,6 @@ esplugin {
     classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
 }
 
-integTest {
-    cluster {
-        setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-    }
-}
\ No newline at end of file
+integTestCluster {
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+}
diff --git a/plugins/discovery-file/build.gradle b/plugins/discovery-file/build.gradle
index 3b78f06505e80..c6d622f724809 100644
--- a/plugins/discovery-file/build.gradle
+++ b/plugins/discovery-file/build.gradle
@@ -39,7 +39,7 @@ task setupSeedNodeAndUnicastHostsFile(type: DefaultTask) {
 // for unicast discovery
 ClusterConfiguration config = new ClusterConfiguration(project)
 config.clusterName = 'discovery-file-test-cluster'
-List<NodeInfo> nodes = ClusterFormationTasks.setup(project, setupSeedNodeAndUnicastHostsFile, config)
+List<NodeInfo> nodes = ClusterFormationTasks.setup(project, 'initialCluster', setupSeedNodeAndUnicastHostsFile, config)
 File srcUnicastHostsFile = file('build/cluster/unicast_hosts.txt')
 
 // write the unicast_hosts.txt file to a temporary location to be used by the second cluster
@@ -49,11 +49,13 @@ setupSeedNodeAndUnicastHostsFile.doLast {
 }
 
 // second cluster, which will connect to the first via the unicast_hosts.txt file
+integTestCluster {
+    clusterName = 'discovery-file-test-cluster'
+    extraConfigFile 'discovery-file/unicast_hosts.txt', srcUnicastHostsFile
+}
+
+integTestRunner.finalizedBy ':plugins:discovery-file:initialCluster#stop'
+
 integTest {
     dependsOn(setupSeedNodeAndUnicastHostsFile)
-    cluster {
-        clusterName = 'discovery-file-test-cluster'
-        extraConfigFile 'discovery-file/unicast_hosts.txt', srcUnicastHostsFile
-    }
-    finalizedBy ':plugins:discovery-file:setupSeedNodeAndUnicastHostsFile#stop'
 }
diff --git a/plugins/ingest-user-agent/build.gradle b/plugins/ingest-user-agent/build.gradle
index ec599874d155b..2b2669a40042b 100644
--- a/plugins/ingest-user-agent/build.gradle
+++ b/plugins/ingest-user-agent/build.gradle
@@ -22,8 +22,6 @@ esplugin {
   classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin'
 }
 
-integTest {
-  cluster {
-    extraConfigFile 'ingest-user-agent/test-regexes.yaml', 'test/test-regexes.yaml'
-  }
-}
\ No newline at end of file
+integTestCluster {
+  extraConfigFile 'ingest-user-agent/test-regexes.yaml', 'test/test-regexes.yaml'
+}
diff --git a/plugins/jvm-example/build.gradle b/plugins/jvm-example/build.gradle
index c5828d9c86df5..3753d3ae83101 100644
--- a/plugins/jvm-example/build.gradle
+++ b/plugins/jvm-example/build.gradle
@@ -43,6 +43,8 @@ task exampleFixture(type: org.elasticsearch.gradle.test.Fixture) {
 
 integTest {
   dependsOn exampleFixture
+}
+integTestRunner {
   systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"
 }
 
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 125b7bdd6c5c3..a2c3df17d80a1 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -41,10 +41,8 @@ thirdPartyAudit.excludes = [
   'org.slf4j.LoggerFactory',
 ]
 
-integTest {
-  cluster {
-    setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
-    setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
-    setting 'script.stored', 'true'
-  }
-}
\ No newline at end of file
+integTestCluster {
+  setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
+  setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
+  setting 'script.stored', 'true'
+}
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index e85599229a81d..90dd2f2d8bbf6 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -35,12 +35,13 @@ apply plugin: 'elasticsearch.rest-test'
  */
 integTest {
   includePackaged = true
-  cluster {
-    numNodes = 4
-    numBwcNodes = 2
-    bwcVersion = "5.4.0-SNAPSHOT"
-    setting 'logger.org.elasticsearch', 'DEBUG'
-  }
+}
+
+integTestCluster {
+  numNodes = 4
+  numBwcNodes = 2
+  bwcVersion = "5.4.0-SNAPSHOT"
+  setting 'logger.org.elasticsearch', 'DEBUG'
 }
 
 repositories {
diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle
index 385c547cf6dea..c6e34963af9dc 100644
--- a/qa/multi-cluster-search/build.gradle
+++ b/qa/multi-cluster-search/build.gradle
@@ -23,25 +23,33 @@ apply plugin: 'elasticsearch.standalone-test'
 
 task remoteClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
-  cluster {
-    distribution = 'zip'
-    numNodes = 2
-    clusterName = 'remote-cluster'
-    setting 'search.remote.connect', false
-  }
+}
+
+remoteClusterTestCluster {
+  distribution = 'zip'
+  numNodes = 2
+  clusterName = 'remote-cluster'
+  setting 'search.remote.connect', false
+}
+
+remoteClusterTestRunner {
   systemProperty 'tests.rest.suite', 'remote_cluster'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
-  dependsOn(remoteClusterTest)
-  cluster {
-    distribution = 'zip'
-    setting 'search.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
-    setting 'search.remote.connections_per_cluster', 1
-    setting 'search.remote.connect', true
-  }
+  dependsOn(remoteClusterTestRunner)
+}
+
+mixedClusterTestCluster {
+  distribution = 'zip'
+  setting 'search.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
+  setting 'search.remote.connections_per_cluster', 1
+  setting 'search.remote.connect', true
+}
+
+mixedClusterTestRunner {
   systemProperty 'tests.rest.suite', 'multi_cluster'
-  finalizedBy 'remoteClusterTest#node0.stop','remoteClusterTest#node1.stop'
+  finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop'
 }
 
 task integTest {
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 1d79bb404fdd9..f4cde58d75577 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -23,43 +23,55 @@ apply plugin: 'elasticsearch.standalone-test'
 
 task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
-  cluster {
-    distribution = 'zip'
-    bwcVersion = '5.4.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop
-    numBwcNodes = 2
-    numNodes = 2
-    clusterName = 'rolling-upgrade'
-    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-    setting 'http.content_type.required', 'true'
-  }
+}
+
+oldClusterTestCluster {
+  distribution = 'zip'
+  bwcVersion = '5.4.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop
+  numBwcNodes = 2
+  numNodes = 2
+  clusterName = 'rolling-upgrade'
+  setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+  setting 'http.content_type.required', 'true'
+}
+
+oldClusterTestRunner {
   systemProperty 'tests.rest.suite', 'old_cluster'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
-  dependsOn(oldClusterTest, 'oldClusterTest#node1.stop')
-  cluster {
-    distribution = 'zip'
-    clusterName = 'rolling-upgrade'
-    unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
-    dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
-    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-  }
+  dependsOn(oldClusterTestRunner, 'oldClusterTestCluster#node1.stop')
+}
+
+mixedClusterTestCluster {
+  distribution = 'zip'
+  clusterName = 'rolling-upgrade'
+  unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
+  dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+  setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+}
+
+mixedClusterTestRunner {
   systemProperty 'tests.rest.suite', 'mixed_cluster'
-  finalizedBy 'oldClusterTest#node0.stop'
+  finalizedBy 'oldClusterTestCluster#node0.stop'
 }
 
 task upgradedClusterTest(type: RestIntegTestTask) {
-  dependsOn(mixedClusterTest, 'oldClusterTest#node0.stop')
-  cluster {
-    distribution = 'zip'
-    clusterName = 'rolling-upgrade'
-    unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
-    dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
-    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-  }
+  dependsOn(mixedClusterTestRunner, 'oldClusterTestCluster#node0.stop')
+}
+
+upgradedClusterTestCluster {
+  distribution = 'zip'
+  clusterName = 'rolling-upgrade'
+  unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
+  dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
+  setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+}
+
+upgradedClusterTestRunner {
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
   // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
-  finalizedBy 'mixedClusterTest#stop'
+  finalizedBy 'mixedClusterTestCluster#stop'
 }
 
 task integTest {
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index 4c4d9c2da1259..cf3ca9a713f96 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -24,8 +24,6 @@ dependencies {
     testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
 }
 
-integTest {
-    cluster {
-        setting 'node.ingest', 'false'
-    }
+integTestCluster {
+    setting 'node.ingest', 'false'
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index 2cfa3af434ed9..1608ce9c37c25 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -28,10 +28,8 @@ dependencies {
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
 
-integTest {
-    cluster {
-        plugin ':plugins:ingest-geoip'
-        setting 'script.inline', 'true'
-        setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
-    }
+integTestCluster {
+    plugin ':plugins:ingest-geoip'
+    setting 'script.inline', 'true'
+    setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
 }
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index fc196fd52a4a4..5df77bd0d9513 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -22,7 +22,8 @@ apply plugin: 'elasticsearch.rest-test'
 
 integTest {
   includePackaged = true
-  cluster {
-    numNodes = 2
-  }
+}
+
+integTestCluster {
+  numNodes = 2
 }
diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle
index 6fd722e409c87..d60216dad194f 100644
--- a/qa/smoke-test-plugins/build.gradle
+++ b/qa/smoke-test-plugins/build.gradle
@@ -24,10 +24,8 @@ apply plugin: 'elasticsearch.rest-test'
 
 ext.pluginsCount = 0
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
-  integTest {
-    cluster {
-      plugin subproj.path
-    }
+  integTestCluster {
+    plugin subproj.path
   }
   pluginsCount += 1
 }
diff --git a/qa/smoke-test-reindex-with-painless/build.gradle b/qa/smoke-test-reindex-with-painless/build.gradle
index 7092c0a7b487a..b32f4ee80bef9 100644
--- a/qa/smoke-test-reindex-with-painless/build.gradle
+++ b/qa/smoke-test-reindex-with-painless/build.gradle
@@ -20,8 +20,6 @@
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-integTest {
-  cluster {
-    setting 'script.max_compilations_per_minute', '1000'
-  }
+integTestCluster {
+  setting 'script.max_compilations_per_minute', '1000'
 }
diff --git a/qa/smoke-test-tribe-node/build.gradle b/qa/smoke-test-tribe-node/build.gradle
index 94789b17fdbd9..4ddb178068d2f 100644
--- a/qa/smoke-test-tribe-node/build.gradle
+++ b/qa/smoke-test-tribe-node/build.gradle
@@ -24,47 +24,32 @@ import org.elasticsearch.gradle.test.NodeInfo
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-List<NodeInfo> oneNodes
 
-task setupClusterOne(type: DefaultTask) {
-  mustRunAfter(precommit)
-  ClusterConfiguration configOne = new ClusterConfiguration(project)
-  configOne.clusterName = 'one'
-  configOne.setting('node.name', 'one')
-  oneNodes = ClusterFormationTasks.setup(project, setupClusterOne, configOne)
-}
-
-List<NodeInfo> twoNodes
+ClusterConfiguration configOne = new ClusterConfiguration(project)
+configOne.clusterName = 'one'
+configOne.setting('node.name', 'one')
+List<NodeInfo> oneNodes = ClusterFormationTasks.setup(project, 'clusterOne', integTestRunner, configOne)
 
-task setupClusterTwo(type: DefaultTask) {
-  mustRunAfter(precommit)
-  ClusterConfiguration configTwo = new ClusterConfiguration(project)
-  configTwo.clusterName = 'two'
-  configTwo.setting('node.name', 'two')
-  twoNodes = ClusterFormationTasks.setup(project, setupClusterTwo, configTwo)
-}
+ClusterConfiguration configTwo = new ClusterConfiguration(project)
+configTwo.clusterName = 'two'
+configTwo.setting('node.name', 'two')
+List<NodeInfo> twoNodes = ClusterFormationTasks.setup(project, 'clusterTwo', integTestRunner, configTwo)
 
-integTest {
-  dependsOn(setupClusterOne, setupClusterTwo)
-  cluster {
-    // tribe nodes had a bug where if explicit ports was specified for the tribe node, the dynamic socket permissions that were applied
-    // would not account for the fact that the internal node client needed to bind to sockets too; thus, we use explicit port ranges to
-    // ensure that the code that fixes this bug is exercised
-    setting 'http.port', '40200-40249'
-    setting 'transport.tcp.port', '40300-40349'
-    setting 'node.name', 'quest'
-    setting 'tribe.one.cluster.name', 'one'
-    setting 'tribe.one.discovery.zen.ping.unicast.hosts', "'${-> oneNodes.get(0).transportUri()}'"
-    setting 'tribe.one.http.enabled', 'true'
-    setting 'tribe.one.http.port', '40250-40299'
-    setting 'tribe.one.transport.tcp.port', '40350-40399'
-    setting 'tribe.two.cluster.name', 'two'
-    setting 'tribe.two.discovery.zen.ping.unicast.hosts', "'${-> twoNodes.get(0).transportUri()}'"
-    setting 'tribe.two.http.enabled', 'true'
-    setting 'tribe.two.http.port', '40250-40299'
-    setting 'tribe.two.transport.tcp.port', '40250-40399'
-  }
-  // need to kill the standalone nodes here
-  finalizedBy 'setupClusterOne#stop'
-  finalizedBy 'setupClusterTwo#stop'
+integTestCluster {
+  // tribe nodes had a bug where if explicit ports was specified for the tribe node, the dynamic socket permissions that were applied
+  // would not account for the fact that the internal node client needed to bind to sockets too; thus, we use explicit port ranges to
+  // ensure that the code that fixes this bug is exercised
+  setting 'http.port', '40200-40249'
+  setting 'transport.tcp.port', '40300-40349'
+  setting 'node.name', 'quest'
+  setting 'tribe.one.cluster.name', 'one'
+  setting 'tribe.one.discovery.zen.ping.unicast.hosts', "'${-> oneNodes.get(0).transportUri()}'"
+  setting 'tribe.one.http.enabled', 'true'
+  setting 'tribe.one.http.port', '40250-40299'
+  setting 'tribe.one.transport.tcp.port', '40350-40399'
+  setting 'tribe.two.cluster.name', 'two'
+  setting 'tribe.two.discovery.zen.ping.unicast.hosts', "'${-> twoNodes.get(0).transportUri()}'"
+  setting 'tribe.two.http.enabled', 'true'
+  setting 'tribe.two.http.port', '40250-40299'
+  setting 'tribe.two.transport.tcp.port', '40250-40399'
 }